Switch to simpler fix

This reverts commit b501ab1532.

.github/dependabot.yml

@@ -7,10 +7,3 @@ updates:
     commit-message:
       prefix: "chore(cargo)"
     open-pull-requests-limit: 50
-
-  # Keep GitHub Actions up to date.
-  # <https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot>
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.82.0
+      RUSTUP_TOOLCHAIN: 1.78.0
     steps:
       - uses: actions/checkout@v4
         with:
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           show-progress: false
-      - uses: EmbarkStudios/cargo-deny-action@v2
+      - uses: EmbarkStudios/cargo-deny-action@v1
         with:
           arguments: --all-features --workspace
           command: check
@@ -95,11 +95,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.82.0
+            rust: 1.78.0
           - os: windows-latest
-            rust: 1.82.0
+            rust: 1.78.0
           - os: macos-latest
-            rust: 1.82.0
+            rust: 1.78.0
 
           # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest
@@ -125,6 +125,9 @@ jobs:
       - name: Tests
         env:
           RUST_BACKTRACE: 1
+
+          # Workaround for <https://github.com/nextest-rs/nextest/issues/1493>.
+          RUSTUP_WINDOWS_PATH_ADD_BIN: 1
         run: cargo nextest run --workspace
 
       - name: Doc-Tests
@@ -211,9 +214,9 @@ jobs:
         include:
           # Currently used Rust version.
           - os: ubuntu-latest
-            python: 3.13
+            python: 3.12
           - os: macos-latest
-            python: 3.13
+            python: 3.12
 
           # PyPy tests
           - os: ubuntu-latest
@@ -249,7 +252,7 @@ jobs:
 
       - name: Run python tests
         env:
-          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
           DCC_RS_TARGET: debug
           DCC_RS_DEV: ${{ github.workspace }}
         working-directory: python
@@ -263,11 +266,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            python: 3.13
+            python: 3.12
           - os: macos-latest
-            python: 3.13
+            python: 3.12
           - os: windows-latest
-            python: 3.13
+            python: 3.12
 
           # PyPy tests
           - os: ubuntu-latest
@@ -314,6 +317,6 @@ jobs:
 
       - name: Run deltachat-rpc-client tests
         env:
-          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
         working-directory: deltachat-rpc-client
         run: tox -e py

.github/workflows/deltachat-rpc-server.yml

@@ -401,6 +401,6 @@ jobs:
         working-directory: deltachat-rpc-server/npm-package
         run: |
           ls -lah platform_package
-          for platform in *.tgz; do npm publish --provenance "$platform" --access public; done
+          for platform in *.tgz; do npm publish --provenance "$platform"; done
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/dependabot.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v2.2.0
+        uses: dependabot/fetch-metadata@v1.1.1
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Approve a PR

.github/workflows/jsonrpc-client-npm-package.yml

@@ -33,6 +33,6 @@ jobs:
 
       - name: Publish
         working-directory: deltachat-jsonrpc/typescript
-        run: npm publish --provenance deltachat-jsonrpc-client-* --access public
+        run: npm publish --provenance deltachat-jsonrpc-client-*
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/jsonrpc.yml

@@ -33,7 +33,7 @@ jobs:
         working-directory: deltachat-jsonrpc/typescript
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
       - name: make sure websocket server version still builds
         working-directory: deltachat-jsonrpc
         run: cargo build --bin deltachat-jsonrpc-server --features webserver

.github/workflows/nix.yml

@@ -1,22 +0,0 @@
-name: Test Nix flake
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-jobs:
-  format:
-    name: check flake formatting
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          show-progress: false
-      - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix fmt
-
-      # Check that formatting does not change anything.
-      - run: git diff --exit-code

.github/workflows/node-docs.yml

@@ -31,7 +31,7 @@ jobs:
           mv docs js
 
       - name: Upload
-        uses: horochx/deploy-via-scp@1.1.0
+        uses: horochx/deploy-via-scp@v1.0.1
         with:
           user: ${{ secrets.USERNAME }}
           key: ${{ secrets.KEY }}

.github/workflows/node-tests.yml

@@ -64,5 +64,5 @@ jobs:
         working-directory: node
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
           NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

.github/workflows/upload-docs.yml

@@ -74,7 +74,7 @@ jobs:
           show-progress: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - name: Use Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v2
         with:
           node-version: '18'
       - name: npm install

.gitignore

@@ -50,4 +50,4 @@ result
 
 # direnv
 .envrc
-.direnv
+.direnv

CMakeLists.txt

@@ -27,7 +27,7 @@ add_custom_command(
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release --features jsonrpc
+	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release --no-default-features --features jsonrpc
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
 )
 

CONTRIBUTING.md

@@ -32,66 +32,6 @@ on the contributing page: <https://github.com/deltachat/deltachat-core-rust/cont
 We format the code using `rustfmt`. Run `cargo fmt` prior to committing the code.
 Run `scripts/clippy.sh` to check the code for common mistakes with [Clippy].
 
-### SQL
-
-Multi-line SQL statements should be formatted using string literals,
-for example
-```
-    sql.execute(
-        "CREATE TABLE messages (
-id INTEGER PRIMARY KEY AUTOINCREMENT,
-text TEXT DEFAULT '' NOT NULL -- message text
-) STRICT",
-    )
-    .await?;
-```
-
-Do not use macros like [`concat!`](https://doc.rust-lang.org/std/macro.concat.html)
-or [`indoc!](https://docs.rs/indoc).
-Do not escape newlines like this:
-```
-    sql.execute(
-        "CREATE TABLE messages ( \
-id INTEGER PRIMARY KEY AUTOINCREMENT, \
-text TEXT DEFAULT '' NOT NULL \
-) STRICT",
-    )
-    .await?;
-```
-Escaping newlines
-is prone to errors like this if space before backslash is missing:
-```
-"SELECT foo\
- FROM bar"
-```
-Literal above results in `SELECT fooFROM bar` string.
-This style also does not allow using `--` comments.
-
----
-
-Declare new SQL tables with [`STRICT`](https://sqlite.org/stricttables.html) keyword
-to make SQLite check column types.
-
-Declare primary keys with [`AUTOINCREMENT`](https://www.sqlite.org/autoinc.html) keyword.
-This avoids reuse of the row IDs and can avoid dangerous bugs
-like forwarding wrong message because the message was deleted
-and another message took its row ID.
-
-Declare all new columns as `NOT NULL`
-and set the `DEFAULT` value if it is optional so the column can be skipped in `INSERT` statements.
-Dealing with `NULL` values both in SQL and in Rust is tricky and we try to avoid it.
-If column is already declared without `NOT NULL`, use `IFNULL` function to provide default value when selecting it.
-Use `HAVING COUNT(*) > 0` clause
-to [prevent aggregate functions such as `MIN` and `MAX` from returning `NULL`](https://stackoverflow.com/questions/66527856/aggregate-functions-max-etc-return-null-instead-of-no-rows).
-
-Don't delete unused columns too early, but maybe after several months/releases, unused columns are
-still used by older versions, so deleting them breaks downgrading the core or importing a backup in
-an older version. Also don't change the column type, consider adding a new column with another name
-instead. Finally, never change column semantics, this is especially dangerous because the `STRICT`
-keyword doesn't help here.
-
-### Commit messages
-
 Commit messages follow the [Conventional Commits] notation.
 We use [git-cliff] to generate the changelog from commit messages before the release.
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.149.0"
+version = "1.138.5"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.77"
@@ -34,93 +34,100 @@ strip = true
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 deltachat-time = { path = "./deltachat-time" }
-deltachat-contact-tools = { workspace = true }
+deltachat-contact-tools = { path = "./deltachat-contact-tools" }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
-async-broadcast = "0.7.1"
-async-channel = { workspace = true }
-async-imap = { version = "0.10.2", default-features = false, features = ["runtime-tokio", "compress"] }
+async-broadcast = "0.7.0"
+async-channel = "2.2.1"
+async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
-async_zip = { version = "0.0.17", default-features = false, features = ["deflate", "tokio-fs"] }
-base64 = { workspace = true }
-brotli = { version = "7", default-features=false, features = ["std"] }
-bytes = "1"
-chrono = { workspace = true, features = ["alloc", "clock", "std"] }
+async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
+backtrace = "0.3"
+base64 = "0.22"
+brotli = { version = "6", default-features=false, features = ["std"] }
+chrono = { workspace = true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 fast-socks5 = "0.9"
 fd-lock = "4"
-futures-lite = { workspace = true }
-futures = { workspace = true }
+futures = "0.3"
+futures-lite = "2.3.0"
 hex = "0.4.0"
-hickory-resolver = "=0.25.0-alpha.2"
-http-body-util = "0.1.2"
+hickory-resolver = "0.24"
 humansize = "2"
-hyper = "1"
-hyper-util = "0.1.10"
-image = { version = "0.25.4", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh-gossip = { version = "0.28.1", default-features = false, features = ["net"] }
-iroh-net = { version = "0.28.1", default-features = false }
-kamadak-exif = "0.6.0"
+image = { version = "0.25.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh_old = { version = "0.4.2", default-features = false, package = "iroh"}
+iroh-net = { git = "https://github.com/link2xt/iroh", branch="link2xt/keep-connection" }
+iroh-gossip = { git = "https://github.com/link2xt/iroh", branch="link2xt/keep-connection", features = ["net"] }
+quinn = "0.10.0"
+kamadak-exif = "0.5.3"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = { workspace = true }
+libc = "0.2"
 mailparse = "0.15"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
-num-traits = { workspace = true }
+num-traits = "0.2"
 once_cell = { workspace = true }
-parking_lot = "0.12"
 percent-encoding = "2.3"
-pgp = { version = "0.14.0", default-features = false }
-pin-project = "1"
+parking_lot = "0.12"
+pgp = { version = "0.11", default-features = false }
+pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
-quick-xml = "0.37"
+quick-xml = "0.31"
 quoted_printable = "0.5"
-rand = { workspace = true }
+rand = "0.8"
 regex = { workspace = true }
+reqwest = { version = "0.11.27", features = ["json"] }
 rusqlite = { workspace = true, features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustls-pki-types = "1.10.0"
-rustls = { version = "0.23.14", default-features = false }
-sanitize-filename = { workspace = true }
-serde_json = { workspace = true }
-serde_urlencoded = "0.7.1"
-serde = { workspace = true, features = ["derive"] }
+sanitize-filename = "0.5"
+serde_json = "1"
+serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
-shadowsocks = { version = "1.21.0", default-features = false, features = ["aead-cipher-2022"] }
+sha2 = "0.10"
 smallvec = "1.13.2"
 strum = "0.26"
 strum_macros = "0.26"
 tagger = "4.3.4"
 textwrap = "0.16.1"
-thiserror = { workspace = true }
+thiserror = "1"
+tokio = { version = "1.37.0", features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
-tokio-rustls = { version = "0.26.0", default-features = false }
-tokio-stream = { version = "0.1.16", features = ["fs"] }
+tokio-stream = { version = "0.1.15", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
-tokio-util = { workspace = true }
-tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
+tokio-util = "0.7.9"
 toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
-webpki-roots = "0.26.6"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+# Pin OpenSSL to 3.1 releases.
+# OpenSSL 3.2 has a regression tracked at <https://github.com/openssl/openssl/issues/23376>
+# which results in broken `deltachat-rpc-server` binaries when cross-compiled using Zig toolchain.
+# See <https://github.com/deltachat/deltachat-core-rust/issues/5206> for Delta Chat issue.
+# According to <https://www.openssl.org/policies/releasestrat.html>
+# 3.1 branch will be supported until 2025-03-14.
+openssl-src = "~300.1"
+tracing = "0.1.40"
+
 
 [dev-dependencies]
-anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.
+ansi_term = "0.12.0"
+anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = { workspace = true }
-log = { workspace = true }
-nu-ansi-term = { workspace = true }
-pretty_assertions = "1.4.1"
+futures-lite = "2.3.0"
+log = "0.4"
+pretty_env_logger = "0.5"
 proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = { workspace = true }
+tempfile = "3"
 testdir = "0.9.0"
-tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
+tokio = { version = "1.37.0", features = ["parking_lot", "rt-multi-thread", "macros"] }
+pretty_assertions = "1.3.0"
 
 [workspace]
 members = [
@@ -165,38 +172,16 @@ harness = false
 
 [workspace.dependencies]
 anyhow = "1"
-async-channel = "2.3.1"
-base64 = "0.22"
-chrono = { version = "0.4.38", default-features = false }
-deltachat-contact-tools = { path = "deltachat-contact-tools" }
-deltachat-jsonrpc = { path = "deltachat-jsonrpc", default-features = false }
-deltachat = { path = ".", default-features = false }
-futures = "0.3.31"
-futures-lite = "2.4.0"
-libc = "0.2"
-log = "0.4"
-nu-ansi-term = "0.46"
-num-traits = "0.2"
-once_cell = "1.20.2"
-rand = "0.8"
+once_cell = "1.18.0"
 regex = "1.10"
-rusqlite = "0.32"
-sanitize-filename = "0.5"
-serde = "1.0"
-serde_json = "1"
-tempfile = "3.13.0"
-thiserror = "1"
-tokio = "1"
-tokio-util = "0.7.11"
-tracing-subscriber = "0.3"
-yerpc = "0.6.2"
+rusqlite = "0.31"
+chrono = { version = "0.4.38", default-features=false, features = ["alloc", "clock", "std"] }
 
 [features]
 default = ["vendored"]
 internals = []
 vendored = [
-  "rusqlite/bundled-sqlcipher-vendored-openssl"
+  "async-native-tls/vendored",
+  "rusqlite/bundled-sqlcipher-vendored-openssl",
+  "reqwest/native-tls-vendored"
 ]
-
-[lints.rust]
-unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] }

README.md

@@ -30,13 +30,13 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ cargo run --locked -p deltachat-repl -- ~/deltachat-db
+$ cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Optionally, install `deltachat-repl` binary with
 ```
-$ cargo install --locked --path deltachat-repl/
+$ cargo install --path deltachat-repl/
 ```
 and run as
 ```

assets/qr_overlay_delta.svg-part

@@ -1,12 +0,0 @@
-<path
-   style="fill:#ffffff;fill-opacity:1;stroke:none"
-   d="m 24.015419,1.2870249 c -12.549421,0 -22.7283936,10.1789711 -22.7283936,22.7283931 0,12.549422 10.1789726,22.728395 22.7283936,22.728395 14.337742,-0.342877 9.614352,-4.702705 23.697556,0.969161 -7.545453,-13.001555 -1.082973,-13.32964 -0.969161,-23.697556 0,-12.549422 -10.178973,-22.7283931 -22.728395,-22.7283931 z" />
-<path
-   style="fill:#000000;fill-opacity:1;stroke:none"
-   d="M 23.982249,5.3106163 C 13.645822,5.4364005 5.2618355,13.92999 5.2618355,24.275753 c 0,10.345764 8.3839865,18.635301 18.7204135,18.509516 9.827724,-0.03951 7.516769,-5.489695 18.380082,-0.443187 -5.950849,-9.296115 0.201753,-10.533667 0.340336,-18.521947 0,-10.345766 -8.383989,-18.6353031 -18.720418,-18.5095187 z" />
-<g
-    style="fill:#ffffff"
-    transform="scale(1.1342891,0.88160947)">
-  <path
-     d="m 21.360141,23.513382 q -1.218487,-1.364705 -3.387392,-3.265543 -2.388233,-2.095797 -3.216804,-3.289913 -0.828571,-1.218486 -0.828571,-2.6563 0,-2.144536 1.998318,-3.363022 1.998317,-1.2428565 5.215121,-1.2428565 3.216804,0 5.605037,1.0966375 2.412603,1.096638 2.412603,3.021846 0,0.92605 -0.584873,1.535293 -0.584874,0.609243 -1.364705,0.609243 -1.121008,0 -2.631931,-1.681511 -1.535292,-1.705881 -2.60756,-2.388233 -1.047898,-0.706722 -2.461343,-0.706722 -1.803359,0 -2.973106,0.804201 -1.145377,0.804201 -1.145377,2.047057 0,1.169747 0.950419,2.193275 0.950419,1.023529 4.898315,3.728568 4.215963,2.899998 5.946213,4.532769 1.75462,1.632772 2.851258,3.972265 1.096638,2.339494 1.096638,4.947055 0,4.581508 -3.241174,8.090749 -3.216804,3.484871 -7.530245,3.484871 -3.923526,0 -6.628566,-2.802519 -2.705039,-2.802518 -2.705039,-7.481506 0,-4.508399 2.973106,-7.530245 2.997477,-3.021846 7.359658,-3.655459 z m 1.072268,1.121008 q -6.994112,1.145377 -6.994112,9.601672 0,4.36218 1.730251,6.774783 1.75462,2.412603 4.069744,2.412603 2.412603,0 3.972265,-2.315124 1.559663,-2.339493 1.559663,-6.311759 0,-5.751255 -4.337811,-10.162175 z" />
-</g>

deltachat-contact-tools/Cargo.toml

@@ -12,7 +12,7 @@ anyhow = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true } # Needed in order to `impl rusqlite::types::ToSql for EmailAddress`. Could easily be put behind a feature.
-chrono = { workspace = true, features = ["alloc", "clock", "std"] }
+chrono = { workspace = true }
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.

deltachat-contact-tools/src/lib.rs

@@ -15,16 +15,14 @@
     clippy::explicit_into_iter_loop,
     clippy::cloned_instead_of_copied
 )]
-#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
-#![cfg_attr(not(test), forbid(clippy::string_slice))]
+#![cfg_attr(not(test), warn(clippy::indexing_slicing))]
 #![allow(
     clippy::match_bool,
     clippy::mixed_read_write_in_expression,
     clippy::bool_assert_comparison,
     clippy::manual_split_once,
     clippy::format_push_string,
-    clippy::bool_to_int_with_if,
-    clippy::manual_range_contains
+    clippy::bool_to_int_with_if
 )]
 
 use std::fmt;
@@ -37,6 +35,10 @@ use chrono::{DateTime, NaiveDateTime};
 use once_cell::sync::Lazy;
 use regex::Regex;
 
+// TODOs to clean up:
+// - Check if sanitizing is done correctly everywhere
+// - Apply lints everywhere (https://doc.rust-lang.org/cargo/reference/workspaces.html#the-lints-table)
+
 #[derive(Debug)]
 /// A Contact, as represented in a VCard.
 pub struct VcardContact {
@@ -113,9 +115,7 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
         // If `s` is `EMAIL;TYPE=work:alice@example.com` and `property` is `EMAIL`,
         // then `remainder` is now `;TYPE=work:alice@example.com`
 
-        // Note: This doesn't handle the case where there are quotes around a colon,
-        // like `NAME;Foo="Some quoted text: that contains a colon":value`.
-        // This could be improved in the future, but for now, the parsing is good enough.
+        // TODO this doesn't handle the case where there are quotes around a colon
         let (params, value) = remainder.split_once(':')?;
         // In the example from above, `params` is now `;TYPE=work`
         // and `value` is now `alice@example.com`
@@ -155,7 +155,7 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
     }
 
     // Remove line folding, see https://datatracker.ietf.org/doc/html/rfc6350#section-3.2
-    static NEWLINE_AND_SPACE_OR_TAB: Lazy<Regex> = Lazy::new(|| Regex::new("\r?\n[\t ]").unwrap());
+    static NEWLINE_AND_SPACE_OR_TAB: Lazy<Regex> = Lazy::new(|| Regex::new("\n[\t ]").unwrap());
     let unfolded_lines = NEWLINE_AND_SPACE_OR_TAB.replace_all(vcard, "");
 
     let mut lines = unfolded_lines.lines().peekable();
@@ -175,15 +175,7 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
         let mut photo = None;
         let mut datetime = None;
 
-        for mut line in lines.by_ref() {
-            if let Some(remainder) = remove_prefix(line, "item1.") {
-                // Remove the group name, if the group is called "item1".
-                // If necessary, we can improve this to also remove groups that are called something different that "item1".
-                //
-                // Search "group name" at https://datatracker.ietf.org/doc/html/rfc6350 for more infos.
-                line = remainder;
-            }
-
+        for line in lines.by_ref() {
             if let Some(email) = vcard_property(line, "email") {
                 addr.get_or_insert(email);
             } else if let Some(name) = vcard_property(line, "fn") {
@@ -191,7 +183,6 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
             } else if let Some(k) = remove_prefix(line, "KEY;PGP;ENCODING=BASE64:")
                 .or_else(|| remove_prefix(line, "KEY;TYPE=PGP;ENCODING=b:"))
                 .or_else(|| remove_prefix(line, "KEY:data:application/pgp-keys;base64,"))
-                .or_else(|| remove_prefix(line, "KEY;PREF=1:data:application/pgp-keys;base64,"))
             {
                 key.get_or_insert(k);
             } else if let Some(p) = remove_prefix(line, "PHOTO;JPEG;ENCODING=BASE64:")
@@ -272,27 +263,27 @@ impl rusqlite::types::ToSql for ContactAddress {
     }
 }
 
-/// Takes a name and an address and sanitizes them:
-/// - Extracts a name from the addr if the addr is in form "Alice <alice@example.org>"
-/// - Removes special characters from the name, see [`sanitize_name()`]
-/// - Removes the name if it is equal to the address by setting it to ""
+/// Make the name and address
 pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
     let (name, addr) = if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.is_empty() {
-                captures.get(1).map_or("", |m| m.as_str())
+                strip_rtlo_characters(captures.get(1).map_or("", |m| m.as_str()))
             } else {
-                name
+                strip_rtlo_characters(name)
             },
             captures
                 .get(2)
                 .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
-        (name, addr.to_string())
+        (
+            strip_rtlo_characters(&normalize_name(name)),
+            addr.to_string(),
+        )
     };
-    let mut name = sanitize_name(name);
+    let mut name = normalize_name(&name);
 
     // If the 'display name' is just the address, remove it:
     // Otherwise, the contact would sometimes be shown as "alice@example.com (alice@example.com)" (see `get_name_n_addr()`).
@@ -304,77 +295,31 @@ pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     (name, addr)
 }
 
-/// Sanitizes a name.
+/// Normalize a name.
 ///
-/// - Removes newlines and trims the string
-/// - Removes quotes (come from some bad MUA implementations)
-/// - Removes potentially-malicious bidi characters
-pub fn sanitize_name(name: &str) -> String {
-    let name = sanitize_single_line(name);
-
-    match name.as_bytes() {
-        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => name
-            .get(1..name.len() - 1)
-            .map_or("".to_string(), |s| s.trim().to_string()),
-        _ => name.to_string(),
+/// - Remove quotes (come from some bad MUA implementations)
+/// - Trims the resulting string
+///
+/// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
+pub fn normalize_name(full_name: &str) -> String {
+    let full_name = full_name.trim();
+    if full_name.is_empty() {
+        return full_name.into();
     }
-}
 
-/// Sanitizes user input
-///
-/// - Removes newlines and trims the string
-/// - Removes potentially-malicious bidi characters
-pub fn sanitize_single_line(input: &str) -> String {
-    sanitize_bidi_characters(input.replace(['\n', '\r'], " ").trim())
+    match full_name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
+            .get(1..full_name.len() - 1)
+            .map_or("".to_string(), |s| s.trim().to_string()),
+        _ => full_name.to_string(),
+    }
 }
 
 const RTLO_CHARACTERS: [char; 5] = ['\u{202A}', '\u{202B}', '\u{202C}', '\u{202D}', '\u{202E}'];
-const ISOLATE_CHARACTERS: [char; 3] = ['\u{2066}', '\u{2067}', '\u{2068}'];
-const POP_ISOLATE_CHARACTER: char = '\u{2069}';
-/// Some control unicode characters can influence whether adjacent text is shown from
-/// left to right or from right to left.
-///
-/// Since user input is not supposed to influence how adjacent text looks,
-/// this function removes some of these characters.
-///
-/// Also see https://github.com/deltachat/deltachat-core-rust/issues/3479.
-pub fn sanitize_bidi_characters(input_str: &str) -> String {
-    // RTLO_CHARACTERS are apparently rarely used in practice.
-    // They can impact all following text, so, better remove them all:
-    let input_str = input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "");
-
-    // If the ISOLATE characters are not ended with a POP DIRECTIONAL ISOLATE character,
-    // we regard the input as potentially malicious and simply remove all ISOLATE characters.
-    // See https://en.wikipedia.org/wiki/Bidirectional_text#Unicode_bidi_support
-    // and https://www.w3.org/International/questions/qa-bidi-unicode-controls.en
-    // for an explanation about ISOLATE characters.
-    fn isolate_characters_are_valid(input_str: &str) -> bool {
-        let mut isolate_character_nesting: i32 = 0;
-        for char in input_str.chars() {
-            if ISOLATE_CHARACTERS.contains(&char) {
-                isolate_character_nesting += 1;
-            } else if char == POP_ISOLATE_CHARACTER {
-                isolate_character_nesting -= 1;
-            }
-
-            // According to Wikipedia, 125 levels are allowed:
-            // https://en.wikipedia.org/wiki/Unicode_control_characters
-            // (although, in practice, we could also significantly lower this number)
-            if isolate_character_nesting < 0 || isolate_character_nesting > 125 {
-                return false;
-            }
-        }
-        isolate_character_nesting == 0
-    }
-
-    if isolate_characters_are_valid(&input_str) {
-        input_str
-    } else {
-        input_str.replace(
-            |char| ISOLATE_CHARACTERS.contains(&char) || POP_ISOLATE_CHARACTER == char,
-            "",
-        )
-    }
+/// This method strips all occurrences of the RTLO Unicode character.
+/// [Why is this needed](https://github.com/deltachat/deltachat-core-rust/issues/3479)?
+pub fn strip_rtlo_characters(input_str: &str) -> String {
+    input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "")
 }
 
 /// Returns false if addr is an invalid address, otherwise true.
@@ -698,10 +643,10 @@ END:VCARD
     }
 
     #[test]
-    fn test_vcard_with_base64_avatar() {
-        // This is not an actual base64-encoded avatar, it's just to test the parsing.
-        // This one is Android-like.
-        let vcard0 = "BEGIN:VCARD
+    fn test_android_vcard_with_base64_avatar() {
+        // This is not an actual base64-encoded avatar, it's just to test the parsing
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
 VERSION:2.1
 N:;Bob;;;
 FN:Bob
@@ -711,101 +656,13 @@ PHOTO;ENCODING=BASE64;JPEG:/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEU
  L8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==
 
 END:VCARD
-";
-        // This one is DOS-like.
-        let vcard1 = vcard0.replace('\n', "\r\n");
-        for vcard in [vcard0, vcard1.as_str()] {
-            let contacts = parse_vcard(vcard);
-            assert_eq!(contacts.len(), 1);
-            assert_eq!(contacts[0].addr, "bob@example.org".to_string());
-            assert_eq!(contacts[0].authname, "Bob".to_string());
-            assert_eq!(contacts[0].key, None);
-            assert_eq!(contacts[0].profile_image.as_deref().unwrap(), "/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAL8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==");
-        }
-    }
-
-    #[test]
-    fn test_protonmail_vcard() {
-        let contacts = parse_vcard(
-            "BEGIN:VCARD
-VERSION:4.0
-FN;PREF=1:Alice Wonderland
-UID:proton-web-03747582-328d-38dc-5ddd-000000000000
-ITEM1.EMAIL;PREF=1:alice@example.org
-ITEM1.KEY;PREF=1:data:application/pgp-keys;base64,aaaaaaaaaaaaaaaaaaaaaaaaa
- aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
-ITEM1.KEY;PREF=2:data:application/pgp-keys;base64,bbbbbbbbbbbbbbbbbbbbbbbbb
- bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
-ITEM1.X-PM-ENCRYPT:true
-ITEM1.X-PM-SIGN:true
-END:VCARD",
+",
         );
 
         assert_eq!(contacts.len(), 1);
-        assert_eq!(&contacts[0].addr, "alice@example.org");
-        assert_eq!(&contacts[0].authname, "Alice Wonderland");
-        assert_eq!(contacts[0].key.as_ref().unwrap(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-        assert!(contacts[0].timestamp.is_err());
-        assert_eq!(contacts[0].profile_image, None);
-    }
-
-    #[test]
-    fn test_sanitize_name() {
-        assert_eq!(&sanitize_name(" hello world   "), "hello world");
-        assert_eq!(&sanitize_name("<"), "<");
-        assert_eq!(&sanitize_name(">"), ">");
-        assert_eq!(&sanitize_name("'"), "'");
-        assert_eq!(&sanitize_name("\""), "\"");
-    }
-
-    #[test]
-    fn test_sanitize_single_line() {
-        assert_eq!(sanitize_single_line("Hi\naiae "), "Hi aiae");
-        assert_eq!(sanitize_single_line("\r\nahte\n\r"), "ahte");
-    }
-
-    #[test]
-    fn test_sanitize_bidi_characters() {
-        // Legit inputs:
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat\u{2069}"),
-            "Tes\u{2067}ting Delta Chat\u{2069}"
-        );
-
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"),
-            "Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"
-        );
-
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"),
-            "Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"
-        );
-
-        // Potentially-malicious inputs:
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{202C}ting Delta Chat"),
-            "Testing Delta Chat"
-        );
-
-        assert_eq!(
-            &sanitize_bidi_characters("Testing Delta Chat\u{2069}"),
-            "Testing Delta Chat"
-        );
-
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat"),
-            "Testing Delta Chat"
-        );
-
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{2069}ting Delta Chat\u{2067}"),
-            "Testing Delta Chat"
-        );
-
-        assert_eq!(
-            &sanitize_bidi_characters("Tes\u{2068}ting Delta Chat"),
-            "Testing Delta Chat"
-        );
+        assert_eq!(contacts[0].addr, "bob@example.org".to_string());
+        assert_eq!(contacts[0].authname, "Bob".to_string());
+        assert_eq!(contacts[0].key, None);
+        assert_eq!(contacts[0].profile_image.as_deref().unwrap(), "/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAL8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==");
     }
 }

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.149.0"
+version = "1.138.5"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -14,21 +14,21 @@ name = "deltachat"
 crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
-deltachat = { workspace = true, default-features = false }
-deltachat-jsonrpc = { workspace = true, optional = true }
-libc = { workspace = true }
+deltachat = { path = "../", default-features = false }
+deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
+libc = "0.2"
 human-panic = { version = "2", default-features = false }
-num-traits = { workspace = true }
-serde_json = { workspace = true }
-tokio = { workspace = true, features = ["rt-multi-thread"] }
-anyhow = { workspace = true }
-thiserror = { workspace = true }
-rand = { workspace = true }
-once_cell = { workspace = true }
-yerpc = { workspace = true, features = ["anyhow_expose"] }
+num-traits = "0.2"
+serde_json = "1.0"
+tokio = { version = "1.37.0", features = ["rt-multi-thread"] }
+anyhow = "1"
+thiserror = "1"
+rand = "0.8"
+once_cell = "1.18.0"
+yerpc = { version = "0.5.1", features = ["anyhow_expose"] }
 
 [features]
 default = ["vendored"]
-vendored = ["deltachat/vendored", "deltachat-jsonrpc/vendored"]
+vendored = ["deltachat/vendored"]
 jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/deltachat.h

@@ -403,10 +403,13 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_port`    = SMTP-port, guessed if left out
  * - `send_security`= SMTP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
- * - `proxy_enabled` = Proxy enabled. Disabled by default.
- * - `proxy_url` = Proxy URL. May contain multiple URLs separated by newline, but only the first one is used.
+ * - `socks5_enabled` = SOCKS5 enabled
+ * - `socks5_host` = SOCKS5 proxy server host
+ * - `socks5_port` = SOCKS5 proxy server port
+ * - `socks5_user` = SOCKS5 proxy username
+ * - `socks5_password` = SOCKS5 proxy password
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
- * - `smtp_certificate_checks` = deprecated option, should be set to the same value as `imap_certificate_checks` but ignored by the new core
+ * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages. MUAs are allowed to spread this way e.g. using CC, defaults to empty
  * - `selfstatus`   = Own status to display, e.g. in e-mail footers, defaults to empty
  * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
@@ -417,10 +420,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    and also recoded to a reasonable size.
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
- *                    1=send and request read receipts
- *                    default=send and request read receipts, only send but not reuqest if `bot` is set
- * - `bcc_self`     = 0=do not send a copy of outgoing messages to self,
- *                    1=send a copy of outgoing messages to self (default).
+ *                    1=send and request read receipts (default)
+ * - `bcc_self`     = 0=do not send a copy of outgoing messages to self (default),
+ *                    1=send a copy of outgoing messages to self.
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes,
@@ -479,9 +481,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `bot`          = Set to "1" if this is a bot.
  *                    Prevents adding the "Device messages" and "Saved messages" chats,
  *                    adds Auto-Submitted header to outgoing messages,
- *                    accepts contact requests automatically (calling dc_accept_chat() is not needed),
- *                    does not cut large incoming text messages,
- *                    handles existing messages the same way as new ones if `fetch_existing_msgs=1`.
+ *                    accepts contact requests automatically (calling dc_accept_chat() is not needed for bots)
+ *                    and does not cut large incoming text messages.
  * - `last_msg_id` = database ID of the last message processed by the bot.
  *                   This ID and IDs below it are guaranteed not to be returned
  *                   by dc_get_next_msgs() and dc_wait_next_msgs().
@@ -492,8 +493,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                   For most bots calling `dc_markseen_msgs()` is the
  *                   recommended way to update this value
  *                   even for self-sent messages.
- * - `fetch_existing_msgs` = 0=do not fetch existing messages on configure (default),
- *                    1=fetch most recent existing messages on configure.
+ * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
+ *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
  * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
  *                    0=use IMAP IDLE if the server supports it.
@@ -506,11 +507,6 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
- * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
- *                    This is an experimental option not compatible to other MUAs
- *                    and older Delta Chat versions.
- *                    1 = enable.
- *                    0 = disable (default).
  * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
  *                    seconds. 2 days by default.
  *                    This is not supposed to be changed by UIs and only used for testing.
@@ -522,21 +518,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    1=After the key changed, `dc_chat_can_send()` returns false and `dc_chat_is_protection_broken()` returns true
  *                    until `dc_accept_chat()` is called.
  * - `is_chatmail` = 1 if the the server is a chatmail server, 0 otherwise.
- * - `is_muted`     = Whether a context is muted by the user.
- *                    Muted contexts should not sound, vibrate or show notifications.
- *                    In contrast to `dc_set_chat_mute_duration()`,
- *                    fresh message and badge counters are not changed by this setting,
- *                    but should be tuned down where appropriate.
- * - `private_tag`  = Optional tag as "Work", "Family".
- *                    Meant to help profile owner to differ between profiles with similar names.
  * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
  *                    The prefix should be followed by the system and maybe subsystem,
  *                    e.g. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
  *                    These keys go to backups and allow easy per-account settings when using @ref dc_accounts_t,
  *                    however, are not handled by the core otherwise.
- * - `webxdc_realtime_enabled` = Whether the realtime APIs should be enabled.
- *                               0 = WebXDC realtime API is disabled and behaves as noop.
- *                               1 = WebXDC realtime API is enabled (default).
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -871,10 +857,13 @@ void            dc_maybe_network             (dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
+ * @param addr The e-mail address of the user. This must match the
+ *    configured_addr setting of the context as well as the UID of the key.
+ * @param public_data Ignored, actual public key is extracted from secret_data.
  * @param secret_data ASCII armored secret key.
  * @return 1 on success, 0 on failure.
  */
-int             dc_preconfigure_keypair        (dc_context_t* context, const char *secret_data);
+int             dc_preconfigure_keypair        (dc_context_t* context, const char *addr, const char *public_data, const char *secret_data);
 
 
 // handle chatlists
@@ -1554,6 +1543,30 @@ void            dc_marknoticed_chat          (dc_context_t* context, uint32_t ch
 dc_array_t*     dc_get_chat_media            (dc_context_t* context, uint32_t chat_id, int msg_type, int msg_type2, int msg_type3);
 
 
+/**
+ * Search next/previous message based on a given message and a list of types.
+ * Typically used to implement the "next" and "previous" buttons
+ * in a gallery or in a media player.
+ *
+ * @deprecated Deprecated 2023-10-03, use dc_get_chat_media() and navigate the returned array instead.
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @param msg_id The ID of the current message from which the next or previous message should be searched.
+ * @param dir 1=get the next message, -1=get the previous one.
+ * @param msg_type The message type to search for.
+ *     If 0, the message type from curr_msg_id is used.
+ * @param msg_type2 Alternative message type to search for. 0 to skip.
+ * @param msg_type3 Alternative message type to search for. 0 to skip.
+ * @return Returns the message ID that should be played next.
+ *     The returned message is in the same chat as the given one
+ *     and has one of the given types.
+ *     Typically, this result is passed again to dc_get_next_media()
+ *     later on the next swipe.
+ *     If there is not next/previous message, the function returns 0.
+ */
+uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t msg_id, int dir, int msg_type, int msg_type2, int msg_type3);
+
+
 /**
  * Set chat visibility to pinned, archived or normal.
  *
@@ -2482,9 +2495,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
 #define         DC_QR_BACKUP                 251
-#define         DC_QR_BACKUP2                252
 #define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
-#define         DC_QR_PROXY                  271 // text1=address (e.g. "127.0.0.1:9050")
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
 #define         DC_QR_URL                    332 // text1=URL
@@ -2530,7 +2541,6 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr() and then dc_configure().
  *
  * - DC_QR_BACKUP:
- * - DC_QR_BACKUP2:
  *   ask the user if they want to set up a new device.
  *   If so, pass the qr-code to dc_receive_backup().
  *
@@ -2538,12 +2548,6 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   ask the user if they want to use the given service for video chats;
  *   if so, call dc_set_config_from_qr().
  *
- * - DC_QR_PROXY with dc_lot_t::text1=address:
- *   ask the user if they want to use the given proxy.
- *   if so, call dc_set_config_from_qr() and restart I/O.
- *   On success, dc_get_config(context, "proxy_url")
- *   will contain the new proxy in normalized form as the first element.
- *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
  *   e-mail address scanned, optionally, a draft message could be set in
  *   dc_lot_t::text1 in which case dc_lot_t::text1_meaning will be DC_TEXT1_DRAFT;
@@ -2618,7 +2622,6 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  * Get QR code image from the QR code text generated by dc_get_securejoin_qr().
  * See dc_get_securejoin_qr() for details about the contained QR code.
  *
- * @deprecated 2024-10 use dc_create_qr_svg(dc_get_securejoin_qr()) instead.
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id group-chat-id for secure-join or 0 for setup-contact,
@@ -2799,22 +2802,6 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
 void        dc_delete_all_locations         (dc_context_t* context);
 
 
-// misc
-
-/**
- * Create a QR code from any input data.
- *
- * The QR code is returned as a square SVG image.
- *
- * @memberof dc_context_t
- * @param payload The content for the QR code.
- * @return SVG image with the QR code.
- *     On errors, an empty string is returned.
- *     The returned string must be released using dc_str_unref() after usage.
- */
-char*           dc_create_qr_svg             (const char* payload);
-
-
 /**
  * Get last error string.
  *
@@ -2903,7 +2890,6 @@ char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider);
  * This works like dc_backup_provider_qr() but returns the text of a rendered
  * SVG image containing the QR code.
  *
- * @deprecated 2024-10 use dc_create_qr_svg(dc_backup_provider_get_qr()) instead.
  * @memberof dc_backup_provider_t
  * @param backup_provider The backup provider object as created by
  *    dc_backup_provider_new().
@@ -2943,7 +2929,7 @@ void dc_backup_provider_unref (dc_backup_provider_t* backup_provider);
  * Gets a backup offered by a dc_backup_provider_t object on another device.
  *
  * This function is called on a device that scanned the QR code offered by
- * dc_backup_provider_get_qr().  Typically this is a
+ * dc_backup_sender_qr() or dc_backup_sender_qr_svg().  Typically this is a
  * different device than that which provides the backup.
  *
  * This call will block while the backup is being transferred and only
@@ -5711,14 +5697,8 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_CERTCK_STRICT 1
 
 /**
- * Accept certificates that are expired, self-signed
- * or not valid for the server hostname.
- */
-#define DC_CERTCK_ACCEPT_INVALID 2
-
-/**
- * For API compatibility only: Treat this as DC_CERTCK_ACCEPT_INVALID on reading.
- * Must not be written.
+ * Accept invalid certificates, including self-signed ones
+ * or having incorrect hostname.
  */
 #define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
 
@@ -6064,21 +6044,6 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_REACTIONS_CHANGED        2001
 
 
-/**
- * A reaction to one's own sent message received.
- * Typically, the UI will show a notification for that.
- *
- * In addition to this event, DC_EVENT_REACTIONS_CHANGED is emitted.
- *
- * @param data1 (int) contact_id ID of the contact sending this reaction.
- * @param data2 (int) msg_id + (char*) reaction.
- *      ID of the message for which a reaction was received in dc_event_get_data2_int(),
- *      and the reaction as dc_event_get_data2_str().
- *      string must be passed to dc_str_unref() afterwards.
- */
-#define DC_EVENT_INCOMING_REACTION        2002
-
-
 /**
  * There is a fresh message. Typically, the user will show an notification
  * when receiving this message.
@@ -6296,7 +6261,7 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * Webxdc status update received.
+ * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
  * `serial` set to the last known update
  * (in case of special bots, `status_update_serial` from `data2`
@@ -6319,27 +6284,6 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
 
-/**
- * Data received over an ephemeral peer channel.
- *
- * @param data1 (int) msg_id
- * @param data2 (int) + (char*) binary data.
- *     length is returned as integer with dc_event_get_data2_int()
- *     and binary data is returned as dc_event_get_data2_str().
- *     Binary data must be passed to dc_str_unref() afterwards.
- */
-
-#define DC_EVENT_WEBXDC_REALTIME_DATA             2150
-
-/**
- * Advertisement for ephemeral peer channel communication received.
- * This can be used by bots to initiate peer-to-peer communication from their side.
- * @param data1 (int) msg_id
- * @param data2 0
- */
-
-#define DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT    2151
-
 /**
  * Tells that the Background fetch was completed (or timed out).
  *
@@ -6368,14 +6312,6 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_CHATLIST_ITEM_CHANGED         2301
 
-
-/**
- * Inform that some events have been skipped due to event channel overflow.
- *
- * @param data1 (int) number of events that have been skipped
- */
-#define DC_EVENT_CHANNEL_OVERFLOW              2400
-
 /**
  * @}
  */
@@ -6683,16 +6619,12 @@ void dc_event_unref(dc_event_t* event);
 /// "Message opened"
 ///
 /// Used in subjects of outgoing read receipts.
-///
-/// @deprecated Deprecated 2024-07-26
 #define DC_STR_READRCPT                   31
 
 /// "The message '%1$s' you sent was displayed on the screen of the recipient."
 ///
 /// Used as message text of outgoing read receipts.
 /// - %1$s will be replaced by the subject of the displayed message
-///
-/// @deprecated Deprecated 2024-06-23
 #define DC_STR_READRCPT_MAILBODY          32
 
 /// @deprecated Deprecated, this string is no longer needed.
@@ -7411,7 +7343,7 @@ void dc_event_unref(dc_event_t* event);
 /// Used as info message.
 #define DC_STR_SECUREJOIN_WAIT_TIMEOUT 191
 
-/// "Contact". Deprecated, currently unused.
+/// "Contact"
 #define DC_STR_CONTACT 200
 
 /**

deltachat-ffi/src/lib.rs

@@ -30,7 +30,7 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
 use deltachat::message::MsgId;
-use deltachat::qr_code_generator::{create_qr_svg, generate_backup_qr, get_securejoin_qr_svg};
+use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
@@ -541,7 +541,6 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::ErrorSelfNotInGroup(_) => 410,
         EventType::MsgsChanged { .. } => 2000,
         EventType::ReactionsChanged { .. } => 2001,
-        EventType::IncomingReaction { .. } => 2002,
         EventType::IncomingMsg { .. } => 2005,
         EventType::IncomingMsgBunch { .. } => 2006,
         EventType::MsgsNoticed { .. } => 2008,
@@ -564,14 +563,9 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
         EventType::WebxdcRealtimeData { .. } => 2150,
-        EventType::WebxdcRealtimeAdvertisementReceived { .. } => 2151,
         EventType::AccountsBackgroundFetchDone => 2200,
         EventType::ChatlistChanged => 2300,
         EventType::ChatlistItemChanged { .. } => 2301,
-        EventType::EventChannelOverflow { .. } => 2400,
-        #[allow(unreachable_patterns)]
-        #[cfg(test)]
-        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -602,7 +596,6 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_)
         | EventType::AccountsBackgroundFetchDone => 0,
         EventType::ChatlistChanged => 0,
-        EventType::IncomingReaction { contact_id, .. } => contact_id.to_u32() as libc::c_int,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -627,15 +620,10 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         }
         EventType::WebxdcRealtimeData { msg_id, .. }
         | EventType::WebxdcStatusUpdate { msg_id, .. }
-        | EventType::WebxdcRealtimeAdvertisementReceived { msg_id }
         | EventType::WebxdcInstanceDeleted { msg_id, .. } => msg_id.to_u32() as libc::c_int,
         EventType::ChatlistItemChanged { chat_id } => {
             chat_id.unwrap_or_default().to_u32() as libc::c_int
         }
-        EventType::EventChannelOverflow { n } => *n as libc::c_int,
-        #[allow(unreachable_patterns)]
-        #[cfg(test)]
-        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -670,17 +658,15 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
+        | EventType::WebxdcRealtimeData { .. }
         | EventType::SelfavatarChanged
         | EventType::AccountsBackgroundFetchDone
         | EventType::ChatlistChanged
         | EventType::ChatlistItemChanged { .. }
-        | EventType::ConfigSynced { .. }
-        | EventType::ChatModified(_)
-        | EventType::WebxdcRealtimeAdvertisementReceived { .. }
-        | EventType::EventChannelOverflow { .. } => 0,
+        | EventType::ConfigSynced { .. } => 0,
+        EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
-        | EventType::IncomingReaction { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
         | EventType::MsgFailed { msg_id, .. }
@@ -693,10 +679,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
             status_update_serial,
             ..
         } => status_update_serial.to_u32() as libc::c_int,
-        EventType::WebxdcRealtimeData { data, .. } => data.len() as libc::c_int,
-        #[allow(unreachable_patterns)]
-        #[cfg(test)]
-        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -743,13 +725,12 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
         | EventType::WebxdcInstanceDeleted { .. }
+        | EventType::WebxdcRealtimeData { .. }
         | EventType::AccountsBackgroundFetchDone
         | EventType::ChatEphemeralTimerModified { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::ChatlistItemChanged { .. }
-        | EventType::ChatlistChanged
-        | EventType::WebxdcRealtimeAdvertisementReceived { .. }
-        | EventType::EventChannelOverflow { .. } => ptr::null_mut(),
+        | EventType::ChatlistChanged => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
                 comment.to_c_string().unwrap_or_default().into_raw()
@@ -765,19 +746,6 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             let data2 = key.to_string().to_c_string().unwrap_or_default();
             data2.into_raw()
         }
-        EventType::WebxdcRealtimeData { data, .. } => {
-            let ptr = libc::malloc(data.len());
-            libc::memcpy(ptr, data.as_ptr() as *mut libc::c_void, data.len());
-            ptr as *mut libc::c_char
-        }
-        EventType::IncomingReaction { reaction, .. } => reaction
-            .as_str()
-            .to_c_string()
-            .unwrap_or_default()
-            .into_raw(),
-        #[allow(unreachable_patterns)]
-        #[cfg(test)]
-        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -859,6 +827,8 @@ pub unsafe extern "C" fn dc_maybe_network(context: *mut dc_context_t) {
 #[no_mangle]
 pub unsafe extern "C" fn dc_preconfigure_keypair(
     context: *mut dc_context_t,
+    addr: *const libc::c_char,
+    _public_data: *const libc::c_char,
     secret_data: *const libc::c_char,
 ) -> i32 {
     if context.is_null() {
@@ -866,8 +836,9 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
         return 0;
     }
     let ctx = &*context;
+    let addr = to_string_lossy(addr);
     let secret_data = to_string_lossy(secret_data);
-    block_on(preconfigure_keypair(ctx, &secret_data))
+    block_on(preconfigure_keypair(ctx, &addr, &secret_data))
         .context("Failed to save keypair")
         .log_err(ctx)
         .is_ok() as libc::c_int
@@ -1467,6 +1438,48 @@ pub unsafe extern "C" fn dc_get_chat_media(
     })
 }
 
+#[no_mangle]
+#[allow(deprecated)]
+pub unsafe extern "C" fn dc_get_next_media(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    dir: libc::c_int,
+    msg_type: libc::c_int,
+    or_msg_type2: libc::c_int,
+    or_msg_type3: libc::c_int,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_next_media()");
+        return 0;
+    }
+    let direction = if dir < 0 {
+        chat::Direction::Backward
+    } else {
+        chat::Direction::Forward
+    };
+
+    let ctx = &*context;
+    let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {msg_type}"));
+    let or_msg_type2 =
+        from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {or_msg_type2}"));
+    let or_msg_type3 =
+        from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {or_msg_type3}"));
+
+    block_on(async move {
+        chat::get_next_media(
+            ctx,
+            MsgId::new(msg_id),
+            direction,
+            msg_type,
+            or_msg_type2,
+            or_msg_type3,
+        )
+        .await
+        .map(|msg_id| msg_id.map(|id| id.to_u32()).unwrap_or_default())
+        .unwrap_or(0)
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_chat_visibility(
     context: *mut dc_context_t,
@@ -2594,18 +2607,6 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
     });
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_create_qr_svg(payload: *const libc::c_char) -> *mut libc::c_char {
-    if payload.is_null() {
-        eprintln!("ignoring careless call to dc_create_qr_svg()");
-        return "".strdup();
-    }
-
-    create_qr_svg(&to_string_lossy(payload))
-        .unwrap_or_else(|_| "".to_string())
-        .strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
@@ -4355,7 +4356,7 @@ pub unsafe extern "C" fn dc_backup_provider_wait(provider: *mut dc_backup_provid
     let ctx = &*ffi_provider.context;
     let provider = &mut ffi_provider.provider;
     block_on(provider)
-        .context("Failed to await backup provider")
+        .context("Failed to await BackupProvider")
         .log_err(ctx)
         .set_last_error(ctx)
         .ok();
@@ -4409,7 +4410,7 @@ trait ResultExt<T, E> {
     /// Like `log_err()`, but:
     /// - returns the default value instead of an Err value.
     /// - emits an error instead of a warning for an [Err] result. This means
-    ///   that the error will be shown to the user in a small pop-up.
+    /// that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
 }
 
@@ -4528,16 +4529,19 @@ pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
     let addr = to_string_lossy(addr);
 
     let ctx = &*context;
-    let proxy_enabled = block_on(ctx.get_config_bool(config::Config::ProxyEnabled))
-        .context("Can't get config")
-        .log_err(ctx);
+    let socks5_enabled = block_on(async move {
+        ctx.get_config_bool(config::Config::Socks5Enabled)
+            .await
+            .context("Can't get config")
+            .log_err(ctx)
+    });
 
-    match proxy_enabled {
-        Ok(proxy_enabled) => {
+    match socks5_enabled {
+        Ok(socks5_enabled) => {
             match block_on(provider::get_provider_info_by_addr(
                 ctx,
                 addr.as_str(),
-                proxy_enabled,
+                socks5_enabled,
             ))
             .log_err(ctx)
             .unwrap_or_default()
@@ -4868,7 +4872,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     }
 
     let accounts = &*accounts;
-    block_on(async move { accounts.read().await.maybe_network_lost().await });
+    block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
 #[no_mangle]
@@ -4882,12 +4886,12 @@ pub unsafe extern "C" fn dc_accounts_background_fetch(
     }
 
     let accounts = &*accounts;
-    let background_fetch_future = {
-        let lock = block_on(accounts.read());
-        lock.background_fetch(Duration::from_secs(timeout_in_seconds))
-    };
-    // At this point account manager is not locked anymore.
-    block_on(background_fetch_future);
+    block_on(async move {
+        let accounts = accounts.read().await;
+        accounts
+            .background_fetch(Duration::from_secs(timeout_in_seconds))
+            .await;
+    });
     1
 }
 
@@ -4905,7 +4909,7 @@ pub unsafe extern "C" fn dc_accounts_set_push_device_token(
     let token = to_string_lossy(token);
 
     block_on(async move {
-        let accounts = accounts.read().await;
+        let mut accounts = accounts.write().await;
         if let Err(err) = accounts.set_push_device_token(&token).await {
             accounts.emit_event(EventType::Error(format!(
                 "Failed to set notify token: {err:#}."

deltachat-ffi/src/lot.rs

@@ -34,34 +34,33 @@ pub enum Meaning {
 }
 
 impl Lot {
-    pub fn get_text1(&self) -> Option<Cow<str>> {
+    pub fn get_text1(&self) -> Option<&str> {
         match self {
             Self::Summary(summary) => match &summary.prefix {
                 None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(Cow::Borrowed(text)),
-                Some(SummaryPrefix::Username(username)) => Some(Cow::Borrowed(username)),
-                Some(SummaryPrefix::Me(text)) => Some(Cow::Borrowed(text)),
+                Some(SummaryPrefix::Draft(text)) => Some(text),
+                Some(SummaryPrefix::Username(username)) => Some(username),
+                Some(SummaryPrefix::Me(text)) => Some(text),
             },
             Self::Qr(qr) => match qr {
                 Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
+                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
                 Qr::FprOk { .. } => None,
                 Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(Cow::Borrowed(fingerprint)),
-                Qr::Account { domain } => Some(Cow::Borrowed(domain)),
-                Qr::Backup2 { .. } => None,
-                Qr::WebrtcInstance { domain, .. } => Some(Cow::Borrowed(domain)),
-                Qr::Proxy { host, port, .. } => Some(Cow::Owned(format!("{host}:{port}"))),
-                Qr::Addr { draft, .. } => draft.as_deref().map(Cow::Borrowed),
-                Qr::Url { url } => Some(Cow::Borrowed(url)),
-                Qr::Text { text } => Some(Cow::Borrowed(text)),
+                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
+                Qr::Account { domain } => Some(domain),
+                Qr::Backup { .. } => None,
+                Qr::WebrtcInstance { domain, .. } => Some(domain),
+                Qr::Addr { draft, .. } => draft.as_deref(),
+                Qr::Url { url } => Some(url),
+                Qr::Text { text } => Some(text),
                 Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
+                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
                 Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
-                Qr::Login { address, .. } => Some(Cow::Borrowed(address)),
+                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::Login { address, .. } => Some(address),
             },
-            Self::Error(err) => Some(Cow::Borrowed(err)),
+            Self::Error(err) => Some(err),
         }
     }
 
@@ -102,9 +101,8 @@ impl Lot {
                 Qr::FprMismatch { .. } => LotState::QrFprMismatch,
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
                 Qr::Account { .. } => LotState::QrAccount,
-                Qr::Backup2 { .. } => LotState::QrBackup2,
+                Qr::Backup { .. } => LotState::QrBackup,
                 Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
-                Qr::Proxy { .. } => LotState::QrProxy,
                 Qr::Addr { .. } => LotState::QrAddr,
                 Qr::Url { .. } => LotState::QrUrl,
                 Qr::Text { .. } => LotState::QrText,
@@ -128,9 +126,8 @@ impl Lot {
                 Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
-                Qr::Backup2 { .. } => Default::default(),
+                Qr::Backup { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Proxy { .. } => Default::default(),
                 Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
@@ -180,14 +177,9 @@ pub enum LotState {
 
     QrBackup = 251,
 
-    QrBackup2 = 252,
-
     /// text1=domain, text2=instance pattern
     QrWebrtcInstance = 260,
 
-    /// text1=address, text2=protocol
-    QrProxy = 271,
-
     /// id=contact
     QrAddr = 320,
 

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.149.0"
+version = "1.138.5"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -13,30 +13,30 @@ path = "src/webserver.rs"
 required-features = ["webserver"]
 
 [dependencies]
-anyhow = { workspace = true }
-deltachat = { workspace = true }
-deltachat-contact-tools = { workspace = true }
-num-traits = { workspace = true }
-schemars = "0.8.21"
-serde = { workspace = true, features = ["derive"] }
-tempfile = { workspace = true }
-log = { workspace = true }
-async-channel = { workspace = true }
-futures = { workspace = true }
-serde_json = { workspace = true }
-yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
-typescript-type-def = { version = "0.5.13", features = ["json_value"] }
-tokio = { workspace = true }
-sanitize-filename = { workspace = true }
+anyhow = "1"
+deltachat = { path = ".." }
+deltachat-contact-tools = { path = "../deltachat-contact-tools" }
+num-traits = "0.2"
+schemars = "0.8.19"
+serde = { version = "1.0", features = ["derive"] }
+tempfile = "3.10.1"
+log = "0.4"
+async-channel = { version = "2.2.1" }
+futures = { version = "0.3.30" }
+serde_json = "1"
+yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+typescript-type-def = { version = "0.5.8", features = ["json_value"] }
+tokio = { version = "1.37.0" }
+sanitize-filename = "0.5"
 walkdir = "2.5.0"
-base64 = { workspace = true }
+base64 = "0.22"
 
 # optional dependencies
 axum = { version = "0.7", optional = true, features = ["ws"] }
-env_logger = { version = "0.11.5", optional = true }
+env_logger = { version = "0.11.3", optional = true }
 
 [dev-dependencies]
-tokio = { workspace = true, features = ["full", "rt-multi-thread"] }
+tokio = { version = "1.37.0", features = ["full", "rt-multi-thread"] }
 
 
 [features]

deltachat-jsonrpc/src/api.rs

@@ -254,12 +254,11 @@ impl CommandApi {
     /// Process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
     async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
-        let future = {
-            let lock = self.accounts.read().await;
-            lock.background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
-        };
-        // At this point account manager is not locked anymore.
-        future.await;
+        self.accounts
+            .write()
+            .await
+            .background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
+            .await;
         Ok(())
     }
 
@@ -322,12 +321,12 @@ impl CommandApi {
     ) -> Result<Option<ProviderInfo>> {
         let ctx = self.get_context(account_id).await?;
 
-        let proxy_enabled = ctx
-            .get_config_bool(deltachat::config::Config::ProxyEnabled)
+        let socks5_enabled = ctx
+            .get_config_bool(deltachat::config::Config::Socks5Enabled)
             .await?;
 
         let provider_info =
-            get_provider_info(&ctx, email.split('@').last().unwrap_or(""), proxy_enabled).await;
+            get_provider_info(&ctx, email.split('@').last().unwrap_or(""), socks5_enabled).await;
         Ok(ProviderInfo::from_dc_type(provider_info))
     }
 
@@ -708,22 +707,7 @@ impl CommandApi {
         ChatId::new(chat_id).get_encryption_info(&ctx).await
     }
 
-    /// Get QR code text that will offer a [SecureJoin](https://securejoin.delta.chat/) invitation.
-    ///
-    /// If `chat_id` is a group chat ID, SecureJoin QR code for the group is returned.
-    /// If `chat_id` is unset, setup contact QR code is returned.
-    async fn get_chat_securejoin_qr_code(
-        &self,
-        account_id: u32,
-        chat_id: Option<u32>,
-    ) -> Result<String> {
-        let ctx = self.get_context(account_id).await?;
-        let chat = chat_id.map(ChatId::new);
-        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
-        Ok(qr)
-    }
-
-    /// Get QR code (text and SVG) that will offer a Setup-Contact or Verified-Group invitation.
+    /// Get QR code (text and SVG) that will offer an Setup-Contact or Verified-Group invitation.
     /// The QR code is compatible to the OPENPGP4FPR format
     /// so that a basic fingerprint comparison also works e.g. with OpenKeychain.
     ///
@@ -745,9 +729,10 @@ impl CommandApi {
     ) -> Result<(String, String)> {
         let ctx = self.get_context(account_id).await?;
         let chat = chat_id.map(ChatId::new);
-        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
-        let svg = get_securejoin_qr_svg(&ctx, chat).await?;
-        Ok((qr, svg))
+        Ok((
+            securejoin::get_securejoin_qr(&ctx, chat).await?,
+            get_securejoin_qr_svg(&ctx, chat).await?,
+        ))
     }
 
     /// Continue a Setup-Contact or Verified-Group-Invite protocol
@@ -1135,11 +1120,9 @@ impl CommandApi {
     async fn get_message(&self, account_id: u32, msg_id: u32) -> Result<MessageObject> {
         let ctx = self.get_context(account_id).await?;
         let msg_id = MsgId::new(msg_id);
-        let message_object = MessageObject::from_msg_id(&ctx, msg_id)
+        MessageObject::from_msg_id(&ctx, msg_id)
             .await
-            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))?
-            .with_context(|| format!("Message {msg_id} does not exist for account {account_id}"))?;
-        Ok(message_object)
+            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))
     }
 
     async fn get_message_html(&self, account_id: u32, message_id: u32) -> Result<Option<String>> {
@@ -1163,10 +1146,7 @@ impl CommandApi {
             messages.insert(
                 message_id,
                 match message_result {
-                    Ok(Some(message)) => MessageLoadResult::Message(message),
-                    Ok(None) => MessageLoadResult::LoadingError {
-                        error: "Message does not exist".to_string(),
-                    },
+                    Ok(message) => MessageLoadResult::Message(message),
                     Err(error) => MessageLoadResult::LoadingError {
                         error: format!("{error:#}"),
                     },
@@ -1424,15 +1404,6 @@ impl CommandApi {
         Ok(())
     }
 
-    /// Resets contact encryption.
-    async fn reset_contact_encryption(&self, account_id: u32, contact_id: u32) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
-        let contact_id = ContactId::new(contact_id);
-
-        contact_id.reset_encryption(&ctx).await?;
-        Ok(())
-    }
-
     async fn change_contact_name(
         &self,
         account_id: u32,
@@ -1476,7 +1447,7 @@ impl CommandApi {
 
     /// Parses a vCard file located at the given path. Returns contacts in their original order.
     async fn parse_vcard(&self, path: String) -> Result<Vec<VcardContact>> {
-        let vcard = fs::read(Path::new(&path)).await?;
+        let vcard = tokio::fs::read(Path::new(&path)).await?;
         let vcard = str::from_utf8(&vcard)?;
         Ok(deltachat_contact_tools::parse_vcard(vcard)
             .into_iter()
@@ -1484,20 +1455,6 @@ impl CommandApi {
             .collect())
     }
 
-    /// Imports contacts from a vCard file located at the given path.
-    ///
-    /// Returns the ids of created/modified contacts in the order they appear in the vCard.
-    async fn import_vcard(&self, account_id: u32, path: String) -> Result<Vec<u32>> {
-        let ctx = self.get_context(account_id).await?;
-        let vcard = tokio::fs::read(Path::new(&path)).await?;
-        let vcard = str::from_utf8(&vcard)?;
-        Ok(deltachat::contact::import_vcard(&ctx, vcard)
-            .await?
-            .into_iter()
-            .map(|c| c.to_u32())
-            .collect())
-    }
-
     /// Returns a vCard containing contacts with the given ids.
     async fn make_vcard(&self, account_id: u32, contacts: Vec<u32>) -> Result<String> {
         let ctx = self.get_context(account_id).await?;
@@ -1505,20 +1462,6 @@ impl CommandApi {
         deltachat::contact::make_vcard(&ctx, &contacts).await
     }
 
-    /// Sets vCard containing the given contacts to the message draft.
-    async fn set_draft_vcard(
-        &self,
-        account_id: u32,
-        msg_id: u32,
-        contacts: Vec<u32>,
-    ) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
-        let contacts: Vec<_> = contacts.iter().map(|&c| ContactId::new(c)).collect();
-        let mut msg = Message::load_from_db(&ctx, MsgId::new(msg_id)).await?;
-        msg.make_vcard(&ctx, &contacts).await?;
-        msg.get_chat_id().set_draft(&ctx, Some(&mut msg)).await
-    }
-
     // ---------------------------------------------
     //                   chat
     // ---------------------------------------------
@@ -1567,6 +1510,55 @@ impl CommandApi {
         Ok(media.iter().map(|msg_id| msg_id.to_u32()).collect())
     }
 
+    /// Search next/previous message based on a given message and a list of types.
+    /// Typically used to implement the "next" and "previous" buttons
+    /// in a gallery or in a media player.
+    ///
+    /// one combined call for getting chat::get_next_media for both directions
+    /// the manual chat::get_next_media in only one direction is not exposed by the jsonrpc yet
+    ///
+    /// Deprecated 2023-10-03, use `get_chat_media` method
+    /// and navigate the returned array instead.
+    #[allow(deprecated)]
+    async fn get_neighboring_chat_media(
+        &self,
+        account_id: u32,
+        msg_id: u32,
+        message_type: MessageViewtype,
+        or_message_type2: Option<MessageViewtype>,
+        or_message_type3: Option<MessageViewtype>,
+    ) -> Result<(Option<u32>, Option<u32>)> {
+        let ctx = self.get_context(account_id).await?;
+
+        let msg_type: Viewtype = message_type.into();
+        let msg_type2: Viewtype = or_message_type2.map(|v| v.into()).unwrap_or_default();
+        let msg_type3: Viewtype = or_message_type3.map(|v| v.into()).unwrap_or_default();
+
+        let prev = chat::get_next_media(
+            &ctx,
+            MsgId::new(msg_id),
+            chat::Direction::Backward,
+            msg_type,
+            msg_type2,
+            msg_type3,
+        )
+        .await?
+        .map(|id| id.to_u32());
+
+        let next = chat::get_next_media(
+            &ctx,
+            MsgId::new(msg_id),
+            chat::Direction::Forward,
+            msg_type,
+            msg_type2,
+            msg_type3,
+        )
+        .await?
+        .map(|id| id.to_u32());
+
+        Ok((prev, next))
+    }
+
     // ---------------------------------------------
     //                   backup
     // ---------------------------------------------
@@ -1638,10 +1630,10 @@ impl CommandApi {
     ///
     /// This call will block until the QR code is ready,
     /// even if there is no concurrent call to [`CommandApi::provide_backup`],
-    /// but will fail after 60 seconds to avoid deadlocks.
+    /// but will fail after 10 seconds to avoid deadlocks.
     async fn get_backup_qr(&self, account_id: u32) -> Result<String> {
         let qr = tokio::time::timeout(
-            Duration::from_secs(60),
+            Duration::from_secs(10),
             self.inner_get_backup_qr(account_id),
         )
         .await
@@ -1657,13 +1649,13 @@ impl CommandApi {
     ///
     /// This call will block until the QR code is ready,
     /// even if there is no concurrent call to [`CommandApi::provide_backup`],
-    /// but will fail after 60 seconds to avoid deadlocks.
+    /// but will fail after 10 seconds to avoid deadlocks.
     ///
     /// Returns the QR code rendered as an SVG image.
     async fn get_backup_qr_svg(&self, account_id: u32) -> Result<String> {
         let ctx = self.get_context(account_id).await?;
         let qr = tokio::time::timeout(
-            Duration::from_secs(60),
+            Duration::from_secs(10),
             self.inner_get_backup_qr(account_id),
         )
         .await
@@ -1961,13 +1953,9 @@ impl CommandApi {
 
     async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
-        let mut message = data
-            .create_message(&ctx)
-            .await
-            .context("Failed to create message")?;
+        let mut message = data.create_message(&ctx).await?;
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
-            .await
-            .context("Failed to send created message")?
+            .await?
             .to_u32();
         Ok(msg_id)
     }
@@ -2004,7 +1992,9 @@ impl CommandApi {
     async fn get_draft(&self, account_id: u32, chat_id: u32) -> Result<Option<MessageObject>> {
         let ctx = self.get_context(account_id).await?;
         if let Some(draft) = ChatId::new(chat_id).get_draft(&ctx).await? {
-            Ok(MessageObject::from_msg_id(&ctx, draft.get_id()).await?)
+            Ok(Some(
+                MessageObject::from_msg_id(&ctx, draft.get_id()).await?,
+            ))
         } else {
             Ok(None)
         }
@@ -2130,7 +2120,8 @@ impl CommandApi {
     ) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
 
-        let mut msg = Message::new_text(text);
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(text);
 
         let message_id = deltachat::chat::send_msg(&ctx, ChatId::new(chat_id), &mut msg).await?;
         Ok(message_id.to_u32())
@@ -2173,9 +2164,7 @@ impl CommandApi {
                 .await?;
         }
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message).await?;
-        let message = MessageObject::from_msg_id(&ctx, msg_id)
-            .await?
-            .context("Just sent message does not exist")?;
+        let message = MessageObject::from_msg_id(&ctx, msg_id).await?;
         Ok((msg_id.to_u32(), message))
     }
 

deltachat-jsonrpc/src/api/types/account.rs

@@ -17,9 +17,6 @@ pub enum Account {
         // size: u32,
         profile_image: Option<String>, // TODO: This needs to be converted to work with blob http server.
         color: String,
-        /// Optional tag as "Work", "Family".
-        /// Meant to help profile owner to differ between profiles with similar names.
-        private_tag: Option<String>,
     },
     #[serde(rename_all = "camelCase")]
     Unconfigured { id: u32 },
@@ -34,14 +31,12 @@ impl Account {
             let color = color_int_to_hex_string(
                 Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
             );
-            let private_tag = ctx.get_config(Config::PrivateTag).await?;
             Ok(Account::Configured {
                 id,
                 display_name,
                 addr,
                 profile_image,
                 color,
-                private_tag,
             })
         } else {
             Ok(Account::Unconfigured { id })

deltachat-jsonrpc/src/api/types/chat.rs

@@ -32,7 +32,6 @@ pub struct FullChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
-    pinned: bool,
     // subtitle  - will be moved to frontend because it uses translation functions
     chat_type: u32,
     is_unpromoted: bool,
@@ -105,7 +104,6 @@ impl FullChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
-            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),
@@ -155,7 +153,6 @@ pub struct BasicChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
-    pinned: bool,
     chat_type: u32,
     is_unpromoted: bool,
     is_self_talk: bool,
@@ -183,7 +180,6 @@ impl BasicChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
-            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -88,17 +88,11 @@ pub(crate) async fn get_chat_list_item_by_id(
 
     let (last_updated, message_type) = match last_msgid {
         Some(id) => {
-            if let Some(last_message) =
-                deltachat::message::Message::load_from_db_optional(ctx, id).await?
-            {
-                (
-                    Some(last_message.get_timestamp() * 1000),
-                    Some(last_message.get_viewtype().into()),
-                )
-            } else {
-                // Message may be deleted by the time we try to load it.
-                (None, None)
-            }
+            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
+            (
+                Some(last_message.get_timestamp() * 1000),
+                Some(last_message.get_viewtype().into()),
+            )
         }
         None => (None, None),
     };

deltachat-jsonrpc/src/api/types/contact.rs

@@ -19,7 +19,6 @@ pub struct ContactObject {
     profile_image: Option<String>, // BLOBS
     name_and_addr: String,
     is_blocked: bool,
-    e2ee_avail: bool,
 
     /// True if the contact can be added to verified groups.
     ///
@@ -80,7 +79,6 @@ impl ContactObject {
             profile_image, //BLOBS
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
-            e2ee_avail: contact.e2ee_avail(context).await?,
             is_verified,
             is_profile_verified,
             verifier_id,

deltachat-jsonrpc/src/api/types/events.rs

@@ -98,14 +98,6 @@ pub enum EventType {
         contact_id: u32,
     },
 
-    /// Incoming reaction, should be notified.
-    #[serde(rename_all = "camelCase")]
-    IncomingReaction {
-        contact_id: u32,
-        msg_id: u32,
-        reaction: String,
-    },
-
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
@@ -252,11 +244,6 @@ pub enum EventType {
     #[serde(rename_all = "camelCase")]
     WebxdcRealtimeData { msg_id: u32, data: Vec<u8> },
 
-    /// Advertisement received over an ephemeral peer channel.
-    /// This can be used by bots to initiate peer-to-peer communication from their side.
-    #[serde(rename_all = "camelCase")]
-    WebxdcRealtimeAdvertisementReceived { msg_id: u32 },
-
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
     WebxdcInstanceDeleted { msg_id: u32 },
@@ -276,9 +263,6 @@ pub enum EventType {
     /// If `chat_id` is set to None, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
     #[serde(rename_all = "camelCase")]
     ChatlistItemChanged { chat_id: Option<u32> },
-
-    /// Inform than some events have been skipped due to event channel overflow.
-    EventChannelOverflow { n: u64 },
 }
 
 impl From<CoreEventType> for EventType {
@@ -310,15 +294,6 @@ impl From<CoreEventType> for EventType {
                 msg_id: msg_id.to_u32(),
                 contact_id: contact_id.to_u32(),
             },
-            CoreEventType::IncomingReaction {
-                contact_id,
-                msg_id,
-                reaction,
-            } => IncomingReaction {
-                contact_id: contact_id.to_u32(),
-                msg_id: msg_id.to_u32(),
-                reaction: reaction.as_str().to_string(),
-            },
             CoreEventType::IncomingMsg { chat_id, msg_id } => IncomingMsg {
                 chat_id: chat_id.to_u32(),
                 msg_id: msg_id.to_u32(),
@@ -395,11 +370,6 @@ impl From<CoreEventType> for EventType {
                 msg_id: msg_id.to_u32(),
                 data,
             },
-            CoreEventType::WebxdcRealtimeAdvertisementReceived { msg_id } => {
-                WebxdcRealtimeAdvertisementReceived {
-                    msg_id: msg_id.to_u32(),
-                }
-            }
             CoreEventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
                 msg_id: msg_id.to_u32(),
             },
@@ -408,10 +378,6 @@ impl From<CoreEventType> for EventType {
                 chat_id: chat_id.map(|id| id.to_u32()),
             },
             CoreEventType::ChatlistChanged => ChatlistChanged,
-            CoreEventType::EventChannelOverflow { n } => EventChannelOverflow { n },
-            #[allow(unreachable_patterns)]
-            #[cfg(test)]
-            _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
         }
     }
 }

deltachat-jsonrpc/src/api/types/message.rs

@@ -112,10 +112,8 @@ enum MessageQuote {
 }
 
 impl MessageObject {
-    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Option<Self>> {
-        let Some(message) = Message::load_from_db_optional(context, msg_id).await? else {
-            return Ok(None);
-        };
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+        let message = Message::load_from_db(context, msg_id).await?;
 
         let sender_contact = Contact::get_by_id(context, message.get_from_id())
             .await
@@ -185,7 +183,7 @@ impl MessageObject {
             .map(Into::into)
             .collect();
 
-        let message_object = MessageObject {
+        Ok(MessageObject {
             id: msg_id.to_u32(),
             chat_id: message.get_chat_id().to_u32(),
             from_id: message.get_from_id().to_u32(),
@@ -246,8 +244,7 @@ impl MessageObject {
             reactions,
 
             vcard_contact: vcard_contacts.first().cloned(),
-        };
-        Ok(Some(message_object))
+        })
     }
 }
 
@@ -493,7 +490,6 @@ pub struct MessageSearchResult {
     author_name: String,
     author_color: String,
     author_id: u32,
-    chat_id: u32,
     chat_profile_image: Option<String>,
     chat_color: String,
     chat_name: String,
@@ -533,7 +529,6 @@ impl MessageSearchResult {
             author_name,
             author_color: color_int_to_hex_string(sender.get_color()),
             author_id: sender.id.to_u32(),
-            chat_id: chat.id.to_u32(),
             chat_name: chat.get_name().to_owned(),
             chat_color,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
@@ -582,9 +577,7 @@ pub struct MessageData {
     pub file: Option<String>,
     pub location: Option<(f64, f64)>,
     pub override_sender_name: Option<String>,
-    /// Quoted message id. Takes preference over `quoted_text` (see below).
     pub quoted_message_id: Option<u32>,
-    pub quoted_text: Option<String>,
 }
 
 impl MessageData {
@@ -610,16 +603,16 @@ impl MessageData {
             message.set_location(latitude, longitude);
         }
         if let Some(id) = self.quoted_message_id {
-            let quoted_message = Message::load_from_db(context, MsgId::new(id))
-                .await
-                .context("Failed to load quoted message")?;
             message
-                .set_quote(context, Some(&quoted_message))
-                .await
-                .context("Failed to set quote")?;
-        } else if let Some(text) = self.quoted_text {
-            let protect = false;
-            message.set_quote_text(Some((text, protect)));
+                .set_quote(
+                    context,
+                    Some(
+                        &Message::load_from_db(context, MsgId::new(id))
+                            .await
+                            .context("message to quote could not be loaded")?,
+                    ),
+                )
+                .await?;
         }
         Ok(message)
     }
@@ -642,7 +635,7 @@ pub struct MessageInfo {
     error: Option<String>,
     rfc724_mid: String,
     server_urls: Vec<String>,
-    hop_info: String,
+    hop_info: Option<String>,
 }
 
 impl MessageInfo {

deltachat-jsonrpc/src/api/types/qr.rs

@@ -6,161 +6,80 @@ use typescript_type_def::TypeDef;
 #[serde(rename = "Qr", rename_all = "camelCase")]
 #[serde(tag = "kind")]
 pub enum QrObject {
-    /// Ask the user whether to verify the contact.
-    ///
-    /// If the user agrees, pass this QR code to [`crate::securejoin::join_securejoin`].
     AskVerifyContact {
-        /// ID of the contact.
         contact_id: u32,
-        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
-        /// Invite number.
         invitenumber: String,
-        /// Authentication code.
         authcode: String,
     },
-    /// Ask the user whether to join the group.
     AskVerifyGroup {
-        /// Group name.
         grpname: String,
-        /// Group ID.
         grpid: String,
-        /// ID of the contact.
         contact_id: u32,
-        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
-        /// Invite number.
         invitenumber: String,
-        /// Authentication code.
         authcode: String,
     },
-    /// Contact fingerprint is verified.
-    ///
-    /// Ask the user if they want to start chatting.
     FprOk {
-        /// Contact ID.
         contact_id: u32,
     },
-    /// Scanned fingerprint does not match the last seen fingerprint.
     FprMismatch {
-        /// Contact ID.
         contact_id: Option<u32>,
     },
-    /// The scanned QR code contains a fingerprint but no e-mail address.
     FprWithoutAddr {
-        /// Key fingerprint.
         fingerprint: String,
     },
-    /// Ask the user if they want to create an account on the given domain.
     Account {
-        /// Server domain name.
         domain: String,
     },
-    /// Provides a backup that can be retrieved using iroh-net based backup transfer protocol.
-    Backup2 {
-        /// Authentication token.
-        auth_token: String,
-        /// Iroh node address.
-        node_addr: String,
+    Backup {
+        ticket: String,
     },
-    /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
     },
-    /// Ask the user if they want to use the given proxy.
-    ///
-    /// Note that HTTP(S) URLs without a path
-    /// and query parameters are treated as HTTP(S) proxy URL.
-    /// UI may want to still offer to open the URL
-    /// in the browser if QR code contents
-    /// starts with `http://` or `https://`
-    /// and the QR code was not scanned from
-    /// the proxy configuration screen.
-    Proxy {
-        /// Proxy URL.
-        ///
-        /// This is the URL that is going to be added.
-        url: String,
-        /// Host extracted from the URL to display in the UI.
-        host: String,
-        /// Port extracted from the URL to display in the UI.
-        port: u16,
-    },
-    /// Contact address is scanned.
-    ///
-    /// Optionally, a draft message could be provided.
-    /// Ask the user if they want to start chatting.
     Addr {
-        /// Contact ID.
         contact_id: u32,
-        /// Draft message.
         draft: Option<String>,
     },
-    /// URL scanned.
-    ///
-    /// Ask the user if they want to open a browser or copy the URL to clipboard.
-    Url { url: String },
-    /// Text scanned.
-    ///
-    /// Ask the user if they want to copy the text to clipboard.
-    Text { text: String },
-    /// Ask the user if they want to withdraw their own QR code.
+    Url {
+        url: String,
+    },
+    Text {
+        text: String,
+    },
     WithdrawVerifyContact {
-        /// Contact ID.
         contact_id: u32,
-        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
-        /// Invite number.
         invitenumber: String,
-        /// Authentication code.
         authcode: String,
     },
-    /// Ask the user if they want to withdraw their own group invite QR code.
     WithdrawVerifyGroup {
-        /// Group name.
         grpname: String,
-        /// Group ID.
         grpid: String,
-        /// Contact ID.
         contact_id: u32,
-        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
-        /// Invite number.
         invitenumber: String,
-        /// Authentication code.
         authcode: String,
     },
-    /// Ask the user if they want to revive their own QR code.
     ReviveVerifyContact {
-        /// Contact ID.
         contact_id: u32,
-        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
-        /// Invite number.
         invitenumber: String,
-        /// Authentication code.
         authcode: String,
     },
-    /// Ask the user if they want to revive their own group invite QR code.
     ReviveVerifyGroup {
-        /// Contact ID.
         grpname: String,
-        /// Group ID.
         grpid: String,
-        /// Contact ID.
         contact_id: u32,
-        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
-        /// Invite number.
         invitenumber: String,
-        /// Authentication code.
         authcode: String,
     },
-    /// `dclogin:` scheme parameters.
-    ///
-    /// Ask the user if they want to login with the email address.
-    Login { address: String },
+    Login {
+        address: String,
+    },
 }
 
 impl From<Qr> for QrObject {
@@ -210,12 +129,8 @@ impl From<Qr> for QrObject {
             }
             Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint },
             Qr::Account { domain } => QrObject::Account { domain },
-            Qr::Backup2 {
-                ref node_addr,
-                auth_token,
-            } => QrObject::Backup2 {
-                node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
-                auth_token,
+            Qr::Backup { ticket } => QrObject::Backup {
+                ticket: ticket.to_string(),
             },
             Qr::WebrtcInstance {
                 domain,
@@ -224,7 +139,6 @@ impl From<Qr> for QrObject {
                 domain,
                 instance_pattern,
             },
-            Qr::Proxy { url, host, port } => QrObject::Proxy { url, host, port },
             Qr::Addr { contact_id, draft } => {
                 let contact_id = contact_id.to_u32();
                 QrObject::Addr { contact_id, draft }

deltachat-jsonrpc/src/lib.rs

@@ -1,6 +1,4 @@
 #![recursion_limit = "256"]
-#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
-#![cfg_attr(not(test), forbid(clippy::string_slice))]
 pub mod api;
 pub use yerpc;
 
@@ -85,7 +83,7 @@ mod tests {
             assert_eq!(result, response.to_owned());
         }
         {
-            let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":""}]}"#;
+            let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
             session.handle_incoming(request).await;
             let result = receiver.recv().await?;

deltachat-jsonrpc/typescript/package.json

@@ -3,7 +3,7 @@
   "dependencies": {
     "@deltachat/tiny-emitter": "3.0.0",
     "isomorphic-ws": "^4.0.1",
-    "yerpc": "^0.6.2"
+    "yerpc": "^0.4.3"
   },
   "devDependencies": {
     "@types/chai": "^4.2.21",
@@ -32,10 +32,6 @@
   "license": "MPL-2.0",
   "main": "dist/deltachat.js",
   "name": "@deltachat/jsonrpc-client",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/deltachat/deltachat-core-rust.git"
-  },
   "scripts": {
     "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
     "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
@@ -58,5 +54,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.149.0"
+  "version": "1.138.5"
 }

deltachat-jsonrpc/typescript/test/online.ts

@@ -86,7 +86,10 @@ describe("online tests", function () {
       null
     );
     const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
-    const eventPromise = waitForEvent(dc, "IncomingMsg", accountId2);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId2),
+      waitForEvent(dc, "IncomingMsg", accountId2),
+    ]);
 
     await dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello");
     const { chatId: chatIdOnAccountB } = await eventPromise;
@@ -116,7 +119,10 @@ describe("online tests", function () {
       null
     );
     const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
-    const eventPromise = waitForEvent(dc, "IncomingMsg", accountId2);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId2),
+      waitForEvent(dc, "IncomingMsg", accountId2),
+    ]);
     dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello2");
     // wait for message from A
     console.log("wait for message from A");
@@ -137,7 +143,10 @@ describe("online tests", function () {
     );
     expect(message.text).equal("Hello2");
     // Send message back from B to A
-    const eventPromise2 = waitForEvent(dc, "IncomingMsg", accountId1);
+    const eventPromise2 = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId1),
+      waitForEvent(dc, "IncomingMsg", accountId1),
+    ]);
     dc.rpc.miscSendTextMessage(accountId2, chatId, "super secret message");
     // Check if answer arrives at A and if it is encrypted
     await eventPromise2;

deltachat-repl/Cargo.toml

@@ -1,21 +1,20 @@
 [package]
 name = "deltachat-repl"
-version = "1.149.0"
+version = "1.138.5"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [dependencies]
-anyhow = { workspace = true }
-deltachat = { workspace = true, features = ["internals"]}
+ansi_term = "0.12.1"
+anyhow = "1"
+deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
-log = { workspace = true }
-nu-ansi-term = { workspace = true }
-qr2term = "0.3.3"
-rusqlite = { workspace = true }
+log = "0.4.21"
+pretty_env_logger = "0.5"
+rusqlite = "0.31"
 rustyline = "14"
-tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
-tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tokio = { version = "1.37.0", features = ["fs", "rt-multi-thread", "macros"] }
 
 [features]
 default = ["vendored"]

deltachat-repl/src/cmdline.rs

@@ -22,7 +22,6 @@ use deltachat::mimeparser::SystemMessage;
 use deltachat::peer_channels::{send_webxdc_realtime_advertisement, send_webxdc_realtime_data};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
-use deltachat::qr_code_generator::create_qr_svg;
 use deltachat::reaction::send_reaction;
 use deltachat::receive_imf::*;
 use deltachat::sql;
@@ -340,6 +339,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  receive-backup <qr>\n\
                  export-keys\n\
                  import-keys\n\
+                 export-setup\n\
                  poke [<eml-file>|<folder>|<addr> <key-file>]\n\
                  reset <flags>\n\
                  stop\n\
@@ -356,7 +356,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  configure\n\
                  connect\n\
                  disconnect\n\
-                 fetch\n\
                  connectivity\n\
                  maybenetwork\n\
                  housekeeping\n\
@@ -426,7 +425,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  checkqr <qr-content>\n\
                  joinqr <qr-content>\n\
                  setqr <qr-content>\n\
-                 createqrsvg <qr-content>\n\
                  providerinfo <addr>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
@@ -489,9 +487,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "send-backup" => {
             let provider = BackupProvider::prepare(&context).await?;
-            let qr = format_backup(&provider.qr())?;
-            println!("QR code: {}", qr);
-            qr2term::print_qr(qr.as_str())?;
+            let qr = provider.qr();
+            println!("QR code: {}", format_backup(&qr)?);
             provider.await?;
         }
         "receive-backup" => {
@@ -507,6 +504,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "import-keys" => {
             imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
         }
+        "export-setup" => {
+            let setup_code = create_setup_code(&context);
+            let file_name = blobdir.join("autocrypt-setup-message.html");
+            let file_content = render_setup_file(&context, &setup_code).await?;
+            fs::write(&file_name, file_content).await?;
+            println!(
+                "Setup message written to: {}\nSetup code: {}",
+                file_name.display(),
+                &setup_code,
+            );
+        }
         "poke" => {
             ensure!(poke_spec(&context, Some(arg1)).await, "Poke failed");
         }
@@ -1004,7 +1012,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             if !arg1.is_empty() {
-                let mut draft = Message::new_text(arg1.to_string());
+                let mut draft = Message::new(Viewtype::Text);
+                draft.set_text(arg1.to_string());
                 sel_chat
                     .as_ref()
                     .unwrap()
@@ -1027,7 +1036,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty(),
                 "Please specify text to add as device message."
             );
-            let mut msg = Message::new_text(arg1.to_string());
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text(arg1.to_string());
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "listmedia" => {
@@ -1249,19 +1259,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Err(err) => println!("Cannot set config from QR code: {err:?}"),
             }
         }
-        "createqrsvg" => {
-            ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let svg = create_qr_svg(arg1)?;
-            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
-            fs::write(&file, svg).await?;
-            println!("{file:#?} written.");
-        }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            let proxy_enabled = context
-                .get_config_bool(config::Config::ProxyEnabled)
+            let socks5_enabled = context
+                .get_config_bool(config::Config::Socks5Enabled)
                 .await?;
-            match provider::get_provider_info(&context, arg1, proxy_enabled).await {
+            match provider::get_provider_info(&context, arg1, socks5_enabled).await {
                 Some(info) => {
                     println!("Information for provider belonging to {arg1}:");
                     println!("status: {}", info.status as u32);

deltachat-repl/src/main.rs

@@ -9,7 +9,10 @@
 extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
+use std::io::{self, Write};
+use std::process::Command;
 
+use ansi_term::Color;
 use anyhow::{bail, Error};
 use deltachat::chat::ChatId;
 use deltachat::config;
@@ -19,7 +22,6 @@ use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
 use deltachat::EventType;
 use log::{error, info, warn};
-use nu_ansi_term::Color;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
@@ -30,7 +32,6 @@ use rustyline::{
 };
 use tokio::fs;
 use tokio::runtime::Handle;
-use tracing_subscriber::EnvFilter;
 
 mod cmdline;
 use self::cmdline::*;
@@ -150,7 +151,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 13] = [
+const IMEX_COMMANDS: [&str; 14] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -161,12 +162,13 @@ const IMEX_COMMANDS: [&str; 13] = [
     "receive-backup",
     "export-keys",
     "import-keys",
+    "export-setup",
     "poke",
     "reset",
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&str; 10] = [
     "info",
     "set",
     "get",
@@ -174,7 +176,6 @@ const DB_COMMANDS: [&str; 11] = [
     "configure",
     "connect",
     "disconnect",
-    "fetch",
     "connectivity",
     "maybenetwork",
     "housekeeping",
@@ -240,13 +241,12 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 12] = [
+const MISC_COMMANDS: [&str; 11] = [
     "getqr",
     "getqrsvg",
     "getbadqr",
     "checkqr",
     "joinqr",
-    "createqrsvg",
     "fileinfo",
     "clear",
     "exit",
@@ -417,9 +417,6 @@ async fn handle_cmd(
         "disconnect" => {
             ctx.stop_io().await;
         }
-        "fetch" => {
-            ctx.background_fetch().await?;
-        }
         "configure" => {
             ctx.configure().await?;
         }
@@ -449,7 +446,12 @@ async fn handle_cmd(
                     qr.replace_range(12..22, "0000000000")
                 }
                 println!("{qr}");
-                qr2term::print_qr(qr.as_str())?;
+                let output = Command::new("qrencode")
+                    .args(["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                    .output()
+                    .expect("failed to execute process");
+                io::stdout().write_all(&output.stdout).unwrap();
+                io::stderr().write_all(&output.stderr).unwrap();
             }
         }
         "getqrsvg" => {
@@ -481,10 +483,9 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    tracing_subscriber::fmt()
-        .with_env_filter(
-            EnvFilter::from_default_env().add_directive("deltachat_repl=info".parse()?),
-        )
+    pretty_env_logger::formatted_timed_builder()
+        .parse_default_env()
+        .filter_module("deltachat_repl", log::LevelFilter::Info)
         .init();
 
     let args = std::env::args().collect();

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.149.0"
+version = "1.138.5"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -13,13 +13,10 @@ classifiers = [
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS :: MacOS X",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Programming Language :: Python :: 3.13",
     "Topic :: Communications :: Chat",
     "Topic :: Communications :: Email"
 ]

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -250,16 +250,12 @@ class Account:
         """
         return Chat(self, self._rpc.secure_join(self.id, qrdata))
 
-    def get_qr_code(self) -> str:
-        """Get Setup-Contact QR Code text.
+    def get_qr_code(self) -> tuple[str, str]:
+        """Get Setup-Contact QR Code text and SVG data.
 
-        This data needs to be transferred to another Delta Chat account
+        this data needs to be transferred to another Delta Chat account
         in a second channel, typically used by mobiles with QRcode-show + scan UX.
         """
-        return self._rpc.get_chat_securejoin_qr_code(self.id, None)
-
-    def get_qr_code_svg(self) -> tuple[str, str]:
-        """Get Setup-Contact QR code text and SVG."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.id, None)
 
     def get_message_by_id(self, msg_id: int) -> Message:
@@ -301,12 +297,6 @@ class Account:
             if event.kind == EventType.INCOMING_MSG:
                 return event
 
-    def wait_for_incoming_msg(self):
-        """Wait for incoming message and return it.
-
-        Consumes all events before the next incoming message event."""
-        return self.get_message_by_id(self.wait_for_incoming_msg_event().msg_id)
-
     def wait_for_securejoin_inviter_success(self):
         while True:
             event = self.wait_for_event()

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import calendar
 from dataclasses import dataclass
-from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Optional, Union
 
 from ._utils import AttrDict
@@ -96,11 +95,7 @@ class Chat:
         """Return encryption info for this chat."""
         return self._rpc.get_chat_encryption_info(self.account.id, self.id)
 
-    def get_qr_code(self) -> str:
-        """Get Join-Group QR code text."""
-        return self._rpc.get_chat_securejoin_qr_code(self.account.id, self.id)
-
-    def get_qr_code_svg(self) -> tuple[str, str]:
+    def get_qr_code(self) -> tuple[str, str]:
         """Get Join-Group QR code text and SVG data."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.account.id, self.id)
 
@@ -270,11 +265,3 @@ class Chat:
             location["message"] = Message(self.account, location.msg_id)
             locations.append(location)
         return locations
-
-    def send_contact(self, contact: Contact):
-        """Send contact to the chat."""
-        vcard = contact.make_vcard()
-        with NamedTemporaryFile(suffix=".vcard") as f:
-            f.write(vcard.encode())
-            f.flush()
-            self._rpc.send_msg(self.account.id, self.id, {"viewtype": ViewType.VCARD, "file": f.name})

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -63,7 +63,6 @@ class EventType(str, Enum):
     CHATLIST_ITEM_CHANGED = "ChatlistItemChanged"
     CONFIG_SYNCED = "ConfigSynced"
     WEBXDC_REALTIME_DATA = "WebxdcRealtimeData"
-    WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED = "WebxdcRealtimeAdvertisementReceived"
 
 
 class ChatId(IntEnum):
@@ -116,7 +115,6 @@ class ViewType(str, Enum):
     FILE = "File"
     VIDEOCHAT_INVITATION = "VideochatInvitation"
     WEBXDC = "Webxdc"
-    VCARD = "Vcard"
 
 
 class SystemMessageType(str, Enum):

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -36,10 +36,6 @@ class Contact:
         """Delete contact."""
         self._rpc.delete_contact(self.account.id, self.id)
 
-    def reset_encryption(self) -> None:
-        """Reset contact encryption."""
-        self._rpc.reset_contact_encryption(self.account.id, self.id)
-
     def set_name(self, name: str) -> None:
         """Change the name of this contact."""
         self._rpc.change_contact_name(self.account.id, self.id, name)
@@ -64,6 +60,3 @@ class Contact:
             self.account,
             self._rpc.create_chat_by_contact_id(self.account.id, self.id),
         )
-
-    def make_vcard(self) -> str:
-        return self._rpc.make_vcard(self.account.id, [self.id])

deltachat-rpc-client/src/deltachat_rpc_client/direct_imap.py

@@ -9,19 +9,18 @@ import io
 import pathlib
 import ssl
 from contextlib import contextmanager
-from typing import TYPE_CHECKING
 
 from imap_tools import (
     AND,
     Header,
     MailBox,
+    MailBoxTls,
     MailMessage,
     MailMessageFlags,
     errors,
 )
 
-if TYPE_CHECKING:
-    from . import Account
+from . import Account, const
 
 FLAGS = b"FLAGS"
 FETCH = b"FETCH"
@@ -36,15 +35,28 @@ class DirectImap:
         self.connect()
 
     def connect(self):
-        # Assume the testing server supports TLS on port 993.
         host = self.account.get_config("configured_mail_server")
-        port = 993
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
 
         user = self.account.get_config("addr")
-        host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
+        if security == const.SocketSecurity.PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.SocketSecurity.STARTTLS:
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
+        elif security == const.SocketSecurity.PLAIN or security == const.SocketSecurity.SSL:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -2,7 +2,7 @@ import json
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Optional, Union
 
-from ._utils import AttrDict, futuremethod
+from ._utils import AttrDict
 from .const import EventType
 from .contact import Contact
 
@@ -70,11 +70,3 @@ class Message:
             event = self.account.wait_for_event()
             if event.kind == EventType.MSG_DELIVERED and event.msg_id == self.id:
                 break
-
-    @futuremethod
-    def send_webxdc_realtime_advertisement(self):
-        yield self._rpc.send_webxdc_realtime_advertisement.future(self.account.id, self.id)
-
-    @futuremethod
-    def send_webxdc_realtime_data(self, data) -> None:
-        yield self._rpc.send_webxdc_realtime_data.future(self.account.id, self.id, list(data))

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -114,13 +114,13 @@ class ACFactory:
         return to_client.run_until(lambda e: e.kind == EventType.INCOMING_MSG)
 
 
-@pytest.fixture
+@pytest.fixture()
 def rpc(tmp_path) -> AsyncGenerator:
     rpc_server = Rpc(accounts_dir=str(tmp_path / "accounts"))
     with rpc_server:
         yield rpc_server
 
 
-@pytest.fixture
+@pytest.fixture()
 def acfactory(rpc) -> AsyncGenerator:
     return ACFactory(DeltaChat(rpc))

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -177,7 +177,7 @@ class Rpc:
                 account_id = event["contextId"]
                 queue = self.get_queue(account_id)
                 event = event["event"]
-                logging.debug("account_id=%d got an event %s", account_id, event)
+                print("account_id=%d got an event %s" % (account_id, event), file=sys.stderr)
                 queue.put(event)
         except Exception:
             # Log an exception if the event loop dies.

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -126,7 +126,8 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
 
     alice.set_config("download_limit", "1")
 
-    msg = bob.wait_for_incoming_msg()
+    event = bob.wait_for_incoming_msg_event()
+    msg = bob.get_message_by_id(event.msg_id)
     chat_id = msg.get_snapshot().chat_id
     msg.get_snapshot().chat.accept()
     bob.get_chat_by_id(chat_id).send_message(
@@ -134,15 +135,13 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
         html=base64.b64encode(os.urandom(300000)).decode("utf-8"),
     )
 
-    message = alice.wait_for_incoming_msg()
-    snapshot = message.get_snapshot()
-    assert snapshot.download_state == const.DownloadState.AVAILABLE
+    msg_id = alice.wait_for_incoming_msg_event().msg_id
+
+    assert alice.get_message_by_id(msg_id).get_snapshot().download_state == const.DownloadState.AVAILABLE
 
     alice.clear_all_events()
-
-    snapshot = message.get_snapshot()
-    chat_id = snapshot.chat_id
-    alice._rpc.download_full_message(alice.id, message.id)
+    chat_id = alice.get_message_by_id(msg_id).get_snapshot().chat_id
+    alice._rpc.download_full_message(alice.id, msg_id)
 
     wait_for_chatlist_specific_item(alice, chat_id)
 
@@ -178,7 +177,8 @@ def test_imap_sync_seen_msgs(acfactory: ACFactory) -> None:
 
     alice_chat_bob.send_text("hello")
 
-    msg = bob.wait_for_incoming_msg()
+    event = bob.wait_for_incoming_msg_event()
+    msg = bob.get_message_by_id(event.msg_id)
     bob_chat_id = msg.get_snapshot().chat_id
     msg.get_snapshot().chat.accept()
 
@@ -189,7 +189,8 @@ def test_imap_sync_seen_msgs(acfactory: ACFactory) -> None:
     # make sure alice_second_device already received the message
     alice_second_device.wait_for_incoming_msg_event()
 
-    msg = alice.wait_for_incoming_msg()
+    event = alice.wait_for_incoming_msg_event()
+    msg = alice.get_message_by_id(event.msg_id)
     alice_second_device.clear_all_events()
     msg.mark_seen()
 
@@ -210,7 +211,6 @@ def test_multidevice_sync_chat(acfactory: ACFactory) -> None:
     alice_second_device.clear_all_events()
     alice_chat_bob.pin()
     wait_for_chatlist_specific_item(alice_second_device, alice_chat_bob.id)
-    assert alice_second_device.get_chat_by_id(alice_chat_bob.id).get_basic_snapshot().pinned
 
     alice_second_device.clear_all_events()
     alice_chat_bob.mute()

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -1,236 +0,0 @@
-#!/usr/bin/env python3
-"""
-Testing webxdc iroh connectivity
-
-If you want to debug iroh at rust-trace/log level set
-
-    RUST_LOG=iroh_net=trace,iroh_gossip=trace
-"""
-
-import logging
-import os
-import threading
-import time
-
-import pytest
-
-from deltachat_rpc_client import EventType
-
-
-@pytest.fixture
-def path_to_webxdc(request):
-    p = request.path.parent.parent.parent.joinpath("test-data/webxdc/chess.xdc")
-    assert p.exists()
-    return str(p)
-
-
-def log(msg):
-    logging.info(msg)
-
-
-def setup_realtime_webxdc(ac1, ac2, path_to_webxdc):
-    assert ac1.get_config("webxdc_realtime_enabled") == "1"
-    assert ac2.get_config("webxdc_realtime_enabled") == "1"
-    ac1_ac2_chat = ac1.create_chat(ac2)
-    ac2.create_chat(ac1)
-
-    # share a webxdc app between ac1 and ac2
-    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="play", file=path_to_webxdc)
-    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
-    assert ac2_webxdc_msg.get_snapshot().text == "play"
-
-    # send iroh announcements simultaneously
-    log("sending ac1 -> ac2 realtime advertisement and additional message")
-    ac1_webxdc_msg.send_webxdc_realtime_advertisement()
-
-    log("sending ac2 -> ac1 realtime advertisement and additional message")
-    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
-
-    return ac1_webxdc_msg, ac2_webxdc_msg
-
-
-def setup_thread_send_realtime_data(msg, data):
-    def thread_run():
-        for _i in range(10):
-            msg.send_webxdc_realtime_data(data)
-            time.sleep(1)
-
-    threading.Thread(target=thread_run, daemon=True).start()
-
-
-def wait_receive_realtime_data(msg_data_list):
-    account = msg_data_list[0][0].account
-    msg_data_list = msg_data_list[:]
-
-    log(f"account {account.id}: waiting for realtime data {msg_data_list}")
-    while msg_data_list:
-        event = account.wait_for_event()
-        if event.kind == EventType.WEBXDC_REALTIME_DATA:
-            for i, (msg, data) in enumerate(msg_data_list):
-                if msg.id == event.msg_id:
-                    assert list(data) == event.data
-                    log(f"msg {msg.id}: got correct realtime data {data}")
-                    del msg_data_list[i]
-                    break
-
-
-def test_realtime_sequentially(acfactory, path_to_webxdc):
-    """Test two peers trying to establish connection sequentially."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("webxdc_realtime_enabled", "1")
-    ac2.set_config("webxdc_realtime_enabled", "1")
-    ac1.create_chat(ac2)
-    ac2.create_chat(ac1)
-
-    # share a webxdc app between ac1 and ac2
-    ac1_webxdc_msg = acfactory.send_message(from_account=ac1, to_account=ac2, text="play", file=path_to_webxdc)
-    ac2_webxdc_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
-    snapshot = ac2_webxdc_msg.get_snapshot()
-    assert snapshot.text == "play"
-
-    # send iroh announcements sequentially
-    log("sending ac1 -> ac2 realtime advertisement and additional message")
-    ac1_webxdc_msg.send_webxdc_realtime_advertisement()
-    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping1")
-
-    log("waiting for incoming message on ac2")
-    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "ping1"
-
-    log("sending ac2 -> ac1 realtime advertisement and additional message")
-    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
-    acfactory.send_message(from_account=ac2, to_account=ac1, text="ping2")
-
-    log("waiting for incoming message on ac1")
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "ping2"
-
-    log("sending realtime data ac1 -> ac2")
-    # Test that 128 KB of data can be sent in a single message.
-    data = os.urandom(128000)
-    ac1_webxdc_msg.send_webxdc_realtime_data(data)
-
-    log("ac2: waiting for realtime data")
-    while 1:
-        event = ac2.wait_for_event()
-        if event.kind == EventType.WEBXDC_REALTIME_DATA:
-            assert event.data == list(data)
-            break
-
-
-def test_realtime_simultaneously(acfactory, path_to_webxdc):
-    """Test two peers trying to establish connection simultaneously."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("webxdc_realtime_enabled", "1")
-    ac2.set_config("webxdc_realtime_enabled", "1")
-
-    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
-
-    setup_thread_send_realtime_data(ac1_webxdc_msg, [10])
-    wait_receive_realtime_data([(ac2_webxdc_msg, [10])])
-
-
-def test_two_parallel_realtime_simultaneously(acfactory, path_to_webxdc):
-    """Test two peers trying to establish connection simultaneously."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("webxdc_realtime_enabled", "1")
-    ac2.set_config("webxdc_realtime_enabled", "1")
-
-    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
-    ac1_webxdc_msg2, ac2_webxdc_msg2 = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
-
-    setup_thread_send_realtime_data(ac1_webxdc_msg, [10])
-    setup_thread_send_realtime_data(ac1_webxdc_msg2, [20])
-    setup_thread_send_realtime_data(ac2_webxdc_msg, [30])
-    setup_thread_send_realtime_data(ac2_webxdc_msg2, [40])
-
-    wait_receive_realtime_data([(ac1_webxdc_msg, [30]), (ac1_webxdc_msg2, [40])])
-    wait_receive_realtime_data([(ac2_webxdc_msg, [10]), (ac2_webxdc_msg2, [20])])
-
-
-def test_no_duplicate_messages(acfactory, path_to_webxdc):
-    """Test that messages are received only once."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("webxdc_realtime_enabled", "1")
-    ac2.set_config("webxdc_realtime_enabled", "1")
-
-    ac1_ac2_chat = ac1.create_chat(ac2)
-
-    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="webxdc", file=path_to_webxdc)
-
-    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
-    ac2_webxdc_msg.get_snapshot().chat.accept()
-    assert ac2_webxdc_msg.get_snapshot().text == "webxdc"
-
-    # Issue a "send" call in parallel with sending advertisement.
-    # Previously due to a bug this caused subscribing to the channel twice.
-    ac2_webxdc_msg.send_webxdc_realtime_data.future(b"foobar")
-    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
-
-    def thread_run():
-        for i in range(10):
-            data = str(i).encode()
-            ac1_webxdc_msg.send_webxdc_realtime_data(data)
-            time.sleep(1)
-
-    threading.Thread(target=thread_run, daemon=True).start()
-
-    while 1:
-        event = ac2.wait_for_event()
-        if event.kind == EventType.WEBXDC_REALTIME_DATA:
-            n = int(bytes(event.data).decode())
-            break
-
-    while 1:
-        event = ac2.wait_for_event()
-        if event.kind == EventType.WEBXDC_REALTIME_DATA:
-            assert int(bytes(event.data).decode()) > n
-            break
-
-
-def test_no_reordering(acfactory, path_to_webxdc):
-    """Test that sending a lot of realtime messages does not result in reordering."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("webxdc_realtime_enabled", "1")
-    ac2.set_config("webxdc_realtime_enabled", "1")
-
-    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
-
-    setup_thread_send_realtime_data(ac1_webxdc_msg, b"hello")
-    wait_receive_realtime_data([(ac2_webxdc_msg, b"hello")])
-
-    for i in range(200):
-        ac1_webxdc_msg.send_webxdc_realtime_data([i])
-
-    for i in range(200):
-        while 1:
-            event = ac2.wait_for_event()
-            if event.kind == EventType.WEBXDC_REALTIME_DATA and bytes(event.data) != b"hello":
-                if event.data[0] == i:
-                    break
-                pytest.fail("Reordering detected")
-
-
-def test_advertisement_after_chatting(acfactory, path_to_webxdc):
-    """Test that realtime advertisement is assigned to the correct message after chatting."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("webxdc_realtime_enabled", "1")
-    ac2.set_config("webxdc_realtime_enabled", "1")
-
-    ac1_ac2_chat = ac1.create_chat(ac2)
-    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="WebXDC", file=path_to_webxdc)
-    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
-    assert ac2_webxdc_msg.get_snapshot().text == "WebXDC"
-
-    ac1_ac2_chat.send_text("Hello!")
-    ac2_hello_msg = ac2.wait_for_incoming_msg()
-    ac2_hello_msg_snapshot = ac2_hello_msg.get_snapshot()
-    assert ac2_hello_msg_snapshot.text == "Hello!"
-    ac2_hello_msg_snapshot.chat.accept()
-
-    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
-    while 1:
-        event = ac1.wait_for_event()
-        if event.kind == EventType.WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED:
-            assert event.msg_id == ac1_webxdc_msg.id
-            break

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,15 +1,13 @@
 import logging
-import time
 
 import pytest
-
 from deltachat_rpc_client import Chat, EventType, SpecialContactId
 
 
 def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code = alice.get_qr_code()
+    qr_code, _svg = alice.get_qr_code()
     bob.secure_join(qr_code)
 
     alice.wait_for_securejoin_inviter_success()
@@ -32,45 +30,23 @@ def test_qr_setup_contact(acfactory, tmp_path) -> None:
     bob2.export_self_keys(tmp_path)
 
     logging.info("Bob imports a key")
-    bob.import_self_keys(tmp_path)
+    bob.import_self_keys(tmp_path / "private-key-default.asc")
 
     assert bob.get_config("key_id") == "2"
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert not bob_contact_alice_snapshot.is_verified
 
 
-def test_qr_setup_contact_svg(acfactory) -> None:
-    alice = acfactory.new_configured_account()
-    _, _, domain = alice.get_config("addr").rpartition("@")
-
-    _qr_code, svg = alice.get_qr_code_svg()
-
-    alice.set_config("displayname", "Alice")
-
-    # Test that display name is used
-    # in SVG and no address is visible.
-    _qr_code, svg = alice.get_qr_code_svg()
-    assert domain not in svg
-    assert "Alice" in svg
-
-
 @pytest.mark.parametrize("protect", [True, False])
-def test_qr_securejoin(acfactory, protect, tmp_path):
-    alice, bob, fiona = acfactory.get_online_accounts(3)
+def test_qr_securejoin(acfactory, protect):
+    alice, bob = acfactory.get_online_accounts(2)
 
-    # Setup second device for Alice
-    # to test observing securejoin protocol.
-    alice.export_backup(tmp_path)
-    files = list(tmp_path.glob("*.tar"))
-    alice2 = acfactory.get_unconfigured_account()
-    alice2.import_backup(files[0])
-
-    logging.info("Alice creates a group")
-    alice_chat = alice.create_group("Group", protect=protect)
+    logging.info("Alice creates a verified group")
+    alice_chat = alice.create_group("Verified group", protect=protect)
     assert alice_chat.get_basic_snapshot().is_protected == protect
 
-    logging.info("Bob joins the group")
-    qr_code = alice_chat.get_qr_code()
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
 
     # Check that at least some of the handshake messages are deleted.
@@ -98,21 +74,6 @@ def test_qr_securejoin(acfactory, protect, tmp_path):
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
-    # Start second Alice device.
-    # Alice observes securejoin protocol and verifies Bob on second device.
-    alice2.start_io()
-    alice2.wait_for_securejoin_inviter_success()
-    alice2_contact_bob = alice2.get_contact_by_addr(bob.get_config("addr"))
-    alice2_contact_bob_snapshot = alice2_contact_bob.get_snapshot()
-    assert alice2_contact_bob_snapshot.is_verified
-
-    # The QR code token is synced, so alice2 must be able to handle join requests.
-    logging.info("Fiona joins the group via alice2")
-    alice.stop_io()
-    fiona.secure_join(qr_code)
-    alice2.wait_for_securejoin_inviter_success()
-    fiona.wait_for_securejoin_joiner_success()
-
 
 def test_qr_securejoin_contact_request(acfactory) -> None:
     """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
@@ -130,7 +91,7 @@ def test_qr_securejoin_contact_request(acfactory) -> None:
 
     alice_chat = alice.create_group("Verified group", protect=True)
     logging.info("Bob joins verified group")
-    qr_code = alice_chat.get_qr_code()
+    qr_code, _svg = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
     while True:
         event = bob.wait_for_event()
@@ -145,7 +106,7 @@ def test_qr_readreceipt(acfactory) -> None:
     alice, bob, charlie = acfactory.get_online_accounts(3)
 
     logging.info("Bob and Charlie setup contact with Alice")
-    qr_code = alice.get_qr_code()
+    qr_code, _svg = alice.get_qr_code()
 
     bob.secure_join(qr_code)
     charlie.secure_join(qr_code)
@@ -207,13 +168,13 @@ def test_setup_contact_resetup(acfactory) -> None:
     """Tests that setup contact works after Alice resets the device and changes the key."""
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code = alice.get_qr_code()
+    qr_code, _svg = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     alice = acfactory.resetup_account(alice)
 
-    qr_code = alice.get_qr_code()
+    qr_code, _svg = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -227,7 +188,7 @@ def test_verified_group_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code = chat.get_qr_code()
+    qr_code, _svg = chat.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -244,7 +205,7 @@ def test_verified_group_recovery(acfactory) -> None:
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code = ac3.get_qr_code()
+    qr_code, _svg = ac3.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -291,7 +252,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code = chat.get_qr_code()
+    qr_code, _svg = chat.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -308,7 +269,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code = ac3.get_qr_code()
+    qr_code, _svg = ac3.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -326,6 +287,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
 
     ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
     ac3_chat.remove_contact(ac3_contact_ac2)
+    ac3_chat.add_contact(ac3_contact_ac2)
 
     msg_id = ac2.wait_for_incoming_msg_event().msg_id
     message = ac2.get_message_by_id(msg_id)
@@ -335,8 +297,6 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert "removed" in snapshot.text
 
-    ac3_chat.add_contact(ac3_contact_ac2)
-
     event = ac2.wait_for_incoming_msg_event()
     msg_id = event.msg_id
     chat_id = event.chat_id
@@ -376,7 +336,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
     ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
 
     logging.info("ac3: verify with ac2")
-    qr_code = ac2.get_qr_code()
+    qr_code, _svg = ac2.get_qr_code()
     ac3.secure_join(qr_code)
     ac2.wait_for_securejoin_inviter_success()
 
@@ -386,7 +346,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
 
     logging.info("ac1: create verified group that ac2 fully joins")
     ch1 = ac1.create_group("Group", protect=True)
-    qr_code = ch1.get_qr_code()
+    qr_code, _svg = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
 
@@ -399,7 +359,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
-    qr_code = ch1.get_qr_code()
+    qr_code, _svg = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.remove()
     logging.info("ac2 now has pending bobstate but ac1 is shutoff")
@@ -421,7 +381,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
-    qr_code = vg.get_qr_code()
+    qr_code, _svg = vg.get_qr_code()
     ac4.secure_join(qr_code)
     ac3.wait_for_securejoin_inviter_success()
     while 1:
@@ -442,7 +402,7 @@ def test_qr_new_group_unblocked(acfactory):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_chat = ac1.create_group("Group for joining", protect=True)
-    qr_code = ac1_chat.get_qr_code()
+    qr_code, _svg = ac1_chat.get_qr_code()
     ac2.secure_join(qr_code)
 
     ac1.wait_for_securejoin_inviter_success()
@@ -460,19 +420,15 @@ def test_qr_new_group_unblocked(acfactory):
 
 def test_aeap_flow_verified(acfactory):
     """Test that a new address is added to a contact when it changes its address."""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    # ac1new is only used to get a new address.
-    ac1new = acfactory.new_preconfigured_account()
+    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
 
     logging.info("ac1: create verified-group QR, ac2 scans and joins")
     chat = ac1.create_group("hello", protect=True)
     assert chat.get_basic_snapshot().is_protected
-    qr_code = chat.get_qr_code()
+    qr_code, _svg = chat.get_qr_code()
     logging.info("ac2: start QR-code based join-group protocol")
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
-    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("sending first message")
     msg_out = chat.send_text("old address").get_snapshot()
@@ -508,12 +464,12 @@ def test_gossip_verification(acfactory) -> None:
     alice, bob, carol = acfactory.get_online_accounts(3)
 
     # Bob verifies Alice.
-    qr_code = alice.get_qr_code()
+    qr_code, _svg = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     # Bob verifies Carol.
-    qr_code = carol.get_qr_code()
+    qr_code, _svg = carol.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -564,17 +520,16 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     ac3_chat = ac3.create_group("Verified group", protect=True)
 
     # ac1 joins ac3 group.
-    ac3_qr_code = ac3_chat.get_qr_code()
+    ac3_qr_code, _svg = ac3_chat.get_qr_code()
     ac1.secure_join(ac3_qr_code)
     ac1.wait_for_securejoin_joiner_success()
 
     # ac1 waits for member added message and creates a QR code.
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Member Me ({}) added by {}.".format(ac1.get_config("addr"), ac3.get_config("addr"))
-    ac1_qr_code = snapshot.chat.get_qr_code()
+    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
 
     # ac2 verifies ac1
-    qr_code = ac1.get_qr_code()
+    qr_code, _svg = ac1.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -585,29 +540,17 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     # ac1 resetups the account.
     ac1 = acfactory.resetup_account(ac1)
 
-    # Loop sending message from ac1 to ac2
-    # until ac2 accepts new ac1 key.
-    #
-    # This may not happen immediately because resetup of ac1
-    # rewinds "smeared timestamp" so Date: header for messages
-    # sent by new ac1 are in the past compared to the last Date:
-    # header sent by old ac1.
-    while True:
-        # ac1 sends a message to ac2.
-        ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
-        ac1_chat_ac2 = ac1_contact_ac2.create_chat()
-        ac1_chat_ac2.send_text("Hello!")
+    # ac1 sends a message to ac2.
+    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
+    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
+    ac1_chat_ac2.send_text("Hello!")
 
-        # ac2 receives a message.
-        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-        assert snapshot.text == "Hello!"
-        logging.info("ac2 received Hello!")
+    # ac2 receives a message.
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
 
-        # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
-        logging.info("ac2 addr={}, ac1 addr={}".format(ac2.get_config("addr"), ac1.get_config("addr")))
-        if not ac2_contact_ac1.get_snapshot().is_verified:
-            break
-        time.sleep(1)
+    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
+    assert not ac2_contact_ac1.get_snapshot().is_verified
 
     # ac1 goes offline.
     ac1.remove()
@@ -646,7 +589,7 @@ def test_withdraw_securejoin_qr(acfactory):
     assert alice_chat.get_basic_snapshot().is_protected
     logging.info("Bob joins verified group")
 
-    qr_code = alice_chat.get_qr_code()
+    qr_code, _svg = alice_chat.get_qr_code()
     bob_chat = bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -669,8 +612,7 @@ def test_withdraw_securejoin_qr(acfactory):
     logging.info("Bob scanned withdrawn QR code")
     while True:
         event = alice.wait_for_event()
-        if (
-            event.kind == EventType.WARNING
-            and "Ignoring vg-request-with-auth message because of invalid auth code." in event.msg
-        ):
+        if event.kind == EventType.MSGS_CHANGED and event.chat_id != 0:
             break
+    snapshot = alice.get_message_by_id(event.msg_id).get_snapshot()
+    assert snapshot.text == "Cannot establish guaranteed end-to-end encryption with {}".format(bob.get_config("addr"))

deltachat-rpc-client/tests/test_something.py

@@ -1,15 +1,12 @@
-import base64
 import concurrent.futures
 import json
 import logging
 import os
-import socket
 import subprocess
 import time
 from unittest.mock import MagicMock
 
 import pytest
-
 from deltachat_rpc_client import Contact, EventType, Message, events
 from deltachat_rpc_client.const import DownloadState, MessageState
 from deltachat_rpc_client.direct_imap import DirectImap
@@ -57,8 +54,8 @@ def test_acfactory(acfactory) -> None:
             if event.progress == 1000:  # Success
                 break
         else:
-            logging.info(event)
-    logging.info("Successful configuration")
+            print(event)
+    print("Successful configuration")
 
 
 def test_configure_starttls(acfactory) -> None:
@@ -71,38 +68,6 @@ def test_configure_starttls(acfactory) -> None:
     assert account.is_configured()
 
 
-def test_configure_ip(acfactory) -> None:
-    account = acfactory.new_preconfigured_account()
-
-    domain = account.get_config("addr").rsplit("@")[-1]
-    ip_address = socket.gethostbyname(domain)
-
-    # This should fail TLS check.
-    account.set_config("mail_server", ip_address)
-    with pytest.raises(JsonRpcError):
-        account.configure()
-
-
-def test_configure_alternative_port(acfactory) -> None:
-    """Test that configuration with alternative port 443 works."""
-    account = acfactory.new_preconfigured_account()
-
-    account.set_config("mail_port", "443")
-    account.set_config("send_port", "443")
-
-    account.configure()
-
-
-def test_configure_username(acfactory) -> None:
-    account = acfactory.new_preconfigured_account()
-
-    addr = account.get_config("addr")
-    account.set_config("mail_user", addr)
-    account.configure()
-
-    assert account.get_config("configured_mail_user") == addr
-
-
 def test_account(acfactory) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
@@ -138,12 +103,12 @@ def test_account(acfactory) -> None:
     assert alice.get_chatlist(snapshot=True)
     assert alice.get_qr_code()
     assert alice.get_fresh_messages()
+    assert alice.get_next_messages()
 
     # Test sending empty message.
     assert len(bob.wait_next_messages()) == 0
     alice_chat_bob.send_text("")
     messages = bob.wait_next_messages()
-    assert bob.get_next_messages() == messages
     assert len(messages) == 1
     message = messages[0]
     snapshot = message.get_snapshot()
@@ -246,7 +211,6 @@ def test_contact(acfactory) -> None:
     assert repr(alice_contact_bob)
     alice_contact_bob.block()
     alice_contact_bob.unblock()
-    alice_contact_bob.reset_encryption()
     alice_contact_bob.set_name("new name")
     alice_contact_bob.get_encryption_info()
     snapshot = alice_contact_bob.get_snapshot()
@@ -434,7 +398,7 @@ def test_provider_info(rpc) -> None:
     assert provider_info["id"] == "gmail"
 
     # Disable MX record resolution.
-    rpc.set_config(account_id, "proxy_enabled", "1")
+    rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
 
@@ -649,31 +613,3 @@ def test_markseen_contact_request(acfactory, tmp_path):
         if event.kind == EventType.MSGS_NOTICED:
             break
     assert message2.get_snapshot().state == MessageState.IN_SEEN
-
-
-def test_get_http_response(acfactory):
-    alice = acfactory.new_configured_account()
-    http_response = alice._rpc.get_http_response(alice.id, "https://example.org")
-    assert http_response["mimetype"] == "text/html"
-    assert b"<title>Example Domain</title>" in base64.b64decode((http_response["blob"] + "==").encode())
-
-
-def test_configured_imap_certificate_checks(acfactory):
-    alice = acfactory.new_configured_account()
-    configured_certificate_checks = alice.get_config("configured_imap_certificate_checks")
-
-    # Certificate checks should be configured (not None)
-    assert configured_certificate_checks
-
-    # 0 is the value old Delta Chat core versions used
-    # to mean user entered "imap_certificate_checks=0" (Automatic)
-    # and configuration failed to use strict TLS checks
-    # so it switched strict TLS checks off.
-    #
-    # New versions of Delta Chat are not disabling TLS checks
-    # unless users explicitly disables them
-    # or provider database says provider has invalid certificates.
-    #
-    # Core 1.142.4, 1.142.5 and 1.142.6 saved this value due to bug.
-    # This test is a regression test to prevent this happening again.
-    assert configured_certificate_checks != "0"

deltachat-rpc-client/tests/test_vcard.py

@@ -1,15 +0,0 @@
-def test_vcard(acfactory) -> None:
-    alice, bob = acfactory.get_online_accounts(2)
-
-    bob_addr = bob.get_config("addr")
-    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
-    alice_contact_charlie = alice.create_contact("charlie@example.org", "Charlie")
-
-    alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_contact(alice_contact_charlie)
-
-    event = bob.wait_for_incoming_msg_event()
-    message = bob.get_message_by_id(event.msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.vcard_contact
-    assert snapshot.vcard_contact.addr == "charlie@example.org"

deltachat-rpc-client/tests/test_webxdc_iroh.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""
+Testing webxdc iroh connectivity
+
+If you want to debug iroh at rust-trace/log level set
+
+    RUST_LOG=iroh_net=trace,iroh_gossip=trace
+"""
+
+import pytest
+
+import time
+import os
+import sys
+import logging
+import random
+import itertools
+import sys
+
+from deltachat_rpc_client import DeltaChat, EventType, SpecialContactId
+
+
+@pytest.fixture()
+def path_to_webxdc():
+    return "../test-data/webxdc/chess.xdc"
+
+
+def test_realtime_sequentially(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection sequentially."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping0")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping0"
+
+    def log(msg):
+        print()
+        print("*" * 80 + "\n" + msg + "\n", file=sys.stderr)
+        print()
+
+    # share a webxdc app between ac1 and ac2
+    ac1_webxdc_msg = acfactory.send_message(from_account=ac1, to_account=ac2, text="play", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    snapshot = ac2_webxdc_msg.get_snapshot()
+    assert snapshot.text == "play"
+
+    # send iroh announcements sequentially
+    log("sending ac1 -> ac2 realtime advertisement and additional message")
+    ac1._rpc.send_webxdc_realtime_advertisement(ac1.id, ac1_webxdc_msg.id)
+    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping1")
+
+    log("waiting for incoming message on ac2")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping1"
+
+    log("sending ac2 -> ac1 realtime advertisement and additional message")
+    ac2._rpc.send_webxdc_realtime_advertisement(ac2.id, ac2_webxdc_msg.id)
+    acfactory.send_message(from_account=ac2, to_account=ac1, text="ping2")
+
+    log("waiting for incoming message on ac1")
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping2"
+
+    log("sending realtime data ac1 -> ac2")
+    ac1._rpc.send_webxdc_realtime_data(ac1.id, ac1_webxdc_msg.id, [13, 15, 17])
+
+    log("ac2: waiting for realtime data")
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            assert event.data == [13, 15, 17]
+            break
+
+
+def test_realtime_simultaneously(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection simultaneously."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping0")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping0"
+
+    def log(msg):
+        print()
+        print("*" * 80 + "\n" + msg + "\n", file=sys.stderr)
+        print()
+
+    # share a webxdc app between ac1 and ac2
+    ac1_webxdc_msg = acfactory.send_message(from_account=ac1, to_account=ac2, text="play", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    snapshot = ac2_webxdc_msg.get_snapshot()
+    assert snapshot.text == "play"
+
+    # send iroh announcements simultaneously
+    log("sending ac1 -> ac2 realtime advertisement and additional message")
+    ac1._rpc.send_webxdc_realtime_advertisement(ac1.id, ac1_webxdc_msg.id)
+    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping1")
+
+    log("sending ac2 -> ac1 realtime advertisement and additional message")
+    ac2._rpc.send_webxdc_realtime_advertisement(ac2.id, ac2_webxdc_msg.id)
+    acfactory.send_message(from_account=ac2, to_account=ac1, text="ping2")
+
+    # Ensure that advertisements have been received.
+
+    log("waiting for incoming message on ac2")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping1"
+
+    log("waiting for incoming message on ac1")
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping2"
+
+    log("sending realtime data ac1 -> ac2")
+    ac1._rpc.send_webxdc_realtime_data(ac1.id, ac1_webxdc_msg.id, [13, 15, 17])
+
+    log("ac2: waiting for realtime data")
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            assert event.data == [13, 15, 17]
+            break

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.149.0"
+version = "1.138.5"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -10,18 +10,19 @@ keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
 
 [dependencies]
-deltachat-jsonrpc = { workspace = true }
-deltachat = { workspace = true }
+deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
+deltachat = { path = "..", default-features = false }
 
-anyhow = { workspace = true }
-futures-lite = { workspace = true }
-log = { workspace = true }
-serde_json = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
-tokio = { workspace = true, features = ["io-std"] }
-tokio-util = { workspace = true }
-tracing-subscriber = { workspace = true, features = ["env-filter"] }
-yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
+anyhow = "1"
+env_logger = { version = "0.11.3" }
+futures-lite = "2.3.0"
+log = "0.4"
+serde_json = "1"
+serde = { version = "1.0", features = ["derive"] }
+tokio = { version = "1.37.0", features = ["io-std"] }
+tokio-util = "0.7.9"
+yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [features]
 default = ["vendored"]

deltachat-rpc-server/npm-package/README.md

@@ -7,7 +7,7 @@ This simplifies cross-compilation and even reduces binary size (no CFFI layer an
 
 ## Usage
 
-> The **minimum** nodejs version for this package is `16`
+> The **minimum** nodejs version for this package is `20.11`
 
 ```
 npm i @deltachat/stdio-rpc-server @deltachat/jsonrpc-client
@@ -20,9 +20,7 @@ import { C } from "@deltachat/jsonrpc-client";
 async function main() {
     const dc = await startDeltaChat("deltachat-data");
     console.log(await dc.rpc.getSystemInfo());
-    dc.close()
 }
-main()
 ```
 
 For a more complete example refer to https://github.com/deltachat-bot/echo/pull/69/files (TODO change link when pr is merged).
@@ -46,11 +44,12 @@ references:
 When you import this package it searches for the rpc server in the following locations and order:
 
 1. `DELTA_CHAT_RPC_SERVER` environment variable
-2. use the PATH when `{takeVersionFromPATH: true}` is supplied in the options. 
+2. in PATH
+   - unless `DELTA_CHAT_SKIP_PATH=1` is specified
+   - searches in .cargo/bin directory first
+   - but there an additional version check is performed
 3. prebuilds in npm packages
 
-so by default it uses the prebuilds.
-
 ## How do you built this package in CI
 
 - To build platform packages, run the `build_platform_package.py` script:

deltachat-rpc-server/npm-package/index.d.ts

@@ -1,8 +1,8 @@
 import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
 
 export interface SearchOptions {
-  /** whether take deltachat-rpc-server inside of $PATH*/
-  takeVersionFromPATH: boolean;
+  /** whether to disable looking for deltachat-rpc-server inside of $PATH */
+  skipSearchInPath: boolean;
 
   /** whether to disable the DELTA_CHAT_RPC_SERVER environment variable */
   disableEnvPath: boolean;

deltachat-rpc-server/npm-package/index.js

@@ -1,9 +1,15 @@
 //@ts-check
-import { spawn } from "node:child_process";
-import { stat } from "node:fs/promises";
+import { execFile, spawn } from "node:child_process";
+import { stat, readdir } from "node:fs/promises";
 import os from "node:os";
+import { join, basename } from "node:path";
 import process from "node:process";
-import { ENV_VAR_NAME, PATH_EXECUTABLE_NAME } from "./src/const.js";
+import { promisify } from "node:util";
+import {
+  ENV_VAR_NAME,
+  PATH_EXECUTABLE_NAME,
+  SKIP_SEARCH_IN_PATH,
+} from "./src/const.js";
 import {
   ENV_VAR_LOCATION_NOT_FOUND,
   FAILED_TO_START_SERVER_EXECUTABLE,
@@ -11,6 +17,9 @@ import {
   NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR,
 } from "./src/errors.js";
 
+// Because this is not compiled by typescript, esm needs this stuff (` with { type: "json" };`,
+// nodejs still complains about it being experimental, but deno also uses it, so treefit bets taht it will become standard)
+import package_json from "./package.json" with { type: "json" };
 import { createRequire } from "node:module";
 
 function findRPCServerInNodeModules() {
@@ -22,12 +31,7 @@ function findRPCServerInNodeModules() {
     return resolve(package_name);
   } catch (error) {
     console.debug("findRpcServerInNodeModules", error);
-    const require = createRequire(import.meta.url);
-    if (
-      Object.keys(require("./package.json").optionalDependencies).includes(
-        package_name
-      )
-    ) {
+    if (Object.keys(package_json.optionalDependencies).includes(package_name)) {
       throw new Error(NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name));
     } else {
       throw new Error(NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR());
@@ -35,13 +39,38 @@ function findRPCServerInNodeModules() {
   }
 }
 
+/**
+ * @returns {Promise<string>}
+ */
+async function getLocationInPath() {
+  const exec = promisify(execFile);
+
+  if (os.platform() === "win32") {
+    const { stdout: executable } = await exec("where", [PATH_EXECUTABLE_NAME], {
+      shell: true,
+    });
+    return executable;
+  }
+
+  try {
+    const { stdout: executable } = await exec(
+      "command",
+      ["-v", PATH_EXECUTABLE_NAME],
+      { shell: true }
+    );
+    return executable;
+  } catch (error) {
+    if (error.code > 0) return "";
+    else throw error;
+  }
+}
+
 /** @type {import("./index").FnTypes.getRPCServerPath} */
-export async function getRPCServerPath(options = {}) {
-  const { takeVersionFromPATH, disableEnvPath } = {
-    takeVersionFromPATH: false,
-    disableEnvPath: false,
-    ...options,
-  };
+export async function getRPCServerPath(
+  options = { skipSearchInPath: false, disableEnvPath: false }
+) {
+  // @TODO: improve confusing naming of these options
+  const { skipSearchInPath, disableEnvPath } = options;
   // 1. check if it is set as env var
   if (process.env[ENV_VAR_NAME] && !disableEnvPath) {
     try {
@@ -56,9 +85,35 @@ export async function getRPCServerPath(options = {}) {
     return process.env[ENV_VAR_NAME];
   }
 
-  // 2. check if PATH should be used
-  if (takeVersionFromPATH) {
-    return PATH_EXECUTABLE_NAME;
+  // 2. check if it can be found in PATH
+  if (!process.env[SKIP_SEARCH_IN_PATH] && !skipSearchInPath) {
+    const executable = await getLocationInPath();
+
+    // by just trying to execute it and then use "command -v deltachat-rpc-server" (unix) or "where deltachat-rpc-server" (windows) to get the path to the executable
+    if (executable.length > 1) {
+      // test if it is the right version
+      try {
+        // for some unknown reason it is in stderr and not in stdout
+        const { stderr } = await promisify(execFile)(
+          executable,
+          ["--version"],
+          { shell: true }
+        );
+        const version = stderr.slice(0, stderr.indexOf("\n"));
+        if (package_json.version !== version) {
+          throw new Error(
+            `version mismatch: (npm package: ${package_json.version})  (installed ${PATH_EXECUTABLE_NAME} version: ${version})`
+          );
+        } else {
+          return executable;
+        }
+      } catch (error) {
+        console.error(
+          "Found executable in PATH, but there was an error: " + error
+        );
+        console.error("So falling back to using prebuild...");
+      }
+    }
   }
   // 3. check for prebuilds
 
@@ -68,11 +123,11 @@ export async function getRPCServerPath(options = {}) {
 import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
 
 /** @type {import("./index").FnTypes.startDeltaChat} */
-export async function startDeltaChat(directory, options = {}) {
+export async function startDeltaChat(directory, options) {
   const pathToServerBinary = await getRPCServerPath(options);
   const server = spawn(pathToServerBinary, {
     env: {
-      RUST_LOG: process.env.RUST_LOG,
+      RUST_LOG: process.env.RUST_LOG || "info",
       DC_ACCOUNTS_PATH: directory,
     },
     stdio: ["pipe", "pipe", options.muteStdErr ? "ignore" : "inherit"],

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "1.149.0"
+  "version": "1.138.5"
 }

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -55,10 +55,9 @@ for (const { folder_name, package_name } of platform_package_names) {
 }
 
 if (is_local) {
-  package_json.peerDependencies["@deltachat/jsonrpc-client"] =
-    `file:${join(expected_cwd, "/../../deltachat-jsonrpc/typescript")}`;
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] = 'file:../../deltachat-jsonrpc/typescript'
 } else {
-  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*";
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*"
 }
 
 await fs.writeFile("./package.json", JSON.stringify(package_json, null, 4));

deltachat-rpc-server/npm-package/src/const.js

@@ -2,4 +2,5 @@
 
 export const PATH_EXECUTABLE_NAME = 'deltachat-rpc-server'
 
-export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"
+export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"
+export const SKIP_SEARCH_IN_PATH = "DELTA_CHAT_SKIP_PATH"

deltachat-rpc-server/src/main.rs

@@ -11,7 +11,7 @@ use deltachat::constants::DC_VERSION_STR;
 use deltachat_jsonrpc::api::{Accounts, CommandApi};
 use futures_lite::stream::StreamExt;
 use tokio::io::{self, AsyncBufReadExt, BufReader};
-use tracing_subscriber::EnvFilter;
+use tracing_subscriber::{prelude::*, EnvFilter};
 use yerpc::RpcServer as _;
 
 #[cfg(target_family = "unix")]
@@ -29,9 +29,6 @@ async fn main() {
     // "For technical reasons, stdin is implemented by using an ordinary blocking read on a separate
     // thread, and it is impossible to cancel that read. This can make shutdown of the runtime hang
     // until the user presses enter."
-    if let Err(error) = &r {
-        log::error!("Fatal error: {error:#}.")
-    }
     std::process::exit(if r.is_ok() { 0 } else { 1 });
 }
 
@@ -64,13 +61,13 @@ async fn main_impl() -> Result<()> {
     #[cfg(target_family = "unix")]
     let mut sigterm = signal_unix::signal(signal_unix::SignalKind::terminate())?;
 
-    // Logs from `log` crate and traces from `tracing` crate
-    // are configurable with `RUST_LOG` environment variable
-    // and go to stderr to avoid interferring with JSON-RPC using stdout.
-    tracing_subscriber::fmt()
-        .with_env_filter(EnvFilter::from_default_env())
-        .with_writer(std::io::stderr)
-        .init();
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    tracing_subscriber::registry()
+        .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr))
+        .with(EnvFilter::builder().from_env_lossy())
+        .try_init()
+        .ok();
 
     let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "accounts".to_string());
     log::info!("Starting with accounts directory `{}`.", path);

deny.toml

@@ -1,6 +1,7 @@
 [advisories]
 ignore = [
     "RUSTSEC-2020-0071",
+    "RUSTSEC-2022-0093",
 
     # Timing attack on RSA.
     # Delta Chat does not use RSA for new keys
@@ -9,15 +10,11 @@ ignore = [
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
 
+    # Unmaintained ansi_term
+    "RUSTSEC-2021-0139",
+
     # Unmaintained encoding
     "RUSTSEC-2021-0153",
-
-    # Unmaintained proc-macro-error
-    # <https://rustsec.org/advisories/RUSTSEC-2024-0370>
-    "RUSTSEC-2024-0370",
-
-    # Unmaintained instant
-    "RUSTSEC-2024-0384",
 ]
 
 [bans]
@@ -26,46 +23,86 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
+     { name = "asn1-rs-derive", version = "0.4.0" },
+     { name = "asn1-rs-impl", version = "0.1.0" },
+     { name = "asn1-rs", version = "0.5.2" },
      { name = "async-channel", version = "1.9.0" },
+     { name = "base16ct", version = "0.1.1" },
      { name = "base64", version = "<0.21" },
      { name = "base64", version = "0.21.7" },
      { name = "bitflags", version = "1.3.2" },
+     { name = "block-buffer", version = "<0.10" },
+     { name = "convert_case", version = "0.4.0" },
+     { name = "curve25519-dalek", version = "3.2.0" },
+     { name = "darling_core", version = "<0.14" },
+     { name = "darling_macro", version = "<0.14" },
+     { name = "darling", version = "<0.14" },
+     { name = "der_derive", version = "0.6.1" },
+     { name = "derive_more", version = "0.99.17" },
+     { name = "der-parser", version = "8.2.0" },
+     { name = "der", version = "0.6.1" },
+     { name = "digest", version = "<0.10" },
+     { name = "dlopen2", version = "0.4.1" },
+     { name = "ed25519-dalek", version = "1.0.1" },
+     { name = "ed25519", version = "1.5.3" },
+     { name = "env_logger", version = "0.10.2" },
      { name = "event-listener", version = "2.5.3" },
      { name = "event-listener", version = "4.0.3" },
      { name = "fastrand", version = "1.9.0" },
-     { name = "fiat-crypto", version = "0.1.20" },
      { name = "futures-lite", version = "1.13.0" },
      { name = "getrandom", version = "<0.2" },
+     { name = "http-body", version = "0.4.6" },
      { name = "http", version = "0.2.12" },
+     { name = "hyper", version = "0.14.28" },
+     { name = "idna", version = "0.4.0" },
+     { name = "netlink-packet-core", version = "0.5.0" },
+     { name = "netlink-packet-route", version = "0.15.0" },
      { name = "nix", version = "0.26.4" },
-     { name = "num_enum_derive", version = "0.5.11" },
-     { name = "num_enum", version = "0.5.11" },
-     { name = "proc-macro-crate", version = "1.3.1" },
+     { name = "oid-registry", version = "0.6.1" },
+     { name = "pem-rfc7468", version = "0.6.0" },
+     { name = "pem", version = "1.1.1" },
+     { name = "pkcs8", version = "0.9.0" },
+     { name = "proc-macro-error-attr", version = "0.4.12" },
+     { name = "proc-macro-error", version = "0.4.12" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
+     { name = "rcgen", version = "<0.12.1" },
      { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
+     { name = "ring", version = "0.16.20" },
+     { name = "sec1", version = "0.3.0" },
+     { name = "sha2", version = "<0.10" },
+     { name = "signature", version = "1.6.4" },
+     { name = "spin", version = "<0.9.6" },
+     { name = "spki", version = "0.6.0" },
+     { name = "ssh-encoding", version = "0.1.0" },
+     { name = "ssh-key", version = "0.5.1" },
      { name = "sync_wrapper", version = "0.1.2" },
+     { name = "synstructure", version = "0.12.6" },
      { name = "syn", version = "1.0.109" },
+     { name = "system-configuration-sys", version = "0.5.0" },
+     { name = "system-configuration", version = "0.5.1" },
      { name = "time", version = "<0.3" },
-     { name = "toml_edit", version = "0.19.15" },
+     { name = "toml_edit", version = "0.21.1" },
+     { name = "untrusted", version = "0.7.1" },
      { name = "wasi", version = "<0.11" },
      { name = "windows_aarch64_gnullvm", version = "<0.52" },
      { name = "windows_aarch64_msvc", version = "<0.52" },
      { name = "windows-core", version = "<0.54.0" },
      { name = "windows_i686_gnu", version = "<0.52" },
      { name = "windows_i686_msvc", version = "<0.52" },
-     { name = "windows-sys", version = "<0.59" },
+     { name = "windows-sys", version = "<0.52" },
      { name = "windows-targets", version = "<0.52" },
+     { name = "windows", version = "0.32.0" },
      { name = "windows", version = "<0.54.0" },
      { name = "windows_x86_64_gnullvm", version = "<0.52" },
      { name = "windows_x86_64_gnu", version = "<0.52" },
      { name = "windows_x86_64_msvc", version = "<0.52" },
      { name = "winnow", version = "0.5.40" },
-     { name = "winreg", version = "0.50.0" },
+     { name = "x509-parser", version = "<0.16.0" },
 ]
 
 

flake.lock

@@ -3,15 +3,15 @@
     "android": {
       "inputs": {
         "devshell": "devshell",
-        "flake-utils": "flake-utils",
+        "flake-utils": "flake-utils_2",
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1731356359,
-        "narHash": "sha256-vYqJnu6jotmWpPT4DgzHVdvNIZcKZCIUqS8QaptsZA0=",
+        "lastModified": 1712088936,
+        "narHash": "sha256-mVjeSWQiR/t4UZ9fUawY9OEPAhY1R3meYG+0oh8DUBs=",
         "owner": "tadfisher",
         "repo": "android-nixpkgs",
-        "rev": "c028ead7e88edb2e94cd7c90ee37593f63ae494a",
+        "rev": "2d8181caef279f19c4a33dc694723f89ffc195d4",
         "type": "github"
       },
       "original": {
@@ -22,17 +22,18 @@
     },
     "devshell": {
       "inputs": {
+        "flake-utils": "flake-utils",
         "nixpkgs": [
           "android",
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1728330715,
-        "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
+        "lastModified": 1711099426,
+        "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
+        "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8",
         "type": "github"
       },
       "original": {
@@ -47,11 +48,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1731393059,
-        "narHash": "sha256-rmzi0GHEwpzg1LGfGPO4SRD7D6QGV3UYGQxkJvn+J5U=",
+        "lastModified": 1714112748,
+        "narHash": "sha256-jq6Cpf/pQH85p+uTwPPrGG8Ky/zUOTwMJ7mcqc5M4So=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "fda8d5b59bb0dc0021ad3ba1d722f9ef6d36e4d9",
+        "rev": "3ae4b908a795b6a3824d401a0702e11a7157d7e1",
         "type": "github"
       },
       "original": {
@@ -65,11 +66,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1726560853,
-        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
         "type": "github"
       },
       "original": {
@@ -83,11 +84,29 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1726560853,
-        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_3"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
@@ -101,11 +120,11 @@
         "nixpkgs": "nixpkgs_3"
       },
       "locked": {
-        "lastModified": 1721727458,
-        "narHash": "sha256-r/xppY958gmZ4oTfLiHN0ZGuQ+RSTijDblVgVLFi1mw=",
+        "lastModified": 1713520724,
+        "narHash": "sha256-CO8MmVDmqZX2FovL75pu5BvwhW+Vugc7Q6ze7Hj8heI=",
         "owner": "nix-community",
         "repo": "naersk",
-        "rev": "3fb418eaf352498f6b6c30592e3beb63df42ef11",
+        "rev": "c5037590290c6c7dae2e42e7da1e247e54ed2d49",
         "type": "github"
       },
       "original": {
@@ -116,11 +135,11 @@
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1730207686,
-        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
         "type": "github"
       },
       "original": {
@@ -131,11 +150,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1731139594,
-        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
+        "lastModified": 1711703276,
+        "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
+        "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089",
         "type": "github"
       },
       "original": {
@@ -147,11 +166,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1731139594,
-        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
+        "lastModified": 1713895582,
+        "narHash": "sha256-cfh1hi+6muQMbi9acOlju3V1gl8BEaZBXBR9jQfQi4U=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
+        "rev": "572af610f6151fd41c212f897c71f7056e3fb518",
         "type": "github"
       },
       "original": {
@@ -163,9 +182,10 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 0,
-        "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=",
-        "path": "/nix/store/zq2axpgzd5kykk1v446rkffj3bxa2m2h-source",
+        "lastModified": 1711668574,
+        "narHash": "sha256-u1dfs0ASQIEr1icTVrsKwg2xToIpn7ZXxW3RHfHxshg=",
+        "path": "/nix/store/9fpv0kjq9a80isa1wkkvrdqsh9dpcn05-source",
+        "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659",
         "type": "path"
       },
       "original": {
@@ -175,11 +195,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1731139594,
-        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
+        "lastModified": 1714076141,
+        "narHash": "sha256-Drmja/f5MRHZCskS6mvzFqxEaZMeciScCTFxWVLqWEY=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
+        "rev": "7bb2ccd8cdc44c91edba16c48d2c8f331fb3d856",
         "type": "github"
       },
       "original": {
@@ -193,7 +213,7 @@
       "inputs": {
         "android": "android",
         "fenix": "fenix",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_3",
         "naersk": "naersk",
         "nix-filter": "nix-filter",
         "nixpkgs": "nixpkgs_4"
@@ -202,11 +222,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1731342671,
-        "narHash": "sha256-36eYDHoPzjavnpmEpc2MXdzMk557S0YooGms07mDuKk=",
+        "lastModified": 1714031783,
+        "narHash": "sha256-xS/niQsq1CQPOe4M4jvVPO2cnXS/EIeRG5gIopUbk+Q=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "fc98e0657abf3ce07eed513e38274c89bbb2f8ad",
+        "rev": "56bee2ddafa6177b19c631eedc88d43366553223",
         "type": "github"
       },
       "original": {
@@ -245,6 +265,21 @@
         "repo": "default",
         "type": "github"
       }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -18,9 +18,9 @@
         manifest = (pkgs.lib.importTOML ./Cargo.toml).package;
         androidSdk = android.sdk.${system} (sdkPkgs:
           builtins.attrValues {
-            inherit (sdkPkgs) ndk-27-0-11902837 cmdline-tools-latest;
+            inherit (sdkPkgs) ndk-24-0-8215888 cmdline-tools-latest;
           });
-        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/27.0.11902837";
+        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/24.0.8215888";
 
         rustSrc = nix-filter.lib {
           root = ./.;
@@ -257,21 +257,13 @@
 
         androidAttrs = {
           armeabi-v7a = {
-            cc = "armv7a-linux-androideabi21-clang";
+            cc = "armv7a-linux-androideabi19-clang";
             rustTarget = "armv7-linux-androideabi";
           };
           arm64-v8a = {
             cc = "aarch64-linux-android21-clang";
             rustTarget = "aarch64-linux-android";
           };
-          x86 = {
-            cc = "i686-linux-android21-clang";
-            rustTarget = "i686-linux-android";
-          };
-          x86_64 = {
-            cc = "x86_64-linux-android21-clang";
-            rustTarget = "x86_64-linux-android";
-          };
         };
 
         mkAndroidRustPackage = arch: packageName:
@@ -533,30 +525,27 @@
               };
           };
 
-        devShells.default =
-          let
-            pkgs = import nixpkgs {
-              system = system;
-              overlays = [ fenix.overlays.default ];
-            };
-          in
-          pkgs.mkShell {
-
-            buildInputs = with pkgs; [
-              (fenix.packages.${system}.complete.withComponents [
-                "cargo"
-                "clippy"
-                "rust-src"
-                "rustc"
-                "rustfmt"
-              ])
-              cargo-deny
-              rust-analyzer-nightly
-              cargo-nextest
-              perl # needed to build vendored OpenSSL
-              git-cliff
-            ];
+        devShells.default = let 
+          pkgs = import nixpkgs {
+            system = system;
+            overlays = [ fenix.overlays.default ];
           };
+          in pkgs.mkShell {
+
+          buildInputs = with pkgs; [
+            (fenix.packages.${system}.complete.withComponents [
+              "cargo"
+              "clippy"
+              "rust-src"
+              "rustc"
+              "rustfmt"
+            ])
+            cargo-deny
+            rust-analyzer-nightly
+            perl # needed to build vendored OpenSSL
+            git-cliff
+          ];
+        };
       }
     );
 }

format-flowed/src/lib.rs

@@ -8,8 +8,6 @@
 //! is assumed to be set to "no".
 //!
 //! For received messages, DelSp parameter is honoured.
-#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
-#![cfg_attr(not(test), forbid(clippy::string_slice))]
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///

node/constants.js

@@ -1,7 +1,6 @@
 // Generated!
 
 module.exports = {
-  DC_CERTCK_ACCEPT_INVALID: 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES: 3,
   DC_CERTCK_AUTO: 0,
   DC_CERTCK_STRICT: 1,
@@ -31,7 +30,6 @@ module.exports = {
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
-  DC_EVENT_CHANNEL_OVERFLOW: 2400,
   DC_EVENT_CHATLIST_CHANGED: 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED: 2301,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
@@ -51,7 +49,6 @@ module.exports = {
   DC_EVENT_IMEX_PROGRESS: 2051,
   DC_EVENT_INCOMING_MSG: 2005,
   DC_EVENT_INCOMING_MSG_BUNCH: 2006,
-  DC_EVENT_INCOMING_REACTION: 2002,
   DC_EVENT_INFO: 100,
   DC_EVENT_LOCATION_CHANGED: 2035,
   DC_EVENT_MSGS_CHANGED: 2000,
@@ -69,8 +66,6 @@ module.exports = {
   DC_EVENT_SMTP_MESSAGE_SENT: 103,
   DC_EVENT_WARNING: 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED: 2121,
-  DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT: 2151,
-  DC_EVENT_WEBXDC_REALTIME_DATA: 2150,
   DC_EVENT_WEBXDC_STATUS_UPDATE: 2120,
   DC_GCL_ADD_ALLDONE_HINT: 4,
   DC_GCL_ADD_SELF: 2,
@@ -131,13 +126,11 @@ module.exports = {
   DC_QR_ASK_VERIFYCONTACT: 200,
   DC_QR_ASK_VERIFYGROUP: 202,
   DC_QR_BACKUP: 251,
-  DC_QR_BACKUP2: 252,
   DC_QR_ERROR: 400,
   DC_QR_FPR_MISMATCH: 220,
   DC_QR_FPR_OK: 210,
   DC_QR_FPR_WITHOUT_ADDR: 230,
   DC_QR_LOGIN: 520,
-  DC_QR_PROXY: 271,
   DC_QR_REVIVE_VERIFYCONTACT: 510,
   DC_QR_REVIVE_VERIFYGROUP: 512,
   DC_QR_TEXT: 330,

node/events.js

@@ -16,7 +16,6 @@ module.exports = {
   410: 'DC_EVENT_ERROR_SELF_NOT_IN_GROUP',
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
-  2002: 'DC_EVENT_INCOMING_REACTION',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -38,10 +37,7 @@ module.exports = {
   2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
-  2150: 'DC_EVENT_WEBXDC_REALTIME_DATA',
-  2151: 'DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT',
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
-  2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
-  2400: 'DC_EVENT_CHANNEL_OVERFLOW'
+  2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED'
 }

node/lib/constants.ts

@@ -1,7 +1,6 @@
 // Generated!
 
 export enum C {
-  DC_CERTCK_ACCEPT_INVALID = 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3,
   DC_CERTCK_AUTO = 0,
   DC_CERTCK_STRICT = 1,
@@ -31,7 +30,6 @@ export enum C {
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
-  DC_EVENT_CHANNEL_OVERFLOW = 2400,
   DC_EVENT_CHATLIST_CHANGED = 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED = 2301,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
@@ -51,7 +49,6 @@ export enum C {
   DC_EVENT_IMEX_PROGRESS = 2051,
   DC_EVENT_INCOMING_MSG = 2005,
   DC_EVENT_INCOMING_MSG_BUNCH = 2006,
-  DC_EVENT_INCOMING_REACTION = 2002,
   DC_EVENT_INFO = 100,
   DC_EVENT_LOCATION_CHANGED = 2035,
   DC_EVENT_MSGS_CHANGED = 2000,
@@ -69,8 +66,6 @@ export enum C {
   DC_EVENT_SMTP_MESSAGE_SENT = 103,
   DC_EVENT_WARNING = 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED = 2121,
-  DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT = 2151,
-  DC_EVENT_WEBXDC_REALTIME_DATA = 2150,
   DC_EVENT_WEBXDC_STATUS_UPDATE = 2120,
   DC_GCL_ADD_ALLDONE_HINT = 4,
   DC_GCL_ADD_SELF = 2,
@@ -131,13 +126,11 @@ export enum C {
   DC_QR_ASK_VERIFYCONTACT = 200,
   DC_QR_ASK_VERIFYGROUP = 202,
   DC_QR_BACKUP = 251,
-  DC_QR_BACKUP2 = 252,
   DC_QR_ERROR = 400,
   DC_QR_FPR_MISMATCH = 220,
   DC_QR_FPR_OK = 210,
   DC_QR_FPR_WITHOUT_ADDR = 230,
   DC_QR_LOGIN = 520,
-  DC_QR_PROXY = 271,
   DC_QR_REVIVE_VERIFYCONTACT = 510,
   DC_QR_REVIVE_VERIFYGROUP = 512,
   DC_QR_TEXT = 330,
@@ -324,7 +317,6 @@ export const EventId2EventName: { [key: number]: string } = {
   410: 'DC_EVENT_ERROR_SELF_NOT_IN_GROUP',
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
-  2002: 'DC_EVENT_INCOMING_REACTION',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -346,10 +338,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
-  2150: 'DC_EVENT_WEBXDC_REALTIME_DATA',
-  2151: 'DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT',
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
   2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
-  2400: 'DC_EVENT_CHANNEL_OVERFLOW',
 }

node/lib/context.ts

@@ -475,6 +475,47 @@ export class Context extends EventEmitter {
     return binding.dcn_get_msg_html(this.dcn_context, Number(messageId))
   }
 
+  getNextMediaMessage(
+    messageId: number,
+    msgType1: number,
+    msgType2: number,
+    msgType3: number
+  ) {
+    debug(
+      `getNextMediaMessage ${messageId} ${msgType1} ${msgType2} ${msgType3}`
+    )
+    return this._getNextMedia(messageId, 1, msgType1, msgType2, msgType3)
+  }
+
+  getPreviousMediaMessage(
+    messageId: number,
+    msgType1: number,
+    msgType2: number,
+    msgType3: number
+  ) {
+    debug(
+      `getPreviousMediaMessage ${messageId} ${msgType1} ${msgType2} ${msgType3}`
+    )
+    return this._getNextMedia(messageId, -1, msgType1, msgType2, msgType3)
+  }
+
+  _getNextMedia(
+    messageId: number,
+    dir: number,
+    msgType1: number,
+    msgType2: number,
+    msgType3: number
+  ): number {
+    return binding.dcn_get_next_media(
+      this.dcn_context,
+      Number(messageId),
+      dir,
+      msgType1 || 0,
+      msgType2 || 0,
+      msgType3 || 0
+    )
+  }
+
   getSecurejoinQrCode(chatId: number): string {
     debug(`getSecurejoinQrCode ${chatId}`)
     return binding.dcn_get_securejoin_qr(this.dcn_context, Number(chatId))

node/src/module.c

@@ -1053,6 +1053,27 @@ NAPI_METHOD(dcn_get_msg_html) {
   NAPI_RETURN_AND_UNREF_STRING(msg_html);
 }
 
+NAPI_METHOD(dcn_get_next_media) {
+  NAPI_ARGV(6);
+  NAPI_DCN_CONTEXT();
+  NAPI_ARGV_UINT32(msg_id, 1);
+  NAPI_ARGV_INT32(dir, 2);
+  NAPI_ARGV_INT32(msg_type1, 3);
+  NAPI_ARGV_INT32(msg_type2, 4);
+  NAPI_ARGV_INT32(msg_type3, 5);
+
+  //TRACE("calling..");
+  uint32_t next_id = dc_get_next_media(dcn_context->dc_context,
+                                       msg_id,
+                                       dir,
+                                       msg_type1,
+                                       msg_type2,
+                                       msg_type3);
+  //TRACE("result %d", next_id);
+
+  NAPI_RETURN_UINT32(next_id);
+}
+
 NAPI_METHOD(dcn_set_chat_visibility) {
   NAPI_ARGV(3);
   NAPI_DCN_CONTEXT();
@@ -3422,6 +3443,7 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_get_msg_cnt);
   NAPI_EXPORT_FUNCTION(dcn_get_msg_info);
   NAPI_EXPORT_FUNCTION(dcn_get_msg_html);
+  NAPI_EXPORT_FUNCTION(dcn_get_next_media);
   NAPI_EXPORT_FUNCTION(dcn_set_chat_visibility);
   NAPI_EXPORT_FUNCTION(dcn_get_securejoin_qr);
   NAPI_EXPORT_FUNCTION(dcn_get_securejoin_qr_svg);

node/test/test.mjs

@@ -271,7 +271,7 @@ describe('Basic offline Tests', function () {
       'sync_msgs',
       'sentbox_watch',
       'show_emails',
-      'proxy_enabled',
+      'socks5_enabled',
       'sqlite_version',
       'uptime',
       'used_account_settings',

package.json

@@ -11,7 +11,7 @@
     "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
     "mocha": "^8.2.1",
-    "node-gyp": "~10.1.0",
+    "node-gyp": "^10.0.0",
     "prebuildify": "^5.0.1",
     "prebuildify-ci": "^1.0.5",
     "prettier": "^3.0.3",
@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.149.0"
+  "version": "1.138.5"
 }

python/examples/test_examples.py

@@ -44,6 +44,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
 
+    botproc.fnmatch_lines(
+        """
+        *ac_configure_completed*
+    """,
+    )
     ac1.add_account_plugin(FFIEventLogger(ac1))
     ac2.add_account_plugin(FFIEventLogger(ac2))
 

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.149.0"
+version = "1.138.5"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.7"

python/src/deltachat/account.py

@@ -194,13 +194,15 @@ class Account:
         assert res != ffi.NULL, f"config value not found for: {name!r}"
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, secret: str) -> None:
+    def _preconfigure_keypair(self, addr: str, secret: str) -> None:
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
         """
         res = lib.dc_preconfigure_keypair(
             self._dc_context,
+            as_dc_charpointer(addr),
+            ffi.NULL,
             as_dc_charpointer(secret),
         )
         if res == 0:

python/src/deltachat/chat.py

@@ -308,7 +308,7 @@ class Chat:
         msg = as_dc_charpointer(text)
         msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
         if msg_id == 0:
-            raise ValueError("The message could not be sent. Does the chat exist?")
+            raise ValueError("message could not be send, does chat exist?")
         return Message.from_db(self.account, msg_id)
 
     def send_file(self, path, mime_type="application/octet-stream"):

python/src/deltachat/direct_imap.py

@@ -8,19 +8,19 @@ import io
 import pathlib
 import ssl
 from contextlib import contextmanager
-from typing import List, TYPE_CHECKING
+from typing import List
 
 from imap_tools import (
     AND,
     Header,
     MailBox,
+    MailBoxTls,
     MailMessage,
     MailMessageFlags,
     errors,
 )
 
-if TYPE_CHECKING:
-    from deltachat import Account
+from deltachat import Account, const
 
 FLAGS = b"FLAGS"
 FETCH = b"FETCH"
@@ -28,7 +28,7 @@ ALL = "1:*"
 
 
 class DirectImap:
-    def __init__(self, account: "Account") -> None:
+    def __init__(self, account: Account) -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False
@@ -36,13 +36,27 @@ class DirectImap:
 
     def connect(self):
         host = self.account.get_config("configured_mail_server")
-        port = 993
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
 
         user = self.account.get_config("addr")
-        host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
+        if security == const.DC_SOCKET_PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.DC_SOCKET_STARTTLS:
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
+        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

python/src/deltachat/testplugin.py

@@ -462,7 +462,7 @@ class ACFactory:
     def remove_preconfigured_keys(self) -> None:
         self._preconfigured_keys = []
 
-    def _preconfigure_key(self, account):
+    def _preconfigure_key(self, account, addr):
         # Only set a preconfigured key if we haven't used it yet for another account.
         try:
             keyname = self._preconfigured_keys.pop(0)
@@ -471,9 +471,9 @@ class ACFactory:
         else:
             fname_sec = self.data.read_path(f"key/{keyname}-secret.asc")
             if fname_sec:
-                account._preconfigure_keypair(fname_sec)
+                account._preconfigure_keypair(addr, fname_sec)
                 return True
-            print("WARN: could not use preconfigured keys")
+            print(f"WARN: could not use preconfigured keys for {addr!r}")
 
     def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account:
         # do a pseudo-configured account
@@ -492,7 +492,7 @@ class ACFactory:
                 "configured": "1",
             },
         )
-        self._preconfigure_key(ac)
+        self._preconfigure_key(ac, addr)
         self._acsetup.init_logging(ac)
         return ac
 
@@ -525,10 +525,9 @@ class ACFactory:
         configdict.setdefault("mvbox_move", False)
         configdict.setdefault("sentbox_watch", False)
         configdict.setdefault("sync_msgs", False)
-        configdict.setdefault("delete_server_after", 0)
         ac.update_config(configdict)
         self._acsetup._account2config[ac] = configdict
-        self._preconfigure_key(ac)
+        self._preconfigure_key(ac, configdict["addr"])
         return ac
 
     def wait_configured(self, account) -> None:
@@ -553,15 +552,6 @@ class ACFactory:
 
         bot_cfg = self.get_next_liveconfig()
         bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
-        self._acsetup.start_configure(bot_ac)
-        self.wait_configured(bot_ac)
-        bot_ac.start_io()
-        # Wait for DC_EVENT_IMAP_INBOX_IDLE so that all emails appeared in the bot's Inbox later are
-        # considered new and not existing ones, and thus processed by the bot.
-        print(bot_ac._logid, "waiting for inbox IDLE to become ready")
-        bot_ac._evtracker.wait_idle_inbox_ready()
-        bot_ac.stop_io()
-        self._acsetup._account2state[bot_ac] = self._acsetup.IDLEREADY
 
         # Forget ac as it will be opened by the bot subprocess
         # but keep something in the list to not confuse account generation

python/tests/test_0_complex_or_slow.py

@@ -1,5 +1,4 @@
 import sys
-import time
 
 import pytest
 import deltachat as dc
@@ -676,17 +675,3 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     assert msg_in.chat == chat2_offl
     assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
     assert ac2_offl_ac1_contact.is_verified()
-
-
-def test_deleted_msgs_dont_reappear(acfactory):
-    ac1 = acfactory.new_online_configuring_account()
-    acfactory.bring_accounts_online()
-    ac1.set_config("bcc_self", "1")
-    chat = ac1.get_self_contact().create_chat()
-    msg = chat.send_text("hello")
-    ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
-    ac1.delete_messages([msg])
-    ac1._evtracker.get_matching("DC_EVENT_MSG_DELETED")
-    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
-    time.sleep(5)
-    assert len(chat.get_messages()) == 0

python/tests/test_1_online.py

@@ -484,24 +484,6 @@ def test_move_works_on_self_sent(acfactory):
     ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
 
-def test_move_sync_msgs(acfactory):
-    ac1 = acfactory.new_online_configuring_account(bcc_self=True, sync_msgs=True, fix_is_chatmail=True)
-    acfactory.bring_accounts_online()
-
-    ac1.direct_imap.select_folder("DeltaChat")
-    # Sync messages may also be sent during the configuration.
-    mvbox_msg_cnt = len(ac1.direct_imap.get_all_messages())
-
-    ac1.set_config("displayname", "Alice")
-    ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-    ac1.set_config("displayname", "Bob")
-    ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-    ac1.direct_imap.select_folder("Inbox")
-    assert len(ac1.direct_imap.get_all_messages()) == 0
-    ac1.direct_imap.select_folder("DeltaChat")
-    assert len(ac1.direct_imap.get_all_messages()) == mvbox_msg_cnt + 2
-
-
 def test_forward_messages(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     chat = ac1.create_chat(ac2)
@@ -628,7 +610,7 @@ def test_long_group_name(acfactory, lp):
 
 
 def test_send_self_message(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=True, bcc_self=True)
+    ac1 = acfactory.new_online_configuring_account(mvbox_move=True)
     acfactory.bring_accounts_online()
     lp.sec("ac1: create self chat")
     chat = ac1.get_self_contact().create_chat()
@@ -1580,6 +1562,8 @@ def test_import_export_online_all(acfactory, tmp_path, data, lp):
 
         # check progress events for import
         assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+        assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+        assert imex_tracker.wait_progress(750, progress_upper_limit=999)
         assert imex_tracker.wait_progress(1000)
 
     assert_account_is_proper(ac1)
@@ -2084,11 +2068,12 @@ def test_send_receive_locations(acfactory, lp):
 def test_immediate_autodelete(acfactory, lp):
     ac1 = acfactory.new_online_configuring_account()
     ac2 = acfactory.new_online_configuring_account()
-    acfactory.bring_accounts_online()
 
     # "1" means delete immediately, while "0" means do not delete
     ac2.set_config("delete_server_after", "1")
 
+    acfactory.bring_accounts_online()
+
     lp.sec("ac1: create chat with ac2")
     chat1 = ac1.create_chat(ac2)
     ac2.create_chat(ac1)
@@ -2209,19 +2194,6 @@ def test_configure_error_msgs_wrong_pw(acfactory):
     # Password is wrong so it definitely has to say something about "password"
     assert "password" in ev.data2
 
-    ac1.stop_io()
-    ac1.set_config("mail_pw", "abc")  # Wrong mail pw
-    ac1.configure()
-    while True:
-        ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        print(f"Configuration progress: {ev.data1}")
-        if ev.data1 == 0:
-            break
-    assert "password" in ev.data2
-    # Account will continue to work with the old password, so if it becomes wrong, a notification
-    # must be shown.
-    assert ac1.get_config("notify_about_wrong_pw") == "1"
-
 
 def test_configure_error_msgs_invalid_server(acfactory):
     ac2 = acfactory.get_unconfigured_account()

python/tests/test_3_offline.py

@@ -67,7 +67,7 @@ class TestOfflineAccountBasic:
         ac = acfactory.get_unconfigured_account()
         alice_secret = data.read_path("key/alice-secret.asc")
         assert alice_secret
-        ac._preconfigure_keypair(alice_secret)
+        ac._preconfigure_keypair("alice@example.org", alice_secret)
 
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -705,7 +705,7 @@ class TestOfflineChat:
         ac1 = acfactory.get_pseudo_configured_account()
         ac2 = acfactory.get_pseudo_configured_account()
         qr = ac1.get_setup_contact_qr()
-        assert qr.startswith("https://i.delta.chat")
+        assert qr.startswith("OPENPGP4FPR:")
         res = ac2.check_qr(qr)
         assert res.is_ask_verifycontact()
         assert not res.is_ask_verifygroup()

release-date.in

@@ -1 +1 @@
-2024-11-05
+2024-05-16

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.82.0
+RUST_VERSION=1.78.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/create-provider-data-rs.py

@@ -149,7 +149,7 @@ def process_data(data, file):
     oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
 
     provider = ""
-    before_login_hint = cleanstr(data.get("before_login_hint", "") or "")
+    before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
         provider += (

scripts/make-rpc-testenv.sh

@@ -3,4 +3,4 @@ set -euo pipefail
 
 tox -c deltachat-rpc-client -e py --devenv venv
 venv/bin/pip install --upgrade pip
-cargo install --locked --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug

scripts/run-rpc-test.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
-cargo install --locked --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
 PATH="$PWD/venv/bin:$PATH" tox -c deltachat-rpc-client

scripts/run_all.sh

@@ -31,6 +31,6 @@ unset CHATMAIL_DOMAIN
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,py313,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=77cbf92a8565fdf1bcaba10fa93c1455c750a1e9
+REV=2f3db24107e4802c2df0aa0a40f0e144006c0a9b
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.35.0
+Version: 0.34.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -22,13 +22,7 @@ to implement typical messenger functions.
 - [Locations](#locations)
     - [User locations](#user-locations)
     - [Points of interest](#points-of-interest)
-- [Stickers](#stickers)
-- [Voice messages](#voice-messages)
-- [Reactions](#reactions)
-- [Attaching a contact to a message](#attaching-a-contact-to-a-message)
-- [Transitioning to a new e-mail address (AEAP)](#transitioning-to-a-new-e-mail-address-aeap)
 - [Miscellaneous](#miscellaneous)
-    - [Sync messages](#sync-messages)
 
 
 # Encryption
@@ -467,58 +461,6 @@ As an extension to RFC 9078, it is allowed to send empty reaction message,
 in which case all previously sent reactions are retracted.
 
 
-# Attaching a contact to a message
-
-Messengers MAY allow the user to attach a contact to a message
-in order to share it with the chat partner.
-The contact MUST be sent as a [vCard](https://datatracker.ietf.org/doc/html/rfc6350).
-
-The vCard MUST contain `EMAIL`,
-`FN` (display name),
-and `VERSION` (which version of the vCard standard you're using).
-If available, it SHOULD contain
-`REV` (current timestamp),
-`PHOTO` (avatar), and
-`KEY` (OpenPGP public key,
-in binary format,
-encoded with vanilla base64;
-note that this is different from the OpenPGP 'ASCII Armor' format).
-
-Example vCard:
-
-```
-BEGIN:VCARD
-VERSION:4.0
-EMAIL:alice@example.org
-FN:Alice Wonderland
-KEY:data:application/pgp-keys;base64,[Base64-data]
-PHOTO:data:image/jpeg;base64,[image in Base64]
-REV:20240418T184242Z
-END:VCARD
-```
-
-It is fine if messengers do include a full vCard parser
-and e.g. simply search for the line starting with `EMAIL`
-in order to get the email address.
-
-
-# Transitioning to a new e-mail address (AEAP)
-
-When receiving a message:
-- If the key exists, but belongs to another address
-- AND there is a `Chat-Version` header
-- AND the message is signed correctly
-- AND the From address is (also) in the encrypted (and therefore signed) headers 
-- AND the message timestamp is newer than the contact's `lastseen`
-  (to prevent changing the address back when messages arrive out of order)
-  (this condition is not that important
-  since we will have eventual consistency even without it):
-
-  Replace the contact in _all_ groups,
-  possibly deduplicate the members list,
-  and add a system message to all of these chats.
-
-
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -542,4 +484,21 @@ We define the effective date of a message
 as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
+
+# Transitioning to a new e-mail address (AEAP)
+
+When receiving a message:
+- If the key exists, but belongs to another address
+- AND there is a `Chat-Version` header
+- AND the message is signed correctly
+- AND the From address is (also) in the encrypted (and therefore signed) headers 
+- AND the message timestamp is newer than the contact's `lastseen`
+  (to prevent changing the address back when messages arrive out of order)
+  (this condition is not that important
+  since we will have eventual consistency even without it):
+
+  Replace the contact in _all_ groups,
+  possibly deduplicate the members list,
+  and add a system message to all of these chats.
+
 Copyright © 2017-2021 Delta Chat contributors.

src/accounts.rs

@@ -5,8 +5,7 @@ use std::future::Future;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
-use futures::stream::FuturesUnordered;
-use futures::StreamExt;
+use futures::future::join_all;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
@@ -167,19 +166,6 @@ impl Accounts {
             .remove(&id)
             .with_context(|| format!("no account with id {id}"))?;
         ctx.stop_io().await;
-
-        // Explicitly close the database
-        // to make sure the database file is closed
-        // and can be removed on Windows.
-        // If some spawned task tries to use the database afterwards,
-        // it will fail.
-        //
-        // Previously `stop_io()` aborted the tasks without awaiting them
-        // and this resulted in keeping `Context` clones inside
-        // `Future`s that were not dropped. This bug is fixed now,
-        // but explicitly closing the database ensures that file is freed
-        // even if not all `Context` references are dropped.
-        ctx.sql.close().await;
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id) {
@@ -302,48 +288,20 @@ impl Accounts {
     ///
     /// This is an auxiliary function and not part of public API.
     /// Use [Accounts::background_fetch] instead.
-    async fn background_fetch_no_timeout(accounts: Vec<Context>, events: Events) {
+    async fn background_fetch_without_timeout(&self) {
         async fn background_fetch_and_log_error(account: Context) {
             if let Err(error) = account.background_fetch().await {
                 warn!(account, "{error:#}");
             }
         }
 
-        events.emit(Event {
-            id: 0,
-            typ: EventType::Info(format!(
-                "Starting background fetch for {} accounts.",
-                accounts.len()
-            )),
-        });
-        let mut futures_unordered: FuturesUnordered<_> = accounts
-            .into_iter()
-            .map(background_fetch_and_log_error)
-            .collect();
-        while futures_unordered.next().await.is_some() {}
-    }
-
-    /// Auxiliary function for [Accounts::background_fetch].
-    async fn background_fetch_with_timeout(
-        accounts: Vec<Context>,
-        events: Events,
-        timeout: std::time::Duration,
-    ) {
-        if let Err(_err) = tokio::time::timeout(
-            timeout,
-            Self::background_fetch_no_timeout(accounts, events.clone()),
+        join_all(
+            self.accounts
+                .values()
+                .cloned()
+                .map(background_fetch_and_log_error),
         )
-        .await
-        {
-            events.emit(Event {
-                id: 0,
-                typ: EventType::Warning("Background fetch timed out.".to_string()),
-            });
-        }
-        events.emit(Event {
-            id: 0,
-            typ: EventType::AccountsBackgroundFetchDone,
-        });
+        .await;
     }
 
     /// Performs a background fetch for all accounts in parallel with a timeout.
@@ -351,13 +309,15 @@ impl Accounts {
     /// The `AccountsBackgroundFetchDone` event is emitted at the end,
     /// process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
-    ///
-    /// Returns a future that resolves when background fetch is done,
-    /// but does not capture `&self`.
-    pub fn background_fetch(&self, timeout: std::time::Duration) -> impl Future<Output = ()> {
-        let accounts: Vec<Context> = self.accounts.values().cloned().collect();
-        let events = self.events.clone();
-        Self::background_fetch_with_timeout(accounts, events, timeout)
+    pub async fn background_fetch(&self, timeout: std::time::Duration) {
+        if let Err(_err) =
+            tokio::time::timeout(timeout, self.background_fetch_without_timeout()).await
+        {
+            self.emit_event(EventType::Warning(
+                "Background fetch timed out.".to_string(),
+            ));
+        }
+        self.emit_event(EventType::AccountsBackgroundFetchDone);
     }
 
     /// Emits a single event.
@@ -371,7 +331,7 @@ impl Accounts {
     }
 
     /// Sets notification token for Apple Push Notification service.
-    pub async fn set_push_device_token(&self, token: &str) -> Result<()> {
+    pub async fn set_push_device_token(&mut self, token: &str) -> Result<()> {
         self.push_subscriber.set_device_token(token).await;
         Ok(())
     }
@@ -525,6 +485,10 @@ impl Config {
 
     /// Read a configuration from the given file into memory.
     pub async fn from_file(file: PathBuf, writable: bool) -> Result<Self> {
+        let dir = file
+            .parent()
+            .context("Cannot get config file directory")?
+            .to_path_buf();
         let mut config = Self::new_nosync(file, writable).await?;
         let bytes = fs::read(&config.file)
             .await
@@ -536,13 +500,9 @@ impl Config {
         // Convert them to relative paths.
         let mut modified = false;
         for account in &mut config.inner.accounts {
-            if account.dir.is_absolute() {
-                if let Some(old_path_parent) = account.dir.parent() {
-                    if let Ok(new_path) = account.dir.strip_prefix(old_path_parent) {
-                        account.dir = new_path.to_path_buf();
-                        modified = true;
-                    }
-                }
+            if let Ok(new_dir) = account.dir.strip_prefix(&dir) {
+                account.dir = new_dir.to_path_buf();
+                modified = true;
             }
         }
         if modified && writable {

src/authres.rs

@@ -260,6 +260,7 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
     use tokio::fs;
     use tokio::io::AsyncReadExt;
 
@@ -519,13 +520,8 @@ Authentication-Results: dkim=";
         handle_authres(&t, &mail, "invalid@rom.com").await.unwrap();
     }
 
-    // Test that Autocrypt works with mailing list.
-    //
-    // Previous versions of Delta Chat ignored Autocrypt based on the List-Post header.
-    // This is not needed: comparing of the From address to Autocrypt header address is enough.
-    // If the mailing list is not rewriting the From header, Autocrypt should be applied.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_autocrypt_in_mailinglist_not_ignored() -> Result<()> {
+    async fn test_autocrypt_in_mailinglist_ignored() -> Result<()> {
         let mut tcm = TestContextManager::new();
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
@@ -537,18 +533,28 @@ Authentication-Results: dkim=";
             .insert_str(0, "List-Post: <mailto:deltachat-community.example.net>\n");
         bob.recv_msg(&sent).await;
         let peerstate = Peerstate::from_addr(&bob, "alice@example.org").await?;
+        assert!(peerstate.is_none());
+
+        // Do the same without the mailing list header, this time the peerstate should be accepted
+        let sent = alice
+            .send_text(alice_bob_chat.id, "hellooo without mailing list")
+            .await;
+        bob.recv_msg(&sent).await;
+        let peerstate = Peerstate::from_addr(&bob, "alice@example.org").await?;
         assert!(peerstate.is_some());
 
-        // Bob can now write encrypted to Alice:
+        // This also means that Bob can now write encrypted to Alice:
         let mut sent = bob
             .send_text(bob_alice_chat.id, "hellooo in the mailinglist again")
             .await;
         assert!(sent.load_from_db().await.get_showpadlock());
 
+        // But if Bob writes to a mailing list, Alice doesn't show a padlock
+        // since she can't verify the signature without accepting Bob's key:
         sent.payload
             .insert_str(0, "List-Post: <mailto:deltachat-community.example.net>\n");
         let rcvd = alice.recv_msg(&sent).await;
-        assert!(rcvd.get_showpadlock());
+        assert!(!rcvd.get_showpadlock());
         assert_eq!(&rcvd.text, "hellooo in the mailinglist again");
 
         Ok(())

src/blob.rs

@@ -3,7 +3,7 @@
 use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
-use std::io::{Cursor, Seek};
+use std::io::Cursor;
 use std::iter::FusedIterator;
 use std::mem;
 use std::path::{Path, PathBuf};
@@ -12,7 +12,6 @@ use anyhow::{format_err, Context as _, Result};
 use base64::Engine as _;
 use futures::StreamExt;
 use image::codecs::jpeg::JpegEncoder;
-use image::ImageReader;
 use image::{DynamicImage, GenericImage, GenericImageView, ImageFormat, Pixel, Rgba};
 use num_traits::FromPrimitive;
 use tokio::io::AsyncWriteExt;
@@ -253,16 +252,16 @@ impl<'a> BlobObject<'a> {
     ///
     /// The extension part will always be lowercased.
     fn sanitise_name(name: &str) -> (String, String) {
-        let mut name = name;
+        let mut name = name.to_string();
         for part in name.rsplit('/') {
             if !part.is_empty() {
-                name = part;
+                name = part.to_string();
                 break;
             }
         }
         for part in name.rsplit('\\') {
             if !part.is_empty() {
-                name = part;
+                name = part.to_string();
                 break;
             }
         }
@@ -272,39 +271,32 @@ impl<'a> BlobObject<'a> {
             replacement: "",
         };
 
-        let name = sanitize_filename::sanitize_with_options(name, opts);
-        // Let's take a tricky filename,
+        let clean = sanitize_filename::sanitize_with_options(name, opts);
+        // Let's take the tricky filename
         // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
-        // Assume that the extension is 32 chars maximum.
-        let ext: String = name
-            .chars()
+        // Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
+        let mut iter = clean.splitn(2, '.');
+
+        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
+        // stem == "file"
+
+        let ext_chars = iter.next().unwrap_or_default().chars();
+        let ext: String = ext_chars
             .rev()
-            .take_while(|c| !c.is_whitespace())
-            .take(33)
+            .take(32)
             .collect::<Vec<_>>()
             .iter()
             .rev()
             .collect();
-        // ext == "nd_point_and_double_ending.tar.gz"
+        // ext == "d_point_and_double_ending.tar.gz"
 
-        // Split it into "nd_point_and_double_ending" and "tar.gz":
-        let mut iter = ext.splitn(2, '.');
-        iter.next();
-
-        let ext = iter.next().unwrap_or_default();
-        let ext = if ext.is_empty() {
-            String::new()
+        if ext.is_empty() {
+            (stem, "".to_string())
         } else {
-            format!(".{ext}")
-            // ".tar.gz"
-        };
-        let stem = name
-            .strip_suffix(&ext)
-            .unwrap_or_default()
-            .chars()
-            .take(64)
-            .collect();
-        (stem, ext.to_lowercase())
+            (stem, format!(".{ext}").to_lowercase())
+            // Return ("file", ".d_point_and_double_ending.tar.gz")
+            // which is not perfect but acceptable.
+        }
     }
 
     /// Checks whether a name is a valid blob name.
@@ -434,25 +426,9 @@ impl<'a> BlobObject<'a> {
         let mut no_exif = false;
         let no_exif_ref = &mut no_exif;
         let res = tokio::task::block_in_place(move || {
-            let mut file = std::fs::File::open(self.to_abs_path())?;
-            let (nr_bytes, exif) = image_metadata(&file)?;
+            let (nr_bytes, exif) = self.metadata()?;
             *no_exif_ref = exif.is_none();
-            // It's strange that BufReader modifies a file position while it takes a non-mut
-            // reference. Ok, just rewind it.
-            file.rewind()?;
-            let imgreader = ImageReader::new(std::io::BufReader::new(&file)).with_guessed_format();
-            let imgreader = match imgreader {
-                Ok(ir) => ir,
-                _ => {
-                    file.rewind()?;
-                    ImageReader::with_format(
-                        std::io::BufReader::new(&file),
-                        ImageFormat::from_path(&blob_abs)?,
-                    )
-                }
-            };
-            let fmt = imgreader.format().context("No format??")?;
-            let mut img = imgreader.decode().context("image decode failure")?;
+            let mut img = image::open(&blob_abs).context("image decode failure")?;
             let orientation = exif.as_ref().map(|exif| exif_orientation(exif, context));
             let mut encoded = Vec::new();
             let mut changed_name = None;
@@ -481,9 +457,10 @@ impl<'a> BlobObject<'a> {
             let exceeds_max_bytes = nr_bytes > max_bytes as u64;
 
             let jpeg_quality = 75;
+            let fmt = ImageFormat::from_path(&blob_abs);
             let ofmt = match fmt {
-                ImageFormat::Png if !exceeds_max_bytes => ImageOutputFormat::Png,
-                ImageFormat::Jpeg => {
+                Ok(ImageFormat::Png) if !exceeds_max_bytes => ImageOutputFormat::Png,
+                Ok(ImageFormat::Jpeg) => {
                     add_white_bg = false;
                     ImageOutputFormat::Jpeg {
                         quality: jpeg_quality,
@@ -520,7 +497,7 @@ impl<'a> BlobObject<'a> {
                     img_wh = max(img.width(), img.height());
                     // PNGs and WebPs may be huge because of animation, which is lost by the `image`
                     // crate when recoding, so don't scale them down.
-                    if matches!(fmt, ImageFormat::Jpeg) || !encoded.is_empty() {
+                    if matches!(fmt, Ok(ImageFormat::Jpeg)) || !encoded.is_empty() {
                         img_wh = img_wh * 2 / 3;
                     }
                 }
@@ -561,7 +538,7 @@ impl<'a> BlobObject<'a> {
 
             if do_scale || exif.is_some() {
                 // The file format is JPEG/PNG now, we may have to change the file extension
-                if !matches!(fmt, ImageFormat::Jpeg)
+                if !matches!(fmt, Ok(ImageFormat::Jpeg))
                     && matches!(ofmt, ImageOutputFormat::Jpeg { .. })
                 {
                     blob_abs = blob_abs.with_extension("jpg");
@@ -598,14 +575,15 @@ impl<'a> BlobObject<'a> {
             }
         }
     }
-}
 
-/// Returns image file size and Exif.
-pub fn image_metadata(file: &std::fs::File) -> Result<(u64, Option<exif::Exif>)> {
-    let len = file.metadata()?.len();
-    let mut bufreader = std::io::BufReader::new(file);
-    let exif = exif::Reader::new().read_from_container(&mut bufreader).ok();
-    Ok((len, exif))
+    /// Returns image file size and Exif.
+    pub fn metadata(&self) -> Result<(u64, Option<exif::Exif>)> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let len = file.metadata()?.len();
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exif = exif::Reader::new().read_from_container(&mut bufreader).ok();
+        Ok((len, exif))
+    }
 }
 
 fn exif_orientation(exif: &exif::Exif, context: &Context) -> i32 {
@@ -622,7 +600,7 @@ fn exif_orientation(exif: &exif::Exif, context: &Context) -> i32 {
     0
 }
 
-impl fmt::Display for BlobObject<'_> {
+impl<'a> fmt::Display for BlobObject<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "$BLOBDIR/{}", self.name)
     }
@@ -673,6 +651,10 @@ impl<'a> BlobDirContents<'a> {
     pub(crate) fn iter(&self) -> BlobDirIter<'_> {
         BlobDirIter::new(self.context, self.inner.iter())
     }
+
+    pub(crate) fn len(&self) -> usize {
+        self.inner.len()
+    }
 }
 
 /// A iterator over all the [`BlobObject`]s in the blobdir.
@@ -970,19 +952,6 @@ mod tests {
         assert!(!stem.contains(':'));
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
-
-        let (stem, ext) = BlobObject::sanitise_name(
-            "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz",
-        );
-        assert_eq!(
-            stem,
-            "file.with_lots_of_characters_behind_point_and_double_ending"
-        );
-        assert_eq!(ext, ".tar.gz");
-
-        let (stem, ext) = BlobObject::sanitise_name("a. tar.tar.gz");
-        assert_eq!(stem, "a. tar");
-        assert_eq!(ext, ".tar.gz");
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1117,34 +1086,32 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "0",
+        send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("0"),
             bytes,
-            extension: "jpg",
-            has_exif: true,
-            original_width: 1000,
-            original_height: 1000,
-            compressed_width: 1000,
-            compressed_height: 1000,
-            ..Default::default()
-        }
-        .test()
+            "jpg",
+            true, // has Exif
+            1000,
+            1000,
+            0,
+            1000,
+            1000,
+        )
         .await
         .unwrap();
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "1",
+        send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("1"),
             bytes,
-            extension: "jpg",
-            has_exif: true,
-            original_width: 1000,
-            original_height: 1000,
-            compressed_width: 1000,
-            compressed_height: 1000,
-            ..Default::default()
-        }
-        .test()
+            "jpg",
+            true, // has Exif
+            1000,
+            1000,
+            0,
+            1000,
+            1000,
+        )
         .await
         .unwrap();
     }
@@ -1153,20 +1120,18 @@ mod tests {
     async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
-        let img_rotated = SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "0",
+        let img_rotated = send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("0"),
             bytes,
-            extension: "jpg",
-            has_exif: true,
-            original_width: 2000,
-            original_height: 1800,
-            orientation: 270,
-            compressed_width: 1800,
-            compressed_height: 2000,
-            ..Default::default()
-        }
-        .test()
+            "jpg",
+            true, // has Exif
+            2000,
+            1800,
+            270,
+            1800,
+            2000,
+        )
         .await
         .unwrap();
         assert_correct_rotation(&img_rotated);
@@ -1175,18 +1140,18 @@ mod tests {
         img_rotated.write_to(&mut buf, ImageFormat::Jpeg).unwrap();
         let bytes = buf.into_inner();
 
-        let img_rotated = SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "1",
-            bytes: &bytes,
-            extension: "jpg",
-            original_width: 1800,
-            original_height: 2000,
-            compressed_width: 1800,
-            compressed_height: 2000,
-            ..Default::default()
-        }
-        .test()
+        let img_rotated = send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("1"),
+            &bytes,
+            "jpg",
+            false, // no Exif
+            1800,
+            2000,
+            0,
+            1800,
+            2000,
+        )
         .await
         .unwrap();
         assert_correct_rotation(&img_rotated);
@@ -1196,80 +1161,49 @@ mod tests {
     async fn test_recode_image_balanced_png() {
         let bytes = include_bytes!("../test-data/image/screenshot.png");
 
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "0",
+        send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("0"),
             bytes,
-            extension: "png",
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: 1920,
-            compressed_height: 1080,
-            ..Default::default()
-        }
-        .test()
+            "png",
+            false, // no Exif
+            1920,
+            1080,
+            0,
+            1920,
+            1080,
+        )
         .await
         .unwrap();
 
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "1",
+        send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("1"),
             bytes,
-            extension: "png",
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: constants::WORSE_IMAGE_SIZE,
-            compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-            ..Default::default()
-        }
-        .test()
-        .await
-        .unwrap();
-
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::File,
-            media_quality_config: "1",
-            bytes,
-            extension: "png",
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: 1920,
-            compressed_height: 1080,
-            ..Default::default()
-        }
-        .test()
-        .await
-        .unwrap();
-
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::File,
-            media_quality_config: "1",
-            bytes,
-            extension: "png",
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: 1920,
-            compressed_height: 1080,
-            set_draft: true,
-            ..Default::default()
-        }
-        .test()
+            "png",
+            false, // no Exif
+            1920,
+            1080,
+            0,
+            constants::WORSE_IMAGE_SIZE,
+            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+        )
         .await
         .unwrap();
 
         // This will be sent as Image, see [`BlobObject::maybe_sticker`] for explanation.
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Sticker,
-            media_quality_config: "0",
+        send_image_check_mediaquality(
+            Viewtype::Sticker,
+            Some("0"),
             bytes,
-            extension: "png",
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: 1920,
-            compressed_height: 1080,
-            ..Default::default()
-        }
-        .test()
+            "png",
+            false, // no Exif
+            1920,
+            1080,
+            0,
+            1920,
+            1080,
+        )
         .await
         .unwrap();
     }
@@ -1280,18 +1214,18 @@ mod tests {
     async fn test_recode_image_rgba_png_to_jpeg() {
         let bytes = include_bytes!("../test-data/image/screenshot-rgba.png");
 
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "1",
+        send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("1"),
             bytes,
-            extension: "png",
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: constants::WORSE_IMAGE_SIZE,
-            compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-            ..Default::default()
-        }
-        .test()
+            "png",
+            false, // no Exif
+            1920,
+            1080,
+            0,
+            constants::WORSE_IMAGE_SIZE,
+            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+        )
         .await
         .unwrap();
     }
@@ -1299,19 +1233,18 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_huge_jpg() {
         let bytes = include_bytes!("../test-data/image/screenshot.jpg");
-        SendImageCheckMediaquality {
-            viewtype: Viewtype::Image,
-            media_quality_config: "0",
+        send_image_check_mediaquality(
+            Viewtype::Image,
+            Some("0"),
             bytes,
-            extension: "jpg",
-            has_exif: true,
-            original_width: 1920,
-            original_height: 1080,
-            compressed_width: constants::BALANCED_IMAGE_SIZE,
-            compressed_height: constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
-            ..Default::default()
-        }
-        .test()
+            "jpg",
+            true, // has Exif
+            1920,
+            1080,
+            0,
+            constants::BALANCED_IMAGE_SIZE,
+            constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
+        )
         .await
         .unwrap();
     }
@@ -1333,93 +1266,68 @@ mod tests {
         assert_eq!(luma, 0);
     }
 
-    #[derive(Default)]
-    struct SendImageCheckMediaquality<'a> {
-        pub(crate) viewtype: Viewtype,
-        pub(crate) media_quality_config: &'a str,
-        pub(crate) bytes: &'a [u8],
-        pub(crate) extension: &'a str,
-        pub(crate) has_exif: bool,
-        pub(crate) original_width: u32,
-        pub(crate) original_height: u32,
-        pub(crate) orientation: i32,
-        pub(crate) compressed_width: u32,
-        pub(crate) compressed_height: u32,
-        pub(crate) set_draft: bool,
-    }
+    #[allow(clippy::too_many_arguments)]
+    async fn send_image_check_mediaquality(
+        viewtype: Viewtype,
+        media_quality_config: Option<&str>,
+        bytes: &[u8],
+        extension: &str,
+        has_exif: bool,
+        original_width: u32,
+        original_height: u32,
+        orientation: i32,
+        compressed_width: u32,
+        compressed_height: u32,
+    ) -> anyhow::Result<DynamicImage> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        alice
+            .set_config(Config::MediaQuality, media_quality_config)
+            .await?;
+        let file = alice.get_blobdir().join("file").with_extension(extension);
 
-    impl SendImageCheckMediaquality<'_> {
-        pub(crate) async fn test(self) -> anyhow::Result<DynamicImage> {
-            let viewtype = self.viewtype;
-            let media_quality_config = self.media_quality_config;
-            let bytes = self.bytes;
-            let extension = self.extension;
-            let has_exif = self.has_exif;
-            let original_width = self.original_width;
-            let original_height = self.original_height;
-            let orientation = self.orientation;
-            let compressed_width = self.compressed_width;
-            let compressed_height = self.compressed_height;
-            let set_draft = self.set_draft;
+        fs::write(&file, &bytes)
+            .await
+            .context("failed to write file")?;
+        check_image_size(&file, original_width, original_height);
 
-            let alice = TestContext::new_alice().await;
-            let bob = TestContext::new_bob().await;
-            alice
-                .set_config(Config::MediaQuality, Some(media_quality_config))
-                .await?;
-            let file = alice.get_blobdir().join("file").with_extension(extension);
-
-            fs::write(&file, &bytes)
-                .await
-                .context("failed to write file")?;
-            check_image_size(&file, original_width, original_height);
-
-            let (_, exif) = image_metadata(&std::fs::File::open(&file)?)?;
-            if has_exif {
-                let exif = exif.unwrap();
-                assert_eq!(exif_orientation(&exif, &alice), orientation);
-            } else {
-                assert!(exif.is_none());
-            }
-
-            let mut msg = Message::new(viewtype);
-            msg.set_file(file.to_str().unwrap(), None);
-            let chat = alice.create_chat(&bob).await;
-            if set_draft {
-                chat.id.set_draft(&alice, Some(&mut msg)).await.unwrap();
-                msg = chat.id.get_draft(&alice).await.unwrap().unwrap();
-                assert_eq!(msg.get_viewtype(), Viewtype::File);
-            }
-            let sent = alice.send_msg(chat.id, &mut msg).await;
-            let alice_msg = alice.get_last_msg().await;
-            assert_eq!(alice_msg.get_width() as u32, compressed_width);
-            assert_eq!(alice_msg.get_height() as u32, compressed_height);
-            let file_saved = alice
-                .get_blobdir()
-                .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
-            alice_msg.save_file(&alice, &file_saved).await?;
-            check_image_size(file_saved, compressed_width, compressed_height);
-
-            let bob_msg = bob.recv_msg(&sent).await;
-            assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
-            assert_eq!(bob_msg.get_width() as u32, compressed_width);
-            assert_eq!(bob_msg.get_height() as u32, compressed_height);
-            let file_saved = bob
-                .get_blobdir()
-                .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
-            bob_msg.save_file(&bob, &file_saved).await?;
-            if viewtype == Viewtype::File {
-                assert_eq!(file_saved.extension().unwrap(), extension);
-                let bytes1 = fs::read(&file_saved).await?;
-                assert_eq!(&bytes1, bytes);
-            }
-
-            let (_, exif) = image_metadata(&std::fs::File::open(&file_saved)?)?;
+        let blob = BlobObject::new_from_path(&alice, &file).await?;
+        let (_, exif) = blob.metadata()?;
+        if has_exif {
+            let exif = exif.unwrap();
+            assert_eq!(exif_orientation(&exif, &alice), orientation);
+        } else {
             assert!(exif.is_none());
-
-            let img = check_image_size(file_saved, compressed_width, compressed_height);
-            Ok(img)
         }
+
+        let mut msg = Message::new(viewtype);
+        msg.set_file(file.to_str().unwrap(), None);
+        let chat = alice.create_chat(&bob).await;
+        let sent = alice.send_msg(chat.id, &mut msg).await;
+        let alice_msg = alice.get_last_msg().await;
+        assert_eq!(alice_msg.get_width() as u32, compressed_width);
+        assert_eq!(alice_msg.get_height() as u32, compressed_height);
+        let file_saved = alice
+            .get_blobdir()
+            .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
+        alice_msg.save_file(&alice, &file_saved).await?;
+        check_image_size(file_saved, compressed_width, compressed_height);
+
+        let bob_msg = bob.recv_msg(&sent).await;
+        assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
+        assert_eq!(bob_msg.get_width() as u32, compressed_width);
+        assert_eq!(bob_msg.get_height() as u32, compressed_height);
+        let file_saved = bob
+            .get_blobdir()
+            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
+        bob_msg.save_file(&bob, &file_saved).await?;
+
+        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
+        let (_, exif) = blob.metadata()?;
+        assert!(exif.is_none());
+
+        let img = check_image_size(file_saved, compressed_width, compressed_height);
+        Ok(img)
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1451,7 +1359,8 @@ mod tests {
             .get_blobdir()
             .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
         bob_msg.save_file(&bob, &file_saved).await?;
-        let (file_size, _) = image_metadata(&std::fs::File::open(&file_saved)?)?;
+        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
+        let (file_size, _) = blob.metadata()?;
         assert_eq!(file_size, bytes.len() as u64);
         check_image_size(file_saved, width, height);
         Ok(())

src/chatlist.rs

@@ -82,13 +82,11 @@ impl Chatlist {
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
     /// - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
     ///   is added as needed.
-    ///
     /// `query`: An optional query for filtering the list. Only chats matching this query
-    /// are returned. When `is:unread` is contained in the query, the chatlist is
-    /// filtered such that only chats with unread messages show up.
-    ///
+    ///     are returned. When `is:unread` is contained in the query, the chatlist is
+    ///     filtered such that only chats with unread messages show up.
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
-    /// are returned.
+    ///     are returned.
     pub async fn try_load(
         context: &Context,
         listflags: usize,
@@ -394,32 +392,25 @@ impl Chatlist {
             &chat_loaded
         };
 
-        let lastmsg = if let Some(lastmsg_id) = lastmsg_id {
-            // Message may be deleted by the time we try to load it,
-            // so use `load_from_db_optional` instead of `load_from_db`.
-            Message::load_from_db_optional(context, lastmsg_id)
+        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
+            let lastmsg = Message::load_from_db(context, lastmsg_id)
                 .await
-                .context("Loading message failed")?
-        } else {
-            None
-        };
-
-        let lastcontact = if let Some(lastmsg) = &lastmsg {
+                .context("loading message failed")?;
             if lastmsg.from_id == ContactId::SELF {
-                None
+                (Some(lastmsg), None)
             } else {
                 match chat.typ {
                     Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                         let lastcontact = Contact::get_by_id(context, lastmsg.from_id)
                             .await
                             .context("loading contact failed")?;
-                        Some(lastcontact)
+                        (Some(lastmsg), Some(lastcontact))
                     }
-                    Chattype::Single => None,
+                    Chattype::Single => (Some(lastmsg), None),
                 }
             }
         } else {
-            None
+            (None, None)
         };
 
         if chat.id.is_archived_link() {
@@ -483,6 +474,7 @@ mod tests {
         add_contact_to_chat, create_group_chat, get_chat_contacts, remove_contact_from_chat,
         send_text_msg, ProtectionStatus,
     };
+    use crate::message::Viewtype;
     use crate::receive_imf::receive_imf;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
@@ -516,7 +508,8 @@ mod tests {
         // Instead of setting drafts for chat_id1 and chat_id3, we could also sleep
         // 2s here.
         for chat_id in &[chat_id1, chat_id3, chat_id2] {
-            let mut msg = Message::new_text("hello".to_string());
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text("hello".to_string());
             chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
         }
 
@@ -760,7 +753,8 @@ mod tests {
             .await
             .unwrap();
 
-        let mut msg = Message::new_text("foo:\nbar \r\n test".to_string());
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("foo:\nbar \r\n test".to_string());
         chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
@@ -768,25 +762,6 @@ mod tests {
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
 
-    /// Tests that summary does not fail to load
-    /// if the draft was deleted after loading the chatlist.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_summary_deleted_draft() {
-        let t = TestContext::new().await;
-
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
-            .await
-            .unwrap();
-        let mut msg = Message::new_text("Foobar".to_string());
-        chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
-
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        chat_id.set_draft(&t, None).await.unwrap();
-
-        let summary_res = chats.get_summary(&t, 0, None).await;
-        assert!(summary_res.is_ok());
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_broken() {
         let t = TestContext::new_bob().await;

src/config.rs

@@ -6,21 +6,21 @@ use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
 use base64::Engine as _;
-use deltachat_contact_tools::{addr_cmp, sanitize_single_line};
+use deltachat_contact_tools::addr_cmp;
 use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumString};
 use tokio::fs;
 
 use crate::blob::BlobObject;
-use crate::constants;
+use crate::constants::{self, DC_VERSION_STR};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 use crate::sync::{self, Sync::*, SyncData};
-use crate::tools::get_abs_path;
+use crate::tools::{get_abs_path, improve_single_line_input};
 
 /// The available configuration keys.
 #[derive(
@@ -59,10 +59,7 @@ pub enum Config {
     /// IMAP server security (e.g. TLS, STARTTLS).
     MailSecurity,
 
-    /// How to check TLS certificates.
-    ///
-    /// "IMAP" in the name is for compatibility,
-    /// this actually applies to both IMAP and SMTP connections.
+    /// How to check IMAP server TLS certificates.
     ImapCertificateChecks,
 
     /// SMTP server hostname.
@@ -80,9 +77,7 @@ pub enum Config {
     /// SMTP server security (e.g. TLS, STARTTLS).
     SendSecurity,
 
-    /// Deprecated option for backwards compatibilty.
-    ///
-    /// Certificate checks for SMTP are actually controlled by `imap_certificate_checks` config.
+    /// How to check SMTP server TLS certificates.
     SmtpCertificateChecks,
 
     /// Whether to use OAuth 2.
@@ -91,44 +86,21 @@ pub enum Config {
     /// Should not be extended in the future, create new config keys instead.
     ServerFlags,
 
-    /// True if proxy is enabled.
-    ///
-    /// Can be used to disable proxy without erasing known URLs.
-    ProxyEnabled,
-
-    /// Proxy URL.
-    ///
-    /// Supported URLs schemes are `http://` (HTTP), `https://` (HTTPS),
-    /// `socks5://` (SOCKS5) and `ss://` (Shadowsocks).
-    ///
-    /// May contain multiple URLs separated by newline, in which case the first one is used.
-    ProxyUrl,
-
     /// True if SOCKS5 is enabled.
     ///
     /// Can be used to disable SOCKS5 without erasing SOCKS5 configuration.
-    ///
-    /// Deprecated in favor of `ProxyEnabled`.
     Socks5Enabled,
 
     /// SOCKS5 proxy server hostname or address.
-    ///
-    /// Deprecated in favor of `ProxyUrl`.
     Socks5Host,
 
     /// SOCKS5 proxy server port.
-    ///
-    /// Deprecated in favor of `ProxyUrl`.
     Socks5Port,
 
     /// SOCKS5 proxy server username.
-    ///
-    /// Deprecated in favor of `ProxyUrl`.
     Socks5User,
 
     /// SOCKS5 proxy server password.
-    ///
-    /// Deprecated in favor of `ProxyUrl`.
     Socks5Password,
 
     /// Own name to use in the `From:` field when sending messages.
@@ -159,8 +131,7 @@ pub enum Config {
     #[strum(props(default = "0"))]
     SentboxWatch,
 
-    /// True if chat messages should be moved to a separate folder. Auto-sent messages like sync
-    /// ones are moved there anyway.
+    /// True if chat messages should be moved to a separate folder.
     #[strum(props(default = "1"))]
     MvboxMove,
 
@@ -197,12 +168,12 @@ pub enum Config {
     /// Timer in seconds after which the message is deleted from the
     /// server.
     ///
-    /// 0 means messages are never deleted by Delta Chat.
+    /// Equals to 0 by default, which means the message is never
+    /// deleted.
     ///
     /// Value 1 is treated as "delete at once": messages are deleted
     /// immediately, without moving to DeltaChat folder.
-    ///
-    /// Default is 1 for chatmail accounts before a backup export, 0 otherwise.
+    #[strum(props(default = "0"))]
     DeleteServerAfter,
 
     /// Timer in seconds after which the message is deleted from the
@@ -223,74 +194,45 @@ pub enum Config {
     /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
 
-    /// List of configured IMAP servers as a JSON array.
-    ConfiguredImapServers,
-
     /// Configured IMAP server hostname.
-    ///
-    /// This is replaced by `configured_imap_servers` for new configurations.
     ConfiguredMailServer,
 
-    /// Configured IMAP server port.
-    ///
-    /// This is replaced by `configured_imap_servers` for new configurations.
-    ConfiguredMailPort,
-
-    /// Configured IMAP server security (e.g. TLS, STARTTLS).
-    ///
-    /// This is replaced by `configured_imap_servers` for new configurations.
-    ConfiguredMailSecurity,
-
     /// Configured IMAP server username.
-    ///
-    /// This is set if user has configured username manually.
     ConfiguredMailUser,
 
     /// Configured IMAP server password.
     ConfiguredMailPw,
 
-    /// Configured TLS certificate checks.
-    /// This option is saved on successful configuration
-    /// and should not be modified manually.
-    ///
-    /// This actually applies to both IMAP and SMTP connections,
-    /// but has "IMAP" in the name for backwards compatibility.
+    /// Configured IMAP server port.
+    ConfiguredMailPort,
+
+    /// Configured IMAP server security (e.g. TLS, STARTTLS).
+    ConfiguredMailSecurity,
+
+    /// How to check IMAP server TLS certificates.
     ConfiguredImapCertificateChecks,
 
-    /// List of configured SMTP servers as a JSON array.
-    ConfiguredSmtpServers,
-
     /// Configured SMTP server hostname.
-    ///
-    /// This is replaced by `configured_smtp_servers` for new configurations.
     ConfiguredSendServer,
 
-    /// Configured SMTP server port.
-    ///
-    /// This is replaced by `configured_smtp_servers` for new configurations.
-    ConfiguredSendPort,
-
-    /// Configured SMTP server security (e.g. TLS, STARTTLS).
-    ///
-    /// This is replaced by `configured_smtp_servers` for new configurations.
-    ConfiguredSendSecurity,
-
     /// Configured SMTP server username.
-    ///
-    /// This is set if user has configured username manually.
     ConfiguredSendUser,
 
     /// Configured SMTP server password.
     ConfiguredSendPw,
 
-    /// Deprecated, stored for backwards compatibility.
-    ///
-    /// ConfiguredImapCertificateChecks is actually used.
+    /// Configured SMTP server port.
+    ConfiguredSendPort,
+
+    /// How to check SMTP server TLS certificates.
     ConfiguredSmtpCertificateChecks,
 
     /// Whether OAuth 2 is used with configured provider.
     ConfiguredServerFlags,
 
+    /// Configured SMTP server security (e.g. TLS, STARTTLS).
+    ConfiguredSendSecurity,
+
     /// Configured folder for incoming messages.
     ConfiguredInboxFolder,
 
@@ -315,16 +257,6 @@ pub enum Config {
     /// True if account is a chatmail account.
     IsChatmail,
 
-    /// True if `IsChatmail` mustn't be autoconfigured. For tests.
-    FixIsChatmail,
-
-    /// True if account is muted.
-    IsMuted,
-
-    /// Optional tag as "Work", "Family".
-    /// Meant to help profile owner to differ between profiles with similar names.
-    PrivateTag,
-
     /// All secondary self addresses separated by spaces
     /// (`addr1@example.org addr2@example.org addr3@example.org`)
     SecondaryAddrs,
@@ -382,8 +314,7 @@ pub enum Config {
     #[strum(props(default = "0"))]
     DownloadLimit,
 
-    /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set
-    /// and `Bot` unset.
+    /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set.
     #[strum(props(default = "1"))]
     SyncMsgs,
 
@@ -396,12 +327,6 @@ pub enum Config {
     /// Make all outgoing messages with Autocrypt header "multipart/signed".
     SignUnencrypted,
 
-    /// Enable header protection for `Autocrypt` header.
-    ///
-    /// This is an experimental setting not compatible to other MUAs
-    /// and older Delta Chat versions (core version <= v1.149.0).
-    ProtectAutocrypt,
-
     /// Let the core save all events to the database.
     /// This value is used internally to remember the MsgId of the logging xdc
     #[strum(props(default = "0"))]
@@ -438,9 +363,8 @@ pub enum Config {
     /// MsgId of webxdc map integration.
     WebxdcIntegration,
 
-    /// Enable webxdc realtime features.
-    #[strum(props(default = "1"))]
-    WebxdcRealtimeEnabled,
+    /// Iroh secret key.
+    IrohSecretKey,
 }
 
 impl Config {
@@ -454,11 +378,14 @@ impl Config {
     /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
     /// mustn't be controlled by other devices.
     pub(crate) fn is_synced(&self) -> bool {
+        // We don't restart IO from the synchronisation code, so this is to be on the safe side.
+        if self.needs_io_restart() {
+            return false;
+        }
         matches!(
             self,
             Self::Displayname
                 | Self::MdnsEnabled
-                | Self::MvboxMove
                 | Self::ShowEmails
                 | Self::Selfavatar
                 | Self::Selfstatus,
@@ -467,21 +394,21 @@ impl Config {
 
     /// Whether the config option needs an IO scheduler restart to take effect.
     pub(crate) fn needs_io_restart(&self) -> bool {
-        matches!(self, Config::OnlyFetchMvbox | Config::SentboxWatch)
+        matches!(
+            self,
+            Config::MvboxMove | Config::OnlyFetchMvbox | Config::SentboxWatch
+        )
     }
 }
 
 impl Context {
-    /// Returns true if configuration value is set in the db for the given key.
-    ///
-    /// NB: Don't use this to check if the key is configured because this doesn't look into
-    /// environment. The proper use of this function is e.g. checking a key before setting it.
-    pub(crate) async fn config_exists(&self, key: Config) -> Result<bool> {
+    /// Returns true if configuration value is set for the given key.
+    pub async fn config_exists(&self, key: Config) -> Result<bool> {
         Ok(self.sql.get_raw_config(key.as_ref()).await?.is_some())
     }
 
-    /// Get a config key value. Returns `None` if no value is set.
-    pub(crate) async fn get_config_opt(&self, key: Config) -> Result<Option<String>> {
+    /// Get a configuration key. Returns `None` if no value is set, and no default value found.
+    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
         let env_key = format!("DELTACHAT_{}", key.as_ref().to_uppercase());
         if let Ok(value) = env::var(env_key) {
             return Ok(Some(value));
@@ -496,43 +423,24 @@ impl Context {
                         .into_owned()
                 })
             }
-            Config::SysVersion => Some((*constants::DC_VERSION_STR).clone()),
+            Config::SysVersion => Some((*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{RECOMMENDED_FILE_SIZE}")),
             Config::SysConfigKeys => Some(get_config_keys_string()),
             _ => self.sql.get_raw_config(key.as_ref()).await?,
         };
-        Ok(value)
-    }
 
-    /// Get a config key value if set, or a default value. Returns `None` if no value exists.
-    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
-        let value = self.get_config_opt(key).await?;
         if value.is_some() {
             return Ok(value);
         }
 
         // Default values
-        let val = match key {
-            Config::ConfiguredInboxFolder => Some("INBOX"),
-            Config::DeleteServerAfter => match Box::pin(self.is_chatmail()).await? {
-                false => Some("0"),
-                true => Some("1"),
-            },
-            _ => key.get_str("default"),
-        };
-        Ok(val.map(|s| s.to_string()))
+        match key {
+            Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
+            _ => Ok(key.get_str("default").map(|s| s.to_string())),
+        }
     }
 
-    /// Returns Some(T) if a value for the given key is set and was successfully parsed.
-    /// Returns None if could not parse.
-    pub(crate) async fn get_config_opt_parsed<T: FromStr>(&self, key: Config) -> Result<Option<T>> {
-        self.get_config_opt(key)
-            .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()))
-    }
-
-    /// Returns Some(T) if a value for the given key exists (incl. default value) and was
-    /// successfully parsed.
+    /// Returns Some(T) if a value for the given key exists and was successfully parsed.
     /// Returns None if could not parse.
     pub async fn get_config_parsed<T: FromStr>(&self, key: Config) -> Result<Option<T>> {
         self.get_config(key)
@@ -560,28 +468,20 @@ impl Context {
         Ok(self.get_config_parsed(key).await?.unwrap_or_default())
     }
 
-    /// Returns boolean configuration value (if set) for the given key.
-    pub(crate) async fn get_config_bool_opt(&self, key: Config) -> Result<Option<bool>> {
-        Ok(self
-            .get_config_opt_parsed::<i32>(key)
-            .await?
-            .map(|x| x != 0))
+    /// Returns boolean configuration value (if any) for the given key.
+    pub async fn get_config_bool_opt(&self, key: Config) -> Result<Option<bool>> {
+        Ok(self.get_config_parsed::<i32>(key).await?.map(|x| x != 0))
     }
 
     /// Returns boolean configuration value for the given key.
     pub async fn get_config_bool(&self, key: Config) -> Result<bool> {
-        Ok(self
-            .get_config_parsed::<i32>(key)
-            .await?
-            .map(|x| x != 0)
-            .unwrap_or_default())
+        Ok(self.get_config_bool_opt(key).await?.unwrap_or_default())
     }
 
     /// Returns true if movebox ("DeltaChat" folder) should be watched.
     pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
         Ok(self.get_config_bool(Config::MvboxMove).await?
-            || self.get_config_bool(Config::OnlyFetchMvbox).await?
-            || !self.get_config_bool(Config::IsChatmail).await?)
+            || self.get_config_bool(Config::OnlyFetchMvbox).await?)
     }
 
     /// Returns true if sentbox ("Sent" folder) should be watched.
@@ -593,47 +493,16 @@ impl Context {
                 .is_some())
     }
 
-    /// Returns true if sync messages should be sent.
-    pub(crate) async fn should_send_sync_msgs(&self) -> Result<bool> {
-        Ok(self.get_config_bool(Config::SyncMsgs).await?
-            && self.get_config_bool(Config::BccSelf).await?
-            && !self.get_config_bool(Config::Bot).await?)
-    }
-
-    /// Returns whether sync messages should be uploaded to the mvbox.
-    pub(crate) async fn should_move_sync_msgs(&self) -> Result<bool> {
-        Ok(self.get_config_bool(Config::MvboxMove).await?
-            || !self.get_config_bool(Config::IsChatmail).await?)
-    }
-
-    /// Returns whether MDNs should be requested.
-    pub(crate) async fn should_request_mdns(&self) -> Result<bool> {
-        match self.get_config_bool_opt(Config::MdnsEnabled).await? {
-            Some(val) => Ok(val),
-            None => Ok(!self.get_config_bool(Config::Bot).await?),
-        }
-    }
-
-    /// Returns whether MDNs should be sent.
-    pub(crate) async fn should_send_mdns(&self) -> Result<bool> {
-        self.get_config_bool(Config::MdnsEnabled).await
-    }
-
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
     pub async fn get_config_delete_server_after(&self) -> Result<Option<i64>> {
-        let val = match self
-            .get_config_parsed::<i64>(Config::DeleteServerAfter)
-            .await?
-            .unwrap_or(0)
-        {
-            0 => None,
-            1 => Some(0),
-            x => Some(x),
-        };
-        Ok(val)
+        match self.get_config_int(Config::DeleteServerAfter).await? {
+            0 => Ok(None),
+            1 => Ok(Some(0)),
+            x => Ok(Some(i64::from(x))),
+        }
     }
 
     /// Gets the configured provider, as saved in the `configured_provider` value.
@@ -678,7 +547,6 @@ impl Context {
     fn check_config(key: Config, value: Option<&str>) -> Result<()> {
         match key {
             Config::Socks5Enabled
-            | Config::ProxyEnabled
             | Config::BccSelf
             | Config::E2eeEnabled
             | Config::MdnsEnabled
@@ -732,7 +600,7 @@ impl Context {
         mut value: Option<&str>,
     ) -> Result<()> {
         Self::check_config(key, value)?;
-        let sync = sync == Sync && key.is_synced() && self.is_configured().await?;
+        let sync = sync == Sync && key.is_synced();
         let better_value;
 
         match key {
@@ -771,7 +639,7 @@ impl Context {
             }
             Config::Displayname => {
                 if let Some(v) = value {
-                    better_value = sanitize_single_line(v);
+                    better_value = improve_single_line_input(v);
                     value = Some(&better_value);
                 }
                 self.sql.set_raw_config(key.as_ref(), value).await?;
@@ -809,7 +677,7 @@ impl Context {
         {
             return Ok(());
         }
-        self.scheduler.interrupt_inbox().await;
+        Box::pin(self.send_sync_msg()).await.log_err(self).ok();
         Ok(())
     }
 
@@ -868,8 +736,6 @@ impl Context {
     ///
     /// This should only be used by test code and during configure.
     pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
-        self.quota.write().await.take();
-
         // add old primary address (if exists) to secondary addresses
         let mut secondary_addrs = self.get_all_self_addrs().await?;
         // never store a primary address also as a secondary
@@ -882,7 +748,7 @@ impl Context {
 
         self.set_config_internal(Config::ConfiguredAddr, Some(primary_new))
             .await?;
-        self.emit_event(EventType::ConnectivityChanged);
+
         Ok(())
     }
 
@@ -1076,23 +942,6 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_mdns_default_behaviour() -> Result<()> {
-        let t = &TestContext::new_alice().await;
-        assert!(t.should_request_mdns().await?);
-        assert!(t.should_send_mdns().await?);
-        assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
-        // The setting should be displayed correctly.
-        assert!(t.get_config_bool(Config::MdnsEnabled).await?);
-
-        t.set_config_bool(Config::Bot, true).await?;
-        assert!(!t.should_request_mdns().await?);
-        assert!(t.should_send_mdns().await?);
-        assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
-        assert!(t.get_config_bool(Config::MdnsEnabled).await?);
-        Ok(())
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sync() -> Result<()> {
         let alice0 = TestContext::new_alice().await;
@@ -1119,16 +968,20 @@ mod tests {
         // Reset to default. Test that it's not synced because defaults may differ across client
         // versions.
         alice0.set_config(Config::MdnsEnabled, None).await?;
+        assert_eq!(alice0.get_config_bool(Config::MdnsEnabled).await?, true);
         alice0.set_config_bool(Config::MdnsEnabled, false).await?;
         sync(&alice0, &alice1).await;
         assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
 
-        for key in [Config::ShowEmails, Config::MvboxMove] {
-            let val = alice0.get_config_bool(key).await?;
-            alice0.set_config_bool(key, !val).await?;
-            sync(&alice0, &alice1).await;
-            assert_eq!(alice1.get_config_bool(key).await?, !val);
-        }
+        let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
+        alice0
+            .set_config_bool(Config::ShowEmails, !show_emails)
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config_bool(Config::ShowEmails).await?,
+            !show_emails
+        );
 
         // `Config::SyncMsgs` mustn't be synced.
         alice0.set_config_bool(Config::SyncMsgs, false).await?;
@@ -1193,8 +1046,7 @@ mod tests {
 
         let status = "Synced via usual message";
         alice0.set_config(Config::Selfstatus, Some(status)).await?;
-        alice0.send_sync_msg().await?;
-        alice0.pop_sent_sync_msg().await;
+        alice0.pop_sent_msg().await; // Sync message
         let status1 = "Synced via sync message";
         alice1.set_config(Config::Selfstatus, Some(status1)).await?;
         tcm.send_recv(alice0, alice1, "hi Alice!").await;
@@ -1217,8 +1069,7 @@ mod tests {
         alice0
             .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
             .await?;
-        alice0.send_sync_msg().await?;
-        alice0.pop_sent_sync_msg().await;
+        alice0.pop_sent_msg().await; // Sync message
         let file = alice1.dir.path().join("avatar.jpg");
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         tokio::fs::write(&file, bytes).await?;

src/configure.rs

@@ -11,31 +11,28 @@
 
 mod auto_mozilla;
 mod auto_outlook;
-pub(crate) mod server_params;
+mod server_params;
 
-use anyhow::{bail, ensure, format_err, Context as _, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
 use deltachat_contact_tools::EmailAddress;
 use futures::FutureExt;
 use futures_lite::FutureExt as _;
-use percent_encoding::utf8_percent_encode;
+use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use server_params::{expand_param_vector, ServerParams};
 use tokio::task;
 
 use crate::config::{self, Config};
-use crate::constants::NON_ALPHANUMERIC_WITHOUT_DOT;
 use crate::context::Context;
-use crate::imap::Imap;
+use crate::imap::{session::Session as ImapSession, Imap};
 use crate::log::LogExt;
-use crate::login_param::{
-    ConfiguredCertificateChecks, ConfiguredLoginParam, ConfiguredServerLoginParam,
-    ConnectionCandidate, EnteredCertificateChecks, EnteredLoginParam,
-};
-use crate::message::Message;
+use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
+use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
+use crate::socks::Socks5Config;
 use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::tools::time;
@@ -81,7 +78,10 @@ impl Context {
 
         let res = self
             .inner_configure()
-            .race(cancel_channel.recv().map(|_| Err(format_err!("Cancelled"))))
+            .race(cancel_channel.recv().map(|_| {
+                progress!(self, 0);
+                Ok(())
+            }))
             .await;
 
         self.free_ongoing().await;
@@ -110,19 +110,29 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let param = EnteredLoginParam::load(self).await?;
+        let mut param = LoginParam::load_candidate_params(self).await?;
         let old_addr = self.get_config(Config::ConfiguredAddr).await?;
-        let configured_param = configure(self, &param).await?;
+
+        // Reset our knowledge about whether the server is a chatmail server.
+        // We will update it when we connect to IMAP.
+        self.set_config_internal(Config::IsChatmail, None).await?;
+
+        let success = configure(self, &mut param).await;
+        self.set_config_internal(Config::NotifyAboutWrongPw, None)
+            .await?;
+
+        on_configure_completed(self, param, old_addr).await?;
+
+        success?;
         self.set_config_internal(Config::NotifyAboutWrongPw, Some("1"))
             .await?;
-        on_configure_completed(self, configured_param, old_addr).await?;
         Ok(())
     }
 }
 
 async fn on_configure_completed(
     context: &Context,
-    param: ConfiguredLoginParam,
+    param: LoginParam,
     old_addr: Option<String>,
 ) -> Result<()> {
     if let Some(provider) = param.provider {
@@ -143,7 +153,8 @@ async fn on_configure_completed(
         }
 
         if !provider.after_login_hint.is_empty() {
-            let mut msg = Message::new_text(provider.after_login_hint.to_string());
+            let mut msg = Message::new(Viewtype::Text);
+            msg.text = provider.after_login_hint.to_string();
             if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg))
                 .await
                 .is_err()
@@ -156,9 +167,9 @@ async fn on_configure_completed(
     if let Some(new_addr) = context.get_config(Config::ConfiguredAddr).await? {
         if let Some(old_addr) = old_addr {
             if !addr_cmp(&new_addr, &old_addr) {
-                let mut msg = Message::new_text(
-                    stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await,
-                );
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text =
+                    stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await;
                 chat::add_device_msg(context, None, Some(&mut msg))
                     .await
                     .context("Cannot add AEAP explanation")
@@ -171,28 +182,21 @@ async fn on_configure_completed(
     Ok(())
 }
 
-/// Retrieves data from autoconfig and provider database
-/// to transform user-entered login parameters into complete configuration.
-async fn get_configured_param(
-    ctx: &Context,
-    param: &EnteredLoginParam,
-) -> Result<ConfiguredLoginParam> {
-    ensure!(!param.addr.is_empty(), "Missing email address.");
+async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
+    progress!(ctx, 1);
 
-    ensure!(!param.imap.password.is_empty(), "Missing (IMAP) password.");
+    let socks5_config = param.socks5_config.clone();
+    let socks5_enabled = socks5_config.is_some();
 
-    // SMTP password is an "advanced" setting. If unset, use the same password as for IMAP.
-    let smtp_password = if param.smtp.password.is_empty() {
-        param.imap.password.clone()
-    } else {
-        param.smtp.password.clone()
-    };
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
-    let proxy_config = param.proxy_config.clone();
-    let proxy_enabled = proxy_config.is_some();
+    // Step 1: Load the parameters and check email-address and password
 
-    let mut addr = param.addr.clone();
-    if param.oauth2 {
+    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
+    // socks5 requests, this can work with socks5 too.  OAuth is always set either for both
+    // IMAP and SMTP or not at all.
+    if param.imap.oauth2 && !socks5_enabled {
         // the used oauth2 addr may differ, check this.
         // if get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -201,7 +205,7 @@ async fn get_configured_param(
             .and_then(|e| e.parse().ok())
         {
             info!(ctx, "Authorized address is {}", oauth2_addr);
-            addr = oauth2_addr;
+            param.addr = oauth2_addr;
             ctx.sql
                 .set_raw_config("addr", Some(param.addr.as_str()))
                 .await?;
@@ -212,10 +216,11 @@ async fn get_configured_param(
 
     let parsed = EmailAddress::new(&param.addr).context("Bad email-address")?;
     let param_domain = parsed.domain;
+    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
 
+    // Step 2: Autoconfig
     progress!(ctx, 200);
 
-    let provider;
     let param_autoconfig;
     if param.imap.server.is_empty()
         && param.imap.port == 0
@@ -227,48 +232,66 @@ async fn get_configured_param(
         && param.smtp.user.is_empty()
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
+
         info!(
             ctx,
             "checking internal provider-info for offline autoconfig"
         );
 
-        provider = provider::get_provider_info(ctx, &param_domain, proxy_enabled).await;
-        if let Some(provider) = provider {
-            if provider.server.is_empty() {
-                info!(ctx, "Offline autoconfig found, but no servers defined.");
-                param_autoconfig = None;
-            } else {
-                info!(ctx, "Offline autoconfig found.");
-                let servers = provider
-                    .server
-                    .iter()
-                    .map(|s| ServerParams {
-                        protocol: s.protocol,
-                        socket: s.socket,
-                        hostname: s.hostname.to_string(),
-                        port: s.port,
-                        username: match s.username_pattern {
-                            UsernamePattern::Email => param.addr.to_string(),
-                            UsernamePattern::Emaillocalpart => {
-                                if let Some(at) = param.addr.find('@') {
-                                    param.addr.split_at(at).0.to_string()
-                                } else {
-                                    param.addr.to_string()
-                                }
-                            }
-                        },
-                    })
-                    .collect();
+        if let Some(provider) =
+            provider::get_provider_info(ctx, &param_domain, socks5_enabled).await
+        {
+            param.provider = Some(provider);
+            match provider.status {
+                provider::Status::Ok | provider::Status::Preparation => {
+                    if provider.server.is_empty() {
+                        info!(ctx, "offline autoconfig found, but no servers defined");
+                        param_autoconfig = None;
+                    } else {
+                        info!(ctx, "offline autoconfig found");
+                        let servers = provider
+                            .server
+                            .iter()
+                            .map(|s| ServerParams {
+                                protocol: s.protocol,
+                                socket: s.socket,
+                                hostname: s.hostname.to_string(),
+                                port: s.port,
+                                username: match s.username_pattern {
+                                    UsernamePattern::Email => param.addr.to_string(),
+                                    UsernamePattern::Emaillocalpart => {
+                                        if let Some(at) = param.addr.find('@') {
+                                            param.addr.split_at(at).0.to_string()
+                                        } else {
+                                            param.addr.to_string()
+                                        }
+                                    }
+                                },
+                                strict_tls: Some(provider.opt.strict_tls),
+                            })
+                            .collect();
 
-                param_autoconfig = Some(servers)
+                        param_autoconfig = Some(servers)
+                    }
+                }
+                provider::Status::Broken => {
+                    info!(ctx, "offline autoconfig found, provider is broken");
+                    param_autoconfig = None;
+                }
             }
         } else {
             // Try receiving autoconfig
-            info!(ctx, "No offline autoconfig found.");
-            param_autoconfig = get_autoconfig(ctx, param, &param_domain).await;
+            info!(ctx, "no offline autoconfig found");
+            param_autoconfig = if socks5_enabled {
+                // Currently we can't do http requests through socks5, to not leak
+                // the ip, just don't do online autoconfig
+                info!(ctx, "socks5 enabled, skipping autoconfig");
+                None
+            } else {
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
+            }
         }
     } else {
-        provider = None;
         param_autoconfig = None;
     }
 
@@ -285,6 +308,7 @@ async fn get_configured_param(
             port: param.imap.port,
             socket: param.imap.security,
             username: param.imap.user.clone(),
+            strict_tls: None,
         })
     }
     if !servers
@@ -297,150 +321,144 @@ async fn get_configured_param(
             port: param.smtp.port,
             socket: param.smtp.security,
             username: param.smtp.user.clone(),
+            strict_tls: None,
         })
     }
 
+    // respect certificate setting from function parameters
+    for server in &mut servers {
+        let certificate_checks = match server.protocol {
+            Protocol::Imap => param.imap.certificate_checks,
+            Protocol::Smtp => param.smtp.certificate_checks,
+        };
+        server.strict_tls = match certificate_checks {
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
+            CertificateChecks::Strict => Some(true),
+            CertificateChecks::Automatic => server.strict_tls,
+        };
+    }
+
     let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
-    let configured_login_param = ConfiguredLoginParam {
-        addr,
-        imap: servers
-            .iter()
-            .filter_map(|params| {
-                let Ok(security) = params.socket.try_into() else {
-                    return None;
-                };
-                if params.protocol == Protocol::Imap {
-                    Some(ConfiguredServerLoginParam {
-                        connection: ConnectionCandidate {
-                            host: params.hostname.clone(),
-                            port: params.port,
-                            security,
-                        },
-                        user: params.username.clone(),
-                    })
-                } else {
-                    None
-                }
-            })
-            .collect(),
-        imap_user: param.imap.user.clone(),
-        imap_password: param.imap.password.clone(),
-        smtp: servers
-            .iter()
-            .filter_map(|params| {
-                let Ok(security) = params.socket.try_into() else {
-                    return None;
-                };
-                if params.protocol == Protocol::Smtp {
-                    Some(ConfiguredServerLoginParam {
-                        connection: ConnectionCandidate {
-                            host: params.hostname.clone(),
-                            port: params.port,
-                            security,
-                        },
-                        user: params.username.clone(),
-                    })
-                } else {
-                    None
-                }
-            })
-            .collect(),
-        smtp_user: param.smtp.user.clone(),
-        smtp_password,
-        proxy_config: param.proxy_config.clone(),
-        provider,
-        certificate_checks: match param.certificate_checks {
-            EnteredCertificateChecks::Automatic => ConfiguredCertificateChecks::Automatic,
-            EnteredCertificateChecks::Strict => ConfiguredCertificateChecks::Strict,
-            EnteredCertificateChecks::AcceptInvalidCertificates
-            | EnteredCertificateChecks::AcceptInvalidCertificates2 => {
-                ConfiguredCertificateChecks::AcceptInvalidCertificates
-            }
-        },
-        oauth2: param.oauth2,
-    };
-    Ok(configured_login_param)
-}
-
-async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<ConfiguredLoginParam> {
-    progress!(ctx, 1);
-
-    let ctx2 = ctx.clone();
-    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
-
-    let configured_param = get_configured_param(ctx, param).await?;
-    let strict_tls = configured_param.strict_tls();
-
     progress!(ctx, 550);
 
     // Spawn SMTP configuration task
-    // to try SMTP while connecting to IMAP.
+    let mut smtp = Smtp::new();
+
     let context_smtp = ctx.clone();
-    let smtp_param = configured_param.smtp.clone();
-    let smtp_password = configured_param.smtp_password.clone();
-    let smtp_addr = configured_param.addr.clone();
-    let proxy_config = configured_param.proxy_config.clone();
+    let mut smtp_param = param.smtp.clone();
+    let smtp_addr = param.addr.clone();
+    let smtp_servers: Vec<ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::Smtp)
+        .cloned()
+        .collect();
+    let provider_strict_tls = param
+        .provider
+        .map_or(socks5_config.is_some(), |provider| provider.opt.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
-        let mut smtp = Smtp::new();
-        smtp.connect(
-            &context_smtp,
-            &smtp_param,
-            &smtp_password,
-            &proxy_config,
-            &smtp_addr,
-            strict_tls,
-            configured_param.oauth2,
-        )
-        .await?;
+        let mut smtp_configured = false;
+        let mut errors = Vec::new();
+        for smtp_server in smtp_servers {
+            smtp_param.user.clone_from(&smtp_server.username);
+            smtp_param.server.clone_from(&smtp_server.hostname);
+            smtp_param.port = smtp_server.port;
+            smtp_param.security = smtp_server.socket;
+            smtp_param.certificate_checks = match smtp_server.strict_tls {
+                Some(true) => CertificateChecks::Strict,
+                Some(false) => CertificateChecks::AcceptInvalidCertificates,
+                None => CertificateChecks::Automatic,
+            };
 
-        Ok::<(), anyhow::Error>(())
+            match try_smtp_one_param(
+                &context_smtp,
+                &smtp_param,
+                &socks5_config,
+                &smtp_addr,
+                provider_strict_tls,
+                &mut smtp,
+            )
+            .await
+            {
+                Ok(_) => {
+                    smtp_configured = true;
+                    break;
+                }
+                Err(e) => errors.push(e),
+            }
+        }
+
+        if smtp_configured {
+            Ok(smtp_param)
+        } else {
+            Err(errors)
+        }
     });
 
     progress!(ctx, 600);
 
     // Configure IMAP
 
-    let (_s, r) = async_channel::bounded(1);
-    let mut imap = Imap::new(
-        configured_param.imap.clone(),
-        configured_param.imap_password.clone(),
-        configured_param.proxy_config.clone(),
-        &configured_param.addr,
-        strict_tls,
-        configured_param.oauth2,
-        r,
-    );
-    let configuring = true;
-    let mut imap_session = match imap.connect(ctx, configuring).await {
-        Ok(session) => session,
-        Err(err) => bail!("{}", nicer_configuration_error(ctx, err.to_string()).await),
+    let mut imap: Option<(Imap, ImapSession)> = None;
+    let imap_servers: Vec<&ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::Imap)
+        .collect();
+    let imap_servers_count = imap_servers.len();
+    let mut errors = Vec::new();
+    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
+        param.imap.user.clone_from(&imap_server.username);
+        param.imap.server.clone_from(&imap_server.hostname);
+        param.imap.port = imap_server.port;
+        param.imap.security = imap_server.socket;
+        param.imap.certificate_checks = match imap_server.strict_tls {
+            Some(true) => CertificateChecks::Strict,
+            Some(false) => CertificateChecks::AcceptInvalidCertificates,
+            None => CertificateChecks::Automatic,
+        };
+
+        match try_imap_one_param(
+            ctx,
+            &param.imap,
+            &param.socks5_config,
+            &param.addr,
+            provider_strict_tls,
+        )
+        .await
+        {
+            Ok(configured_imap) => {
+                imap = Some(configured_imap);
+                break;
+            }
+            Err(e) => errors.push(e),
+        }
+        progress!(
+            ctx,
+            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
+        );
+    }
+    let (mut imap, mut imap_session) = match imap {
+        Some(imap) => imap,
+        None => bail!(nicer_configuration_error(ctx, errors).await),
     };
 
     progress!(ctx, 850);
 
     // Wait for SMTP configuration
-    smtp_config_task.await.unwrap()?;
+    match smtp_config_task.await.unwrap() {
+        Ok(smtp_param) => {
+            param.smtp = smtp_param;
+        }
+        Err(errors) => {
+            bail!(nicer_configuration_error(ctx, errors).await);
+        }
+    }
 
     progress!(ctx, 900);
 
-    let is_chatmail = match ctx.get_config_bool(Config::FixIsChatmail).await? {
-        false => {
-            let is_chatmail = imap_session.is_chatmail();
-            ctx.set_config(
-                Config::IsChatmail,
-                Some(match is_chatmail {
-                    false => "0",
-                    true => "1",
-                }),
-            )
-            .await?;
-            is_chatmail
-        }
-        true => ctx.get_config_bool(Config::IsChatmail).await?,
-    };
-    if is_chatmail {
+    if imap_session.is_chatmail() {
         ctx.set_config(Config::SentboxWatch, None).await?;
         ctx.set_config(Config::MvboxMove, Some("0")).await?;
         ctx.set_config(Config::OnlyFetchMvbox, None).await?;
@@ -448,7 +466,8 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
         ctx.set_config(Config::E2eeEnabled, Some("1")).await?;
     }
 
-    let create_mvbox = !is_chatmail;
+    let create_mvbox = ctx.should_watch_mvbox().await?;
+
     imap.configure_folders(ctx, &mut imap_session, create_mvbox)
         .await?;
 
@@ -469,7 +488,8 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
         }
     }
 
-    configured_param.save_as_configured_params(ctx).await?;
+    // the trailing underscore is correct
+    param.save_as_configured_params(ctx).await?;
     ctx.set_config_internal(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -487,34 +507,25 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
 
     ctx.sql.set_raw_config_bool("configured", true).await?;
 
-    Ok(configured_param)
+    Ok(())
 }
 
 /// Retrieve available autoconfigurations.
 ///
-/// A. Search configurations from the domain used in the email-address
-/// B. If we have no configuration yet, search configuration in Thunderbird's central database
+/// A Search configurations from the domain used in the email-address, prefer encrypted
+/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
 async fn get_autoconfig(
     ctx: &Context,
-    param: &EnteredLoginParam,
+    param: &LoginParam,
     param_domain: &str,
+    param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
-    // Make sure to not encode `.` as `%2E` here.
-    // Some servers like murena.io on 2024-11-01 produce incorrect autoconfig XML
-    // when address is encoded.
-    // E.g.
-    // <https://autoconfig.murena.io/mail/config-v1.1.xml?emailaddress=foobar%40example%2Eorg>
-    // produced XML file with `<username>foobar@example%2Eorg</username>`
-    // resulting in failure to log in.
-    let param_addr_urlencoded =
-        utf8_percent_encode(&param.addr, NON_ALPHANUMERIC_WITHOUT_DOT).to_string();
-
     if let Ok(res) = moz_autoconfigure(
         ctx,
         &format!(
             "https://autoconfig.{param_domain}/mail/config-v1.1.xml?emailaddress={param_addr_urlencoded}"
         ),
-        &param.addr,
+        param,
     )
     .await
     {
@@ -529,7 +540,7 @@ async fn get_autoconfig(
             "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
             &param_domain, &param_addr_urlencoded
         ),
-        &param.addr,
+        param,
     )
     .await
     {
@@ -565,7 +576,7 @@ async fn get_autoconfig(
     if let Ok(res) = moz_autoconfigure(
         ctx,
         &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
-        &param.addr,
+        param,
     )
     .await
     {
@@ -575,19 +586,140 @@ async fn get_autoconfig(
     None
 }
 
-async fn nicer_configuration_error(context: &Context, e: String) -> String {
-    if e.to_lowercase().contains("could not resolve")
-        || e.to_lowercase().contains("connection attempts")
-        || e.to_lowercase()
-            .contains("temporary failure in name resolution")
-        || e.to_lowercase().contains("name or service not known")
-        || e.to_lowercase()
-            .contains("failed to lookup address information")
+async fn try_imap_one_param(
+    context: &Context,
+    param: &ServerLoginParam,
+    socks5_config: &Option<Socks5Config>,
+    addr: &str,
+    provider_strict_tls: bool,
+) -> Result<(Imap, ImapSession), ConfigurationError> {
+    let inf = format!(
+        "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
+        param.user,
+        param.server,
+        param.port,
+        param.security,
+        param.certificate_checks,
+        param.oauth2,
+        if let Some(socks5_config) = socks5_config {
+            socks5_config.to_string()
+        } else {
+            "None".to_string()
+        }
+    );
+    info!(context, "Trying: {}", inf);
+
+    let (_s, r) = async_channel::bounded(1);
+
+    let mut imap = match Imap::new(param, socks5_config.clone(), addr, provider_strict_tls, r) {
+        Err(err) => {
+            info!(context, "failure: {:#}", err);
+            return Err(ConfigurationError {
+                config: inf,
+                msg: format!("{err:#}"),
+            });
+        }
+        Ok(imap) => imap,
+    };
+
+    match imap.connect(context).await {
+        Err(err) => {
+            info!(context, "failure: {:#}", err);
+            Err(ConfigurationError {
+                config: inf,
+                msg: format!("{err:#}"),
+            })
+        }
+        Ok(session) => {
+            info!(context, "success: {}", inf);
+            Ok((imap, session))
+        }
+    }
+}
+
+async fn try_smtp_one_param(
+    context: &Context,
+    param: &ServerLoginParam,
+    socks5_config: &Option<Socks5Config>,
+    addr: &str,
+    provider_strict_tls: bool,
+    smtp: &mut Smtp,
+) -> Result<(), ConfigurationError> {
+    let inf = format!(
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
+        param.user,
+        param.server,
+        param.port,
+        param.security,
+        param.certificate_checks,
+        param.oauth2,
+        if let Some(socks5_config) = socks5_config {
+            socks5_config.to_string()
+        } else {
+            "None".to_string()
+        }
+    );
+    info!(context, "Trying: {}", inf);
+
+    if let Err(err) = smtp
+        .connect(context, param, socks5_config, addr, provider_strict_tls)
+        .await
     {
+        info!(context, "failure: {}", err);
+        Err(ConfigurationError {
+            config: inf,
+            msg: format!("{err:#}"),
+        })
+    } else {
+        info!(context, "success: {}", inf);
+        smtp.disconnect();
+        Ok(())
+    }
+}
+
+/// Failure to connect and login with email client configuration.
+#[derive(Debug, thiserror::Error)]
+#[error("Trying {config}…\nError: {msg}")]
+pub struct ConfigurationError {
+    /// Tried configuration description.
+    config: String,
+
+    /// Error message.
+    msg: String,
+}
+
+async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
+    let first_err = if let Some(f) = errors.first() {
+        f
+    } else {
+        // This means configuration failed but no errors have been captured. This should never
+        // happen, but if it does, the user will see classic "Error: no error".
+        return "no error".to_string();
+    };
+
+    if errors.iter().all(|e| {
+        e.msg.to_lowercase().contains("could not resolve")
+            || e.msg.to_lowercase().contains("no dns resolution results")
+            || e.msg
+                .to_lowercase()
+                .contains("temporary failure in name resolution")
+            || e.msg.to_lowercase().contains("name or service not known")
+            || e.msg
+                .to_lowercase()
+                .contains("failed to lookup address information")
+    }) {
         return stock_str::error_no_network(context).await;
     }
 
-    e
+    if errors.iter().all(|e| e.msg == first_err.msg) {
+        return first_err.msg.to_string();
+    }
+
+    errors
+        .iter()
+        .map(|e| e.to_string())
+        .collect::<Vec<String>>()
+        .join("\n\n")
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -597,7 +729,7 @@ pub enum Error {
 
     #[error("XML error at position {position}: {error}")]
     InvalidXml {
-        position: u64,
+        position: usize,
         #[source]
         error: quick_xml::Error,
     },
@@ -611,9 +743,9 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    #![allow(clippy::indexing_slicing)]
+
     use crate::config::Config;
-    use crate::login_param::EnteredServerLoginParam;
     use crate::test_utils::TestContext;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -625,24 +757,4 @@ mod tests {
         t.set_config(Config::MailPw, Some("123456")).await.unwrap();
         assert!(t.configure().await.is_err());
     }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_configured_param() -> Result<()> {
-        let t = &TestContext::new().await;
-        let entered_param = EnteredLoginParam {
-            addr: "alice@example.org".to_string(),
-
-            imap: EnteredServerLoginParam {
-                user: "alice@example.net".to_string(),
-                password: "foobar".to_string(),
-                ..Default::default()
-            },
-
-            ..Default::default()
-        };
-        let configured_param = get_configured_param(t, &entered_param).await?;
-        assert_eq!(configured_param.imap_user, "alice@example.net");
-        assert_eq!(configured_param.smtp_user, "");
-        Ok(())
-    }
 }

src/configure/auto_mozilla.rs

@@ -9,6 +9,7 @@ use quick_xml::events::{BytesStart, Event};
 
 use super::{Error, ServerParams};
 use crate::context::Context;
+use crate::login_param::LoginParam;
 use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
@@ -79,7 +80,7 @@ fn parse_server<B: BufRead>(
         })
         .map(|typ| {
             typ.unwrap()
-                .decode_and_unescape_value(reader.decoder())
+                .decode_and_unescape_value(reader)
                 .unwrap_or_default()
                 .to_lowercase()
         })
@@ -190,7 +191,7 @@ fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoco
     };
 
     let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.config_mut().trim_text(true);
+    reader.trim_text(true);
 
     let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
         position: reader.buffer_position(),
@@ -247,6 +248,7 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
                 hostname: server.hostname,
                 port: server.port,
                 username: server.username,
+                strict_tls: None,
             })
         })
         .collect();
@@ -256,11 +258,11 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
-    addr: &str,
+    param_in: &LoginParam,
 ) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_serverparams(addr, &xml_raw);
+    let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,
@@ -272,6 +274,8 @@ pub(crate) async fn moz_autoconfigure(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]