chore(release): prepare for 1.150.0

`receive_imf() is only used in tests and the REPL, which enables the
"internals" feature. This PR marks it as such, so that it's clear not
only from the comment that this function is not used for anything else.

.github/dependabot.yml

@@ -7,3 +7,10 @@ updates:
     commit-message:
       prefix: "chore(cargo)"
     open-pull-requests-limit: 50
+
+  # Keep GitHub Actions up to date.
+  # <https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot>
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.78.0
+      RUSTUP_TOOLCHAIN: 1.82.0
     steps:
       - uses: actions/checkout@v4
         with:
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           show-progress: false
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - uses: EmbarkStudios/cargo-deny-action@v2
         with:
           arguments: --all-features --workspace
           command: check
@@ -95,11 +95,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.78.0
+            rust: 1.82.0
           - os: windows-latest
-            rust: 1.78.0
+            rust: 1.82.0
           - os: macos-latest
-            rust: 1.78.0
+            rust: 1.82.0
 
           # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest
@@ -125,9 +125,6 @@ jobs:
       - name: Tests
         env:
           RUST_BACKTRACE: 1
-
-          # Workaround for <https://github.com/nextest-rs/nextest/issues/1493>.
-          RUSTUP_WINDOWS_PATH_ADD_BIN: 1
         run: cargo nextest run --workspace
 
       - name: Doc-Tests
@@ -214,9 +211,9 @@ jobs:
         include:
           # Currently used Rust version.
           - os: ubuntu-latest
-            python: 3.12
+            python: 3.13
           - os: macos-latest
-            python: 3.12
+            python: 3.13
 
           # PyPy tests
           - os: ubuntu-latest
@@ -252,7 +249,7 @@ jobs:
 
       - name: Run python tests
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
           DCC_RS_TARGET: debug
           DCC_RS_DEV: ${{ github.workspace }}
         working-directory: python
@@ -266,11 +263,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            python: 3.12
+            python: 3.13
           - os: macos-latest
-            python: 3.12
+            python: 3.13
           - os: windows-latest
-            python: 3.12
+            python: 3.13
 
           # PyPy tests
           - os: ubuntu-latest
@@ -317,6 +314,6 @@ jobs:
 
       - name: Run deltachat-rpc-client tests
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
         working-directory: deltachat-rpc-client
         run: tox -e py

.github/workflows/deltachat-rpc-server.yml

@@ -401,6 +401,6 @@ jobs:
         working-directory: deltachat-rpc-server/npm-package
         run: |
           ls -lah platform_package
-          for platform in *.tgz; do npm publish --provenance "$platform"; done
+          for platform in *.tgz; do npm publish --provenance "$platform" --access public; done
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/dependabot.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
+        uses: dependabot/fetch-metadata@v2.2.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Approve a PR

.github/workflows/jsonrpc-client-npm-package.yml

@@ -33,6 +33,6 @@ jobs:
 
       - name: Publish
         working-directory: deltachat-jsonrpc/typescript
-        run: npm publish --provenance deltachat-jsonrpc-client-*
+        run: npm publish --provenance deltachat-jsonrpc-client-* --access public
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/jsonrpc.yml

@@ -33,7 +33,7 @@ jobs:
         working-directory: deltachat-jsonrpc/typescript
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
       - name: make sure websocket server version still builds
         working-directory: deltachat-jsonrpc
         run: cargo build --bin deltachat-jsonrpc-server --features webserver

.github/workflows/nix.yml

@@ -0,0 +1,22 @@
+name: Test Nix flake
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  format:
+    name: check flake formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix fmt
+
+      # Check that formatting does not change anything.
+      - run: git diff --exit-code

.github/workflows/node-docs.yml

@@ -31,7 +31,7 @@ jobs:
           mv docs js
 
       - name: Upload
-        uses: horochx/deploy-via-scp@v1.0.1
+        uses: horochx/deploy-via-scp@1.1.0
         with:
           user: ${{ secrets.USERNAME }}
           key: ${{ secrets.KEY }}

.github/workflows/node-tests.yml

@@ -64,5 +64,5 @@ jobs:
         working-directory: node
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
           NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

.github/workflows/upload-docs.yml

@@ -74,7 +74,7 @@ jobs:
           show-progress: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - name: Use Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version: '18'
       - name: npm install

.gitignore

@@ -50,4 +50,4 @@ result
 
 # direnv
 .envrc
-.direnv
+.direnv

CMakeLists.txt

@@ -27,7 +27,7 @@ add_custom_command(
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release --no-default-features --features jsonrpc
+	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release --features jsonrpc
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
 )
 

CONTRIBUTING.md

@@ -32,6 +32,66 @@ on the contributing page: <https://github.com/deltachat/deltachat-core-rust/cont
 We format the code using `rustfmt`. Run `cargo fmt` prior to committing the code.
 Run `scripts/clippy.sh` to check the code for common mistakes with [Clippy].
 
+### SQL
+
+Multi-line SQL statements should be formatted using string literals,
+for example
+```
+    sql.execute(
+        "CREATE TABLE messages (
+id INTEGER PRIMARY KEY AUTOINCREMENT,
+text TEXT DEFAULT '' NOT NULL -- message text
+) STRICT",
+    )
+    .await?;
+```
+
+Do not use macros like [`concat!`](https://doc.rust-lang.org/std/macro.concat.html)
+or [`indoc!](https://docs.rs/indoc).
+Do not escape newlines like this:
+```
+    sql.execute(
+        "CREATE TABLE messages ( \
+id INTEGER PRIMARY KEY AUTOINCREMENT, \
+text TEXT DEFAULT '' NOT NULL \
+) STRICT",
+    )
+    .await?;
+```
+Escaping newlines
+is prone to errors like this if space before backslash is missing:
+```
+"SELECT foo\
+ FROM bar"
+```
+Literal above results in `SELECT fooFROM bar` string.
+This style also does not allow using `--` comments.
+
+---
+
+Declare new SQL tables with [`STRICT`](https://sqlite.org/stricttables.html) keyword
+to make SQLite check column types.
+
+Declare primary keys with [`AUTOINCREMENT`](https://www.sqlite.org/autoinc.html) keyword.
+This avoids reuse of the row IDs and can avoid dangerous bugs
+like forwarding wrong message because the message was deleted
+and another message took its row ID.
+
+Declare all new columns as `NOT NULL`
+and set the `DEFAULT` value if it is optional so the column can be skipped in `INSERT` statements.
+Dealing with `NULL` values both in SQL and in Rust is tricky and we try to avoid it.
+If column is already declared without `NOT NULL`, use `IFNULL` function to provide default value when selecting it.
+Use `HAVING COUNT(*) > 0` clause
+to [prevent aggregate functions such as `MIN` and `MAX` from returning `NULL`](https://stackoverflow.com/questions/66527856/aggregate-functions-max-etc-return-null-instead-of-no-rows).
+
+Don't delete unused columns too early, but maybe after several months/releases, unused columns are
+still used by older versions, so deleting them breaks downgrading the core or importing a backup in
+an older version. Also don't change the column type, consider adding a new column with another name
+instead. Finally, never change column semantics, this is especially dangerous because the `STRICT`
+keyword doesn't help here.
+
+### Commit messages
+
 Commit messages follow the [Conventional Commits] notation.
 We use [git-cliff] to generate the changelog from commit messages before the release.
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.138.5"
+version = "1.150.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.77"
@@ -34,97 +34,93 @@ strip = true
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 deltachat-time = { path = "./deltachat-time" }
-deltachat-contact-tools = { path = "./deltachat-contact-tools" }
+deltachat-contact-tools = { workspace = true }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
-async-broadcast = "0.7.0"
-async-channel = "2.2.1"
-async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
+async-broadcast = "0.7.1"
+async-channel = { workspace = true }
+async-imap = { version = "0.10.2", default-features = false, features = ["runtime-tokio", "compress"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
-async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
-backtrace = "0.3"
-base64 = "0.22"
-brotli = { version = "6", default-features=false, features = ["std"] }
-chrono = { workspace = true }
+async_zip = { version = "0.0.17", default-features = false, features = ["deflate", "tokio-fs"] }
+base64 = { workspace = true }
+brotli = { version = "7", default-features=false, features = ["std"] }
+bytes = "1"
+chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 fast-socks5 = "0.9"
 fd-lock = "4"
-futures = "0.3"
-futures-lite = "2.3.0"
+futures-lite = { workspace = true }
+futures = { workspace = true }
 hex = "0.4.0"
-hickory-resolver = "0.24"
+hickory-resolver = "=0.25.0-alpha.2"
+http-body-util = "0.1.2"
 humansize = "2"
-image = { version = "0.25.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh_old = { version = "0.4.2", default-features = false, package = "iroh"}
-iroh-net = "0.16.2"
-iroh-gossip = { version = "0.16.2", features = ["net"] }
-quinn = "0.10.0"
-kamadak-exif = "0.5.3"
+hyper = "1"
+hyper-util = "0.1.10"
+image = { version = "0.25.4", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh-gossip = { version = "0.28.1", default-features = false, features = ["net"] }
+iroh-net = { version = "0.28.1", default-features = false }
+kamadak-exif = "0.6.0"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
+libc = { workspace = true }
 mailparse = "0.15"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
-num-traits = "0.2"
+num-traits = { workspace = true }
 once_cell = { workspace = true }
-percent-encoding = "2.3"
 parking_lot = "0.12"
-pgp = { version = "0.11", default-features = false }
-pretty_env_logger = { version = "0.5", optional = true }
+percent-encoding = "2.3"
+pgp = { version = "0.14.0", default-features = false }
+pin-project = "1"
 qrcodegen = "1.7.0"
-quick-xml = "0.31"
+quick-xml = "0.37"
 quoted_printable = "0.5"
-rand = "0.8"
+rand = { workspace = true }
 regex = { workspace = true }
-reqwest = { version = "0.11.27", features = ["json"] }
 rusqlite = { workspace = true, features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-sanitize-filename = "0.5"
-serde_json = "1"
-serde = { version = "1.0", features = ["derive"] }
+rustls-pki-types = "1.10.0"
+rustls = { version = "0.23.14", default-features = false }
+sanitize-filename = { workspace = true }
+serde_json = { workspace = true }
+serde_urlencoded = "0.7.1"
+serde = { workspace = true, features = ["derive"] }
 sha-1 = "0.10"
-sha2 = "0.10"
+shadowsocks = { version = "1.21.0", default-features = false, features = ["aead-cipher-2022"] }
 smallvec = "1.13.2"
 strum = "0.26"
 strum_macros = "0.26"
 tagger = "4.3.4"
 textwrap = "0.16.1"
-thiserror = "1"
-tokio = { version = "1.37.0", features = ["fs", "rt-multi-thread", "macros"] }
+thiserror = { workspace = true }
 tokio-io-timeout = "1.2.0"
-tokio-stream = { version = "0.1.15", features = ["fs"] }
+tokio-rustls = { version = "0.26.0", default-features = false }
+tokio-stream = { version = "0.1.16", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
-tokio-util = "0.7.9"
+tokio-util = { workspace = true }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
-
-# Pin OpenSSL to 3.1 releases.
-# OpenSSL 3.2 has a regression tracked at <https://github.com/openssl/openssl/issues/23376>
-# which results in broken `deltachat-rpc-server` binaries when cross-compiled using Zig toolchain.
-# See <https://github.com/deltachat/deltachat-core-rust/issues/5206> for Delta Chat issue.
-# According to <https://www.openssl.org/policies/releasestrat.html>
-# 3.1 branch will be supported until 2025-03-14.
-openssl-src = "~300.1"
+webpki-roots = "0.26.6"
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
+anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = "2.3.0"
-log = "0.4"
-pretty_env_logger = "0.5"
+futures-lite = { workspace = true }
+log = { workspace = true }
+nu-ansi-term = { workspace = true }
+pretty_assertions = "1.4.1"
 proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+tempfile = { workspace = true }
 testdir = "0.9.0"
-tokio = { version = "1.37.0", features = ["parking_lot", "rt-multi-thread", "macros"] }
-pretty_assertions = "1.3.0"
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 
 [workspace]
 members = [
@@ -169,16 +165,38 @@ harness = false
 
 [workspace.dependencies]
 anyhow = "1"
-once_cell = "1.18.0"
+async-channel = "2.3.1"
+base64 = "0.22"
+chrono = { version = "0.4.38", default-features = false }
+deltachat-contact-tools = { path = "deltachat-contact-tools" }
+deltachat-jsonrpc = { path = "deltachat-jsonrpc", default-features = false }
+deltachat = { path = ".", default-features = false }
+futures = "0.3.31"
+futures-lite = "2.4.0"
+libc = "0.2"
+log = "0.4"
+nu-ansi-term = "0.46"
+num-traits = "0.2"
+once_cell = "1.20.2"
+rand = "0.8"
 regex = "1.10"
-rusqlite = "0.31"
-chrono = { version = "0.4.38", default-features=false, features = ["alloc", "clock", "std"] }
+rusqlite = "0.32"
+sanitize-filename = "0.5"
+serde = "1.0"
+serde_json = "1"
+tempfile = "3.13.0"
+thiserror = "1"
+tokio = "1"
+tokio-util = "0.7.11"
+tracing-subscriber = "0.3"
+yerpc = "0.6.2"
 
 [features]
 default = ["vendored"]
 internals = []
 vendored = [
-  "async-native-tls/vendored",
-  "rusqlite/bundled-sqlcipher-vendored-openssl",
-  "reqwest/native-tls-vendored"
-]
+  "rusqlite/bundled-sqlcipher-vendored-openssl"
+]
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] }

README.md

@@ -30,13 +30,13 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run --locked -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Optionally, install `deltachat-repl` binary with
 ```
-$ cargo install --path deltachat-repl/
+$ cargo install --locked --path deltachat-repl/
 ```
 and run as
 ```

assets/qr_overlay_delta.svg-part

@@ -0,0 +1,12 @@
+<path
+   style="fill:#ffffff;fill-opacity:1;stroke:none"
+   d="m 24.015419,1.2870249 c -12.549421,0 -22.7283936,10.1789711 -22.7283936,22.7283931 0,12.549422 10.1789726,22.728395 22.7283936,22.728395 14.337742,-0.342877 9.614352,-4.702705 23.697556,0.969161 -7.545453,-13.001555 -1.082973,-13.32964 -0.969161,-23.697556 0,-12.549422 -10.178973,-22.7283931 -22.728395,-22.7283931 z" />
+<path
+   style="fill:#000000;fill-opacity:1;stroke:none"
+   d="M 23.982249,5.3106163 C 13.645822,5.4364005 5.2618355,13.92999 5.2618355,24.275753 c 0,10.345764 8.3839865,18.635301 18.7204135,18.509516 9.827724,-0.03951 7.516769,-5.489695 18.380082,-0.443187 -5.950849,-9.296115 0.201753,-10.533667 0.340336,-18.521947 0,-10.345766 -8.383989,-18.6353031 -18.720418,-18.5095187 z" />
+<g
+    style="fill:#ffffff"
+    transform="scale(1.1342891,0.88160947)">
+  <path
+     d="m 21.360141,23.513382 q -1.218487,-1.364705 -3.387392,-3.265543 -2.388233,-2.095797 -3.216804,-3.289913 -0.828571,-1.218486 -0.828571,-2.6563 0,-2.144536 1.998318,-3.363022 1.998317,-1.2428565 5.215121,-1.2428565 3.216804,0 5.605037,1.0966375 2.412603,1.096638 2.412603,3.021846 0,0.92605 -0.584873,1.535293 -0.584874,0.609243 -1.364705,0.609243 -1.121008,0 -2.631931,-1.681511 -1.535292,-1.705881 -2.60756,-2.388233 -1.047898,-0.706722 -2.461343,-0.706722 -1.803359,0 -2.973106,0.804201 -1.145377,0.804201 -1.145377,2.047057 0,1.169747 0.950419,2.193275 0.950419,1.023529 4.898315,3.728568 4.215963,2.899998 5.946213,4.532769 1.75462,1.632772 2.851258,3.972265 1.096638,2.339494 1.096638,4.947055 0,4.581508 -3.241174,8.090749 -3.216804,3.484871 -7.530245,3.484871 -3.923526,0 -6.628566,-2.802519 -2.705039,-2.802518 -2.705039,-7.481506 0,-4.508399 2.973106,-7.530245 2.997477,-3.021846 7.359658,-3.655459 z m 1.072268,1.121008 q -6.994112,1.145377 -6.994112,9.601672 0,4.36218 1.730251,6.774783 1.75462,2.412603 4.069744,2.412603 2.412603,0 3.972265,-2.315124 1.559663,-2.339493 1.559663,-6.311759 0,-5.751255 -4.337811,-10.162175 z" />
+</g>

deltachat-contact-tools/Cargo.toml

@@ -12,7 +12,7 @@ anyhow = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true } # Needed in order to `impl rusqlite::types::ToSql for EmailAddress`. Could easily be put behind a feature.
-chrono = { workspace = true }
+chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.

deltachat-contact-tools/src/lib.rs

@@ -15,14 +15,16 @@
     clippy::explicit_into_iter_loop,
     clippy::cloned_instead_of_copied
 )]
-#![cfg_attr(not(test), warn(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 #![allow(
     clippy::match_bool,
     clippy::mixed_read_write_in_expression,
     clippy::bool_assert_comparison,
     clippy::manual_split_once,
     clippy::format_push_string,
-    clippy::bool_to_int_with_if
+    clippy::bool_to_int_with_if,
+    clippy::manual_range_contains
 )]
 
 use std::fmt;
@@ -35,10 +37,6 @@ use chrono::{DateTime, NaiveDateTime};
 use once_cell::sync::Lazy;
 use regex::Regex;
 
-// TODOs to clean up:
-// - Check if sanitizing is done correctly everywhere
-// - Apply lints everywhere (https://doc.rust-lang.org/cargo/reference/workspaces.html#the-lints-table)
-
 #[derive(Debug)]
 /// A Contact, as represented in a VCard.
 pub struct VcardContact {
@@ -115,7 +113,9 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
         // If `s` is `EMAIL;TYPE=work:alice@example.com` and `property` is `EMAIL`,
         // then `remainder` is now `;TYPE=work:alice@example.com`
 
-        // TODO this doesn't handle the case where there are quotes around a colon
+        // Note: This doesn't handle the case where there are quotes around a colon,
+        // like `NAME;Foo="Some quoted text: that contains a colon":value`.
+        // This could be improved in the future, but for now, the parsing is good enough.
         let (params, value) = remainder.split_once(':')?;
         // In the example from above, `params` is now `;TYPE=work`
         // and `value` is now `alice@example.com`
@@ -155,7 +155,7 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
     }
 
     // Remove line folding, see https://datatracker.ietf.org/doc/html/rfc6350#section-3.2
-    static NEWLINE_AND_SPACE_OR_TAB: Lazy<Regex> = Lazy::new(|| Regex::new("\n[\t ]").unwrap());
+    static NEWLINE_AND_SPACE_OR_TAB: Lazy<Regex> = Lazy::new(|| Regex::new("\r?\n[\t ]").unwrap());
     let unfolded_lines = NEWLINE_AND_SPACE_OR_TAB.replace_all(vcard, "");
 
     let mut lines = unfolded_lines.lines().peekable();
@@ -175,7 +175,15 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
         let mut photo = None;
         let mut datetime = None;
 
-        for line in lines.by_ref() {
+        for mut line in lines.by_ref() {
+            if let Some(remainder) = remove_prefix(line, "item1.") {
+                // Remove the group name, if the group is called "item1".
+                // If necessary, we can improve this to also remove groups that are called something different that "item1".
+                //
+                // Search "group name" at https://datatracker.ietf.org/doc/html/rfc6350 for more infos.
+                line = remainder;
+            }
+
             if let Some(email) = vcard_property(line, "email") {
                 addr.get_or_insert(email);
             } else if let Some(name) = vcard_property(line, "fn") {
@@ -183,6 +191,7 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
             } else if let Some(k) = remove_prefix(line, "KEY;PGP;ENCODING=BASE64:")
                 .or_else(|| remove_prefix(line, "KEY;TYPE=PGP;ENCODING=b:"))
                 .or_else(|| remove_prefix(line, "KEY:data:application/pgp-keys;base64,"))
+                .or_else(|| remove_prefix(line, "KEY;PREF=1:data:application/pgp-keys;base64,"))
             {
                 key.get_or_insert(k);
             } else if let Some(p) = remove_prefix(line, "PHOTO;JPEG;ENCODING=BASE64:")
@@ -263,27 +272,27 @@ impl rusqlite::types::ToSql for ContactAddress {
     }
 }
 
-/// Make the name and address
+/// Takes a name and an address and sanitizes them:
+/// - Extracts a name from the addr if the addr is in form "Alice <alice@example.org>"
+/// - Removes special characters from the name, see [`sanitize_name()`]
+/// - Removes the name if it is equal to the address by setting it to ""
 pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
     let (name, addr) = if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.is_empty() {
-                strip_rtlo_characters(captures.get(1).map_or("", |m| m.as_str()))
+                captures.get(1).map_or("", |m| m.as_str())
             } else {
-                strip_rtlo_characters(name)
+                name
             },
             captures
                 .get(2)
                 .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
-        (
-            strip_rtlo_characters(&normalize_name(name)),
-            addr.to_string(),
-        )
+        (name, addr.to_string())
     };
-    let mut name = normalize_name(&name);
+    let mut name = sanitize_name(name);
 
     // If the 'display name' is just the address, remove it:
     // Otherwise, the contact would sometimes be shown as "alice@example.com (alice@example.com)" (see `get_name_n_addr()`).
@@ -295,31 +304,77 @@ pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     (name, addr)
 }
 
-/// Normalize a name.
+/// Sanitizes a name.
 ///
-/// - Remove quotes (come from some bad MUA implementations)
-/// - Trims the resulting string
-///
-/// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
-pub fn normalize_name(full_name: &str) -> String {
-    let full_name = full_name.trim();
-    if full_name.is_empty() {
-        return full_name.into();
-    }
+/// - Removes newlines and trims the string
+/// - Removes quotes (come from some bad MUA implementations)
+/// - Removes potentially-malicious bidi characters
+pub fn sanitize_name(name: &str) -> String {
+    let name = sanitize_single_line(name);
 
-    match full_name.as_bytes() {
-        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
-            .get(1..full_name.len() - 1)
+    match name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => name
+            .get(1..name.len() - 1)
             .map_or("".to_string(), |s| s.trim().to_string()),
-        _ => full_name.to_string(),
+        _ => name.to_string(),
     }
 }
 
+/// Sanitizes user input
+///
+/// - Removes newlines and trims the string
+/// - Removes potentially-malicious bidi characters
+pub fn sanitize_single_line(input: &str) -> String {
+    sanitize_bidi_characters(input.replace(['\n', '\r'], " ").trim())
+}
+
 const RTLO_CHARACTERS: [char; 5] = ['\u{202A}', '\u{202B}', '\u{202C}', '\u{202D}', '\u{202E}'];
-/// This method strips all occurrences of the RTLO Unicode character.
-/// [Why is this needed](https://github.com/deltachat/deltachat-core-rust/issues/3479)?
-pub fn strip_rtlo_characters(input_str: &str) -> String {
-    input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "")
+const ISOLATE_CHARACTERS: [char; 3] = ['\u{2066}', '\u{2067}', '\u{2068}'];
+const POP_ISOLATE_CHARACTER: char = '\u{2069}';
+/// Some control unicode characters can influence whether adjacent text is shown from
+/// left to right or from right to left.
+///
+/// Since user input is not supposed to influence how adjacent text looks,
+/// this function removes some of these characters.
+///
+/// Also see https://github.com/deltachat/deltachat-core-rust/issues/3479.
+pub fn sanitize_bidi_characters(input_str: &str) -> String {
+    // RTLO_CHARACTERS are apparently rarely used in practice.
+    // They can impact all following text, so, better remove them all:
+    let input_str = input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "");
+
+    // If the ISOLATE characters are not ended with a POP DIRECTIONAL ISOLATE character,
+    // we regard the input as potentially malicious and simply remove all ISOLATE characters.
+    // See https://en.wikipedia.org/wiki/Bidirectional_text#Unicode_bidi_support
+    // and https://www.w3.org/International/questions/qa-bidi-unicode-controls.en
+    // for an explanation about ISOLATE characters.
+    fn isolate_characters_are_valid(input_str: &str) -> bool {
+        let mut isolate_character_nesting: i32 = 0;
+        for char in input_str.chars() {
+            if ISOLATE_CHARACTERS.contains(&char) {
+                isolate_character_nesting += 1;
+            } else if char == POP_ISOLATE_CHARACTER {
+                isolate_character_nesting -= 1;
+            }
+
+            // According to Wikipedia, 125 levels are allowed:
+            // https://en.wikipedia.org/wiki/Unicode_control_characters
+            // (although, in practice, we could also significantly lower this number)
+            if isolate_character_nesting < 0 || isolate_character_nesting > 125 {
+                return false;
+            }
+        }
+        isolate_character_nesting == 0
+    }
+
+    if isolate_characters_are_valid(&input_str) {
+        input_str
+    } else {
+        input_str.replace(
+            |char| ISOLATE_CHARACTERS.contains(&char) || POP_ISOLATE_CHARACTER == char,
+            "",
+        )
+    }
 }
 
 /// Returns false if addr is an invalid address, otherwise true.
@@ -643,10 +698,10 @@ END:VCARD
     }
 
     #[test]
-    fn test_android_vcard_with_base64_avatar() {
-        // This is not an actual base64-encoded avatar, it's just to test the parsing
-        let contacts = parse_vcard(
-            "BEGIN:VCARD
+    fn test_vcard_with_base64_avatar() {
+        // This is not an actual base64-encoded avatar, it's just to test the parsing.
+        // This one is Android-like.
+        let vcard0 = "BEGIN:VCARD
 VERSION:2.1
 N:;Bob;;;
 FN:Bob
@@ -656,13 +711,101 @@ PHOTO;ENCODING=BASE64;JPEG:/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEU
  L8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==
 
 END:VCARD
-",
+";
+        // This one is DOS-like.
+        let vcard1 = vcard0.replace('\n', "\r\n");
+        for vcard in [vcard0, vcard1.as_str()] {
+            let contacts = parse_vcard(vcard);
+            assert_eq!(contacts.len(), 1);
+            assert_eq!(contacts[0].addr, "bob@example.org".to_string());
+            assert_eq!(contacts[0].authname, "Bob".to_string());
+            assert_eq!(contacts[0].key, None);
+            assert_eq!(contacts[0].profile_image.as_deref().unwrap(), "/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAL8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==");
+        }
+    }
+
+    #[test]
+    fn test_protonmail_vcard() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN;PREF=1:Alice Wonderland
+UID:proton-web-03747582-328d-38dc-5ddd-000000000000
+ITEM1.EMAIL;PREF=1:alice@example.org
+ITEM1.KEY;PREF=1:data:application/pgp-keys;base64,aaaaaaaaaaaaaaaaaaaaaaaaa
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ITEM1.KEY;PREF=2:data:application/pgp-keys;base64,bbbbbbbbbbbbbbbbbbbbbbbbb
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+ITEM1.X-PM-ENCRYPT:true
+ITEM1.X-PM-SIGN:true
+END:VCARD",
         );
 
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts[0].addr, "bob@example.org".to_string());
-        assert_eq!(contacts[0].authname, "Bob".to_string());
-        assert_eq!(contacts[0].key, None);
-        assert_eq!(contacts[0].profile_image.as_deref().unwrap(), "/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAL8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==");
+        assert_eq!(&contacts[0].addr, "alice@example.org");
+        assert_eq!(&contacts[0].authname, "Alice Wonderland");
+        assert_eq!(contacts[0].key.as_ref().unwrap(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+        assert!(contacts[0].timestamp.is_err());
+        assert_eq!(contacts[0].profile_image, None);
+    }
+
+    #[test]
+    fn test_sanitize_name() {
+        assert_eq!(&sanitize_name(" hello world   "), "hello world");
+        assert_eq!(&sanitize_name("<"), "<");
+        assert_eq!(&sanitize_name(">"), ">");
+        assert_eq!(&sanitize_name("'"), "'");
+        assert_eq!(&sanitize_name("\""), "\"");
+    }
+
+    #[test]
+    fn test_sanitize_single_line() {
+        assert_eq!(sanitize_single_line("Hi\naiae "), "Hi aiae");
+        assert_eq!(sanitize_single_line("\r\nahte\n\r"), "ahte");
+    }
+
+    #[test]
+    fn test_sanitize_bidi_characters() {
+        // Legit inputs:
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat\u{2069}"),
+            "Tes\u{2067}ting Delta Chat\u{2069}"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"),
+            "Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"),
+            "Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"
+        );
+
+        // Potentially-malicious inputs:
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{202C}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Testing Delta Chat\u{2069}"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2069}ting Delta Chat\u{2067}"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2068}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
     }
 }

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.138.5"
+version = "1.150.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -14,21 +14,21 @@ name = "deltachat"
 crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
-deltachat = { path = "../", default-features = false }
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
-libc = "0.2"
+deltachat = { workspace = true, default-features = false }
+deltachat-jsonrpc = { workspace = true, optional = true }
+libc = { workspace = true }
 human-panic = { version = "2", default-features = false }
-num-traits = "0.2"
-serde_json = "1.0"
-tokio = { version = "1.37.0", features = ["rt-multi-thread"] }
-anyhow = "1"
-thiserror = "1"
-rand = "0.8"
-once_cell = "1.18.0"
-yerpc = { version = "0.5.1", features = ["anyhow_expose"] }
+num-traits = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+anyhow = { workspace = true }
+thiserror = { workspace = true }
+rand = { workspace = true }
+once_cell = { workspace = true }
+yerpc = { workspace = true, features = ["anyhow_expose"] }
 
 [features]
 default = ["vendored"]
-vendored = ["deltachat/vendored"]
+vendored = ["deltachat/vendored", "deltachat-jsonrpc/vendored"]
 jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/deltachat.h

@@ -403,13 +403,10 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_port`    = SMTP-port, guessed if left out
  * - `send_security`= SMTP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
- * - `socks5_enabled` = SOCKS5 enabled
- * - `socks5_host` = SOCKS5 proxy server host
- * - `socks5_port` = SOCKS5 proxy server port
- * - `socks5_user` = SOCKS5 proxy username
- * - `socks5_password` = SOCKS5 proxy password
+ * - `proxy_enabled` = Proxy enabled. Disabled by default.
+ * - `proxy_url` = Proxy URL. May contain multiple URLs separated by newline, but only the first one is used.
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
- * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
+ * - `smtp_certificate_checks` = deprecated option, should be set to the same value as `imap_certificate_checks` but ignored by the new core
  * - `displayname`  = Own name to use when sending messages. MUAs are allowed to spread this way e.g. using CC, defaults to empty
  * - `selfstatus`   = Own status to display, e.g. in e-mail footers, defaults to empty
  * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
@@ -420,9 +417,10 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    and also recoded to a reasonable size.
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
- *                    1=send and request read receipts (default)
- * - `bcc_self`     = 0=do not send a copy of outgoing messages to self (default),
- *                    1=send a copy of outgoing messages to self.
+ *                    1=send and request read receipts
+ *                    default=send and request read receipts, only send but not reuqest if `bot` is set
+ * - `bcc_self`     = 0=do not send a copy of outgoing messages to self,
+ *                    1=send a copy of outgoing messages to self (default).
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes,
@@ -481,8 +479,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `bot`          = Set to "1" if this is a bot.
  *                    Prevents adding the "Device messages" and "Saved messages" chats,
  *                    adds Auto-Submitted header to outgoing messages,
- *                    accepts contact requests automatically (calling dc_accept_chat() is not needed for bots)
- *                    and does not cut large incoming text messages.
+ *                    accepts contact requests automatically (calling dc_accept_chat() is not needed),
+ *                    does not cut large incoming text messages,
+ *                    handles existing messages the same way as new ones if `fetch_existing_msgs=1`.
  * - `last_msg_id` = database ID of the last message processed by the bot.
  *                   This ID and IDs below it are guaranteed not to be returned
  *                   by dc_get_next_msgs() and dc_wait_next_msgs().
@@ -493,8 +492,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                   For most bots calling `dc_markseen_msgs()` is the
  *                   recommended way to update this value
  *                   even for self-sent messages.
- * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
- *                    0=do not fetch existing messages on configure.
+ * - `fetch_existing_msgs` = 0=do not fetch existing messages on configure (default),
+ *                    1=fetch most recent existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
  * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
  *                    0=use IMAP IDLE if the server supports it.
@@ -507,6 +506,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
+ * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
+ *                    This is an experimental option not compatible to other MUAs
+ *                    and older Delta Chat versions.
+ *                    1 = enable.
+ *                    0 = disable (default).
  * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
  *                    seconds. 2 days by default.
  *                    This is not supposed to be changed by UIs and only used for testing.
@@ -518,11 +522,21 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    1=After the key changed, `dc_chat_can_send()` returns false and `dc_chat_is_protection_broken()` returns true
  *                    until `dc_accept_chat()` is called.
  * - `is_chatmail` = 1 if the the server is a chatmail server, 0 otherwise.
+ * - `is_muted`     = Whether a context is muted by the user.
+ *                    Muted contexts should not sound, vibrate or show notifications.
+ *                    In contrast to `dc_set_chat_mute_duration()`,
+ *                    fresh message and badge counters are not changed by this setting,
+ *                    but should be tuned down where appropriate.
+ * - `private_tag`  = Optional tag as "Work", "Family".
+ *                    Meant to help profile owner to differ between profiles with similar names.
  * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
  *                    The prefix should be followed by the system and maybe subsystem,
  *                    e.g. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
  *                    These keys go to backups and allow easy per-account settings when using @ref dc_accounts_t,
  *                    however, are not handled by the core otherwise.
+ * - `webxdc_realtime_enabled` = Whether the realtime APIs should be enabled.
+ *                               0 = WebXDC realtime API is disabled and behaves as noop.
+ *                               1 = WebXDC realtime API is enabled (default).
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -857,13 +871,10 @@ void            dc_maybe_network             (dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
- * @param addr The e-mail address of the user. This must match the
- *    configured_addr setting of the context as well as the UID of the key.
- * @param public_data Ignored, actual public key is extracted from secret_data.
  * @param secret_data ASCII armored secret key.
  * @return 1 on success, 0 on failure.
  */
-int             dc_preconfigure_keypair        (dc_context_t* context, const char *addr, const char *public_data, const char *secret_data);
+int             dc_preconfigure_keypair        (dc_context_t* context, const char *secret_data);
 
 
 // handle chatlists
@@ -1543,30 +1554,6 @@ void            dc_marknoticed_chat          (dc_context_t* context, uint32_t ch
 dc_array_t*     dc_get_chat_media            (dc_context_t* context, uint32_t chat_id, int msg_type, int msg_type2, int msg_type3);
 
 
-/**
- * Search next/previous message based on a given message and a list of types.
- * Typically used to implement the "next" and "previous" buttons
- * in a gallery or in a media player.
- *
- * @deprecated Deprecated 2023-10-03, use dc_get_chat_media() and navigate the returned array instead.
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @param msg_id The ID of the current message from which the next or previous message should be searched.
- * @param dir 1=get the next message, -1=get the previous one.
- * @param msg_type The message type to search for.
- *     If 0, the message type from curr_msg_id is used.
- * @param msg_type2 Alternative message type to search for. 0 to skip.
- * @param msg_type3 Alternative message type to search for. 0 to skip.
- * @return Returns the message ID that should be played next.
- *     The returned message is in the same chat as the given one
- *     and has one of the given types.
- *     Typically, this result is passed again to dc_get_next_media()
- *     later on the next swipe.
- *     If there is not next/previous message, the function returns 0.
- */
-uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t msg_id, int dir, int msg_type, int msg_type2, int msg_type3);
-
-
 /**
  * Set chat visibility to pinned, archived or normal.
  *
@@ -2495,7 +2482,9 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
 #define         DC_QR_BACKUP                 251
+#define         DC_QR_BACKUP2                252
 #define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
+#define         DC_QR_PROXY                  271 // text1=address (e.g. "127.0.0.1:9050")
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
 #define         DC_QR_URL                    332 // text1=URL
@@ -2541,6 +2530,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr() and then dc_configure().
  *
  * - DC_QR_BACKUP:
+ * - DC_QR_BACKUP2:
  *   ask the user if they want to set up a new device.
  *   If so, pass the qr-code to dc_receive_backup().
  *
@@ -2548,6 +2538,12 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   ask the user if they want to use the given service for video chats;
  *   if so, call dc_set_config_from_qr().
  *
+ * - DC_QR_PROXY with dc_lot_t::text1=address:
+ *   ask the user if they want to use the given proxy.
+ *   if so, call dc_set_config_from_qr() and restart I/O.
+ *   On success, dc_get_config(context, "proxy_url")
+ *   will contain the new proxy in normalized form as the first element.
+ *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
  *   e-mail address scanned, optionally, a draft message could be set in
  *   dc_lot_t::text1 in which case dc_lot_t::text1_meaning will be DC_TEXT1_DRAFT;
@@ -2622,6 +2618,7 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  * Get QR code image from the QR code text generated by dc_get_securejoin_qr().
  * See dc_get_securejoin_qr() for details about the contained QR code.
  *
+ * @deprecated 2024-10 use dc_create_qr_svg(dc_get_securejoin_qr()) instead.
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id group-chat-id for secure-join or 0 for setup-contact,
@@ -2802,6 +2799,22 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
 void        dc_delete_all_locations         (dc_context_t* context);
 
 
+// misc
+
+/**
+ * Create a QR code from any input data.
+ *
+ * The QR code is returned as a square SVG image.
+ *
+ * @memberof dc_context_t
+ * @param payload The content for the QR code.
+ * @return SVG image with the QR code.
+ *     On errors, an empty string is returned.
+ *     The returned string must be released using dc_str_unref() after usage.
+ */
+char*           dc_create_qr_svg             (const char* payload);
+
+
 /**
  * Get last error string.
  *
@@ -2890,6 +2903,7 @@ char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider);
  * This works like dc_backup_provider_qr() but returns the text of a rendered
  * SVG image containing the QR code.
  *
+ * @deprecated 2024-10 use dc_create_qr_svg(dc_backup_provider_get_qr()) instead.
  * @memberof dc_backup_provider_t
  * @param backup_provider The backup provider object as created by
  *    dc_backup_provider_new().
@@ -2929,7 +2943,7 @@ void dc_backup_provider_unref (dc_backup_provider_t* backup_provider);
  * Gets a backup offered by a dc_backup_provider_t object on another device.
  *
  * This function is called on a device that scanned the QR code offered by
- * dc_backup_sender_qr() or dc_backup_sender_qr_svg().  Typically this is a
+ * dc_backup_provider_get_qr().  Typically this is a
  * different device than that which provides the backup.
  *
  * This call will block while the backup is being transferred and only
@@ -5697,8 +5711,14 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_CERTCK_STRICT 1
 
 /**
- * Accept invalid certificates, including self-signed ones
- * or having incorrect hostname.
+ * Accept certificates that are expired, self-signed
+ * or not valid for the server hostname.
+ */
+#define DC_CERTCK_ACCEPT_INVALID 2
+
+/**
+ * For API compatibility only: Treat this as DC_CERTCK_ACCEPT_INVALID on reading.
+ * Must not be written.
  */
 #define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
 
@@ -6044,6 +6064,21 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_REACTIONS_CHANGED        2001
 
 
+/**
+ * A reaction to one's own sent message received.
+ * Typically, the UI will show a notification for that.
+ *
+ * In addition to this event, DC_EVENT_REACTIONS_CHANGED is emitted.
+ *
+ * @param data1 (int) contact_id ID of the contact sending this reaction.
+ * @param data2 (int) msg_id + (char*) reaction.
+ *      ID of the message for which a reaction was received in dc_event_get_data2_int(),
+ *      and the reaction as dc_event_get_data2_str().
+ *      string must be passed to dc_str_unref() afterwards.
+ */
+#define DC_EVENT_INCOMING_REACTION        2002
+
+
 /**
  * There is a fresh message. Typically, the user will show an notification
  * when receiving this message.
@@ -6261,7 +6296,7 @@ void dc_event_unref(dc_event_t* event);
 
 
 /**
- * webxdc status update received.
+ * Webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
  * `serial` set to the last known update
  * (in case of special bots, `status_update_serial` from `data2`
@@ -6284,6 +6319,27 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
 
+/**
+ * Data received over an ephemeral peer channel.
+ *
+ * @param data1 (int) msg_id
+ * @param data2 (int) + (char*) binary data.
+ *     length is returned as integer with dc_event_get_data2_int()
+ *     and binary data is returned as dc_event_get_data2_str().
+ *     Binary data must be passed to dc_str_unref() afterwards.
+ */
+
+#define DC_EVENT_WEBXDC_REALTIME_DATA             2150
+
+/**
+ * Advertisement for ephemeral peer channel communication received.
+ * This can be used by bots to initiate peer-to-peer communication from their side.
+ * @param data1 (int) msg_id
+ * @param data2 0
+ */
+
+#define DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT    2151
+
 /**
  * Tells that the Background fetch was completed (or timed out).
  *
@@ -6312,6 +6368,14 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_CHATLIST_ITEM_CHANGED         2301
 
+
+/**
+ * Inform that some events have been skipped due to event channel overflow.
+ *
+ * @param data1 (int) number of events that have been skipped
+ */
+#define DC_EVENT_CHANNEL_OVERFLOW              2400
+
 /**
  * @}
  */
@@ -6619,12 +6683,16 @@ void dc_event_unref(dc_event_t* event);
 /// "Message opened"
 ///
 /// Used in subjects of outgoing read receipts.
+///
+/// @deprecated Deprecated 2024-07-26
 #define DC_STR_READRCPT                   31
 
 /// "The message '%1$s' you sent was displayed on the screen of the recipient."
 ///
 /// Used as message text of outgoing read receipts.
 /// - %1$s will be replaced by the subject of the displayed message
+///
+/// @deprecated Deprecated 2024-06-23
 #define DC_STR_READRCPT_MAILBODY          32
 
 /// @deprecated Deprecated, this string is no longer needed.
@@ -7343,7 +7411,7 @@ void dc_event_unref(dc_event_t* event);
 /// Used as info message.
 #define DC_STR_SECUREJOIN_WAIT_TIMEOUT 191
 
-/// "Contact"
+/// "Contact". Deprecated, currently unused.
 #define DC_STR_CONTACT 200
 
 /**

deltachat-ffi/src/lib.rs

@@ -30,7 +30,7 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
 use deltachat::message::MsgId;
-use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
+use deltachat::qr_code_generator::{create_qr_svg, generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
@@ -541,6 +541,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::ErrorSelfNotInGroup(_) => 410,
         EventType::MsgsChanged { .. } => 2000,
         EventType::ReactionsChanged { .. } => 2001,
+        EventType::IncomingReaction { .. } => 2002,
         EventType::IncomingMsg { .. } => 2005,
         EventType::IncomingMsgBunch { .. } => 2006,
         EventType::MsgsNoticed { .. } => 2008,
@@ -563,9 +564,14 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
         EventType::WebxdcRealtimeData { .. } => 2150,
+        EventType::WebxdcRealtimeAdvertisementReceived { .. } => 2151,
         EventType::AccountsBackgroundFetchDone => 2200,
         EventType::ChatlistChanged => 2300,
         EventType::ChatlistItemChanged { .. } => 2301,
+        EventType::EventChannelOverflow { .. } => 2400,
+        #[allow(unreachable_patterns)]
+        #[cfg(test)]
+        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -596,6 +602,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_)
         | EventType::AccountsBackgroundFetchDone => 0,
         EventType::ChatlistChanged => 0,
+        EventType::IncomingReaction { contact_id, .. } => contact_id.to_u32() as libc::c_int,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -620,10 +627,15 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         }
         EventType::WebxdcRealtimeData { msg_id, .. }
         | EventType::WebxdcStatusUpdate { msg_id, .. }
+        | EventType::WebxdcRealtimeAdvertisementReceived { msg_id }
         | EventType::WebxdcInstanceDeleted { msg_id, .. } => msg_id.to_u32() as libc::c_int,
         EventType::ChatlistItemChanged { chat_id } => {
             chat_id.unwrap_or_default().to_u32() as libc::c_int
         }
+        EventType::EventChannelOverflow { n } => *n as libc::c_int,
+        #[allow(unreachable_patterns)]
+        #[cfg(test)]
+        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -658,15 +670,17 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::WebxdcRealtimeData { .. }
         | EventType::SelfavatarChanged
         | EventType::AccountsBackgroundFetchDone
         | EventType::ChatlistChanged
         | EventType::ChatlistItemChanged { .. }
-        | EventType::ConfigSynced { .. } => 0,
-        EventType::ChatModified(_) => 0,
+        | EventType::ConfigSynced { .. }
+        | EventType::ChatModified(_)
+        | EventType::WebxdcRealtimeAdvertisementReceived { .. }
+        | EventType::EventChannelOverflow { .. } => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
+        | EventType::IncomingReaction { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
         | EventType::MsgFailed { msg_id, .. }
@@ -679,6 +693,10 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
             status_update_serial,
             ..
         } => status_update_serial.to_u32() as libc::c_int,
+        EventType::WebxdcRealtimeData { data, .. } => data.len() as libc::c_int,
+        #[allow(unreachable_patterns)]
+        #[cfg(test)]
+        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -725,12 +743,13 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
         | EventType::WebxdcInstanceDeleted { .. }
-        | EventType::WebxdcRealtimeData { .. }
         | EventType::AccountsBackgroundFetchDone
         | EventType::ChatEphemeralTimerModified { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::ChatlistItemChanged { .. }
-        | EventType::ChatlistChanged => ptr::null_mut(),
+        | EventType::ChatlistChanged
+        | EventType::WebxdcRealtimeAdvertisementReceived { .. }
+        | EventType::EventChannelOverflow { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
                 comment.to_c_string().unwrap_or_default().into_raw()
@@ -746,6 +765,19 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             let data2 = key.to_string().to_c_string().unwrap_or_default();
             data2.into_raw()
         }
+        EventType::WebxdcRealtimeData { data, .. } => {
+            let ptr = libc::malloc(data.len());
+            libc::memcpy(ptr, data.as_ptr() as *mut libc::c_void, data.len());
+            ptr as *mut libc::c_char
+        }
+        EventType::IncomingReaction { reaction, .. } => reaction
+            .as_str()
+            .to_c_string()
+            .unwrap_or_default()
+            .into_raw(),
+        #[allow(unreachable_patterns)]
+        #[cfg(test)]
+        _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
     }
 }
 
@@ -827,8 +859,6 @@ pub unsafe extern "C" fn dc_maybe_network(context: *mut dc_context_t) {
 #[no_mangle]
 pub unsafe extern "C" fn dc_preconfigure_keypair(
     context: *mut dc_context_t,
-    addr: *const libc::c_char,
-    _public_data: *const libc::c_char,
     secret_data: *const libc::c_char,
 ) -> i32 {
     if context.is_null() {
@@ -836,9 +866,8 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
         return 0;
     }
     let ctx = &*context;
-    let addr = to_string_lossy(addr);
     let secret_data = to_string_lossy(secret_data);
-    block_on(preconfigure_keypair(ctx, &addr, &secret_data))
+    block_on(preconfigure_keypair(ctx, &secret_data))
         .context("Failed to save keypair")
         .log_err(ctx)
         .is_ok() as libc::c_int
@@ -1438,48 +1467,6 @@ pub unsafe extern "C" fn dc_get_chat_media(
     })
 }
 
-#[no_mangle]
-#[allow(deprecated)]
-pub unsafe extern "C" fn dc_get_next_media(
-    context: *mut dc_context_t,
-    msg_id: u32,
-    dir: libc::c_int,
-    msg_type: libc::c_int,
-    or_msg_type2: libc::c_int,
-    or_msg_type3: libc::c_int,
-) -> u32 {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_get_next_media()");
-        return 0;
-    }
-    let direction = if dir < 0 {
-        chat::Direction::Backward
-    } else {
-        chat::Direction::Forward
-    };
-
-    let ctx = &*context;
-    let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {msg_type}"));
-    let or_msg_type2 =
-        from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {or_msg_type2}"));
-    let or_msg_type3 =
-        from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {or_msg_type3}"));
-
-    block_on(async move {
-        chat::get_next_media(
-            ctx,
-            MsgId::new(msg_id),
-            direction,
-            msg_type,
-            or_msg_type2,
-            or_msg_type3,
-        )
-        .await
-        .map(|msg_id| msg_id.map(|id| id.to_u32()).unwrap_or_default())
-        .unwrap_or(0)
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_chat_visibility(
     context: *mut dc_context_t,
@@ -2607,6 +2594,18 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_create_qr_svg(payload: *const libc::c_char) -> *mut libc::c_char {
+    if payload.is_null() {
+        eprintln!("ignoring careless call to dc_create_qr_svg()");
+        return "".strdup();
+    }
+
+    create_qr_svg(&to_string_lossy(payload))
+        .unwrap_or_else(|_| "".to_string())
+        .strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_last_error(context: *mut dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
@@ -4356,7 +4355,7 @@ pub unsafe extern "C" fn dc_backup_provider_wait(provider: *mut dc_backup_provid
     let ctx = &*ffi_provider.context;
     let provider = &mut ffi_provider.provider;
     block_on(provider)
-        .context("Failed to await BackupProvider")
+        .context("Failed to await backup provider")
         .log_err(ctx)
         .set_last_error(ctx)
         .ok();
@@ -4410,7 +4409,7 @@ trait ResultExt<T, E> {
     /// Like `log_err()`, but:
     /// - returns the default value instead of an Err value.
     /// - emits an error instead of a warning for an [Err] result. This means
-    /// that the error will be shown to the user in a small pop-up.
+    ///   that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
 }
 
@@ -4529,19 +4528,16 @@ pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
     let addr = to_string_lossy(addr);
 
     let ctx = &*context;
-    let socks5_enabled = block_on(async move {
-        ctx.get_config_bool(config::Config::Socks5Enabled)
-            .await
-            .context("Can't get config")
-            .log_err(ctx)
-    });
+    let proxy_enabled = block_on(ctx.get_config_bool(config::Config::ProxyEnabled))
+        .context("Can't get config")
+        .log_err(ctx);
 
-    match socks5_enabled {
-        Ok(socks5_enabled) => {
+    match proxy_enabled {
+        Ok(proxy_enabled) => {
             match block_on(provider::get_provider_info_by_addr(
                 ctx,
                 addr.as_str(),
-                socks5_enabled,
+                proxy_enabled,
             ))
             .log_err(ctx)
             .unwrap_or_default()
@@ -4872,7 +4868,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     }
 
     let accounts = &*accounts;
-    block_on(async move { accounts.write().await.maybe_network_lost().await });
+    block_on(async move { accounts.read().await.maybe_network_lost().await });
 }
 
 #[no_mangle]
@@ -4886,12 +4882,12 @@ pub unsafe extern "C" fn dc_accounts_background_fetch(
     }
 
     let accounts = &*accounts;
-    block_on(async move {
-        let accounts = accounts.read().await;
-        accounts
-            .background_fetch(Duration::from_secs(timeout_in_seconds))
-            .await;
-    });
+    let background_fetch_future = {
+        let lock = block_on(accounts.read());
+        lock.background_fetch(Duration::from_secs(timeout_in_seconds))
+    };
+    // At this point account manager is not locked anymore.
+    block_on(background_fetch_future);
     1
 }
 
@@ -4909,7 +4905,7 @@ pub unsafe extern "C" fn dc_accounts_set_push_device_token(
     let token = to_string_lossy(token);
 
     block_on(async move {
-        let mut accounts = accounts.write().await;
+        let accounts = accounts.read().await;
         if let Err(err) = accounts.set_push_device_token(&token).await {
             accounts.emit_event(EventType::Error(format!(
                 "Failed to set notify token: {err:#}."

deltachat-ffi/src/lot.rs

@@ -34,33 +34,34 @@ pub enum Meaning {
 }
 
 impl Lot {
-    pub fn get_text1(&self) -> Option<&str> {
+    pub fn get_text1(&self) -> Option<Cow<str>> {
         match self {
             Self::Summary(summary) => match &summary.prefix {
                 None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(text),
-                Some(SummaryPrefix::Username(username)) => Some(username),
-                Some(SummaryPrefix::Me(text)) => Some(text),
+                Some(SummaryPrefix::Draft(text)) => Some(Cow::Borrowed(text)),
+                Some(SummaryPrefix::Username(username)) => Some(Cow::Borrowed(username)),
+                Some(SummaryPrefix::Me(text)) => Some(Cow::Borrowed(text)),
             },
             Self::Qr(qr) => match qr {
                 Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::AskVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
                 Qr::FprOk { .. } => None,
                 Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
-                Qr::Account { domain } => Some(domain),
-                Qr::Backup { .. } => None,
-                Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { draft, .. } => draft.as_deref(),
-                Qr::Url { url } => Some(url),
-                Qr::Text { text } => Some(text),
+                Qr::FprWithoutAddr { fingerprint, .. } => Some(Cow::Borrowed(fingerprint)),
+                Qr::Account { domain } => Some(Cow::Borrowed(domain)),
+                Qr::Backup2 { .. } => None,
+                Qr::WebrtcInstance { domain, .. } => Some(Cow::Borrowed(domain)),
+                Qr::Proxy { host, port, .. } => Some(Cow::Owned(format!("{host}:{port}"))),
+                Qr::Addr { draft, .. } => draft.as_deref().map(Cow::Borrowed),
+                Qr::Url { url } => Some(Cow::Borrowed(url)),
+                Qr::Text { text } => Some(Cow::Borrowed(text)),
                 Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::WithdrawVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
                 Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::Login { address, .. } => Some(address),
+                Qr::ReviveVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
+                Qr::Login { address, .. } => Some(Cow::Borrowed(address)),
             },
-            Self::Error(err) => Some(err),
+            Self::Error(err) => Some(Cow::Borrowed(err)),
         }
     }
 
@@ -101,8 +102,9 @@ impl Lot {
                 Qr::FprMismatch { .. } => LotState::QrFprMismatch,
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
                 Qr::Account { .. } => LotState::QrAccount,
-                Qr::Backup { .. } => LotState::QrBackup,
+                Qr::Backup2 { .. } => LotState::QrBackup2,
                 Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
+                Qr::Proxy { .. } => LotState::QrProxy,
                 Qr::Addr { .. } => LotState::QrAddr,
                 Qr::Url { .. } => LotState::QrUrl,
                 Qr::Text { .. } => LotState::QrText,
@@ -126,8 +128,9 @@ impl Lot {
                 Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
-                Qr::Backup { .. } => Default::default(),
+                Qr::Backup2 { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
+                Qr::Proxy { .. } => Default::default(),
                 Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
@@ -177,9 +180,14 @@ pub enum LotState {
 
     QrBackup = 251,
 
+    QrBackup2 = 252,
+
     /// text1=domain, text2=instance pattern
     QrWebrtcInstance = 260,
 
+    /// text1=address, text2=protocol
+    QrProxy = 271,
+
     /// id=contact
     QrAddr = 320,
 

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.138.5"
+version = "1.150.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -13,30 +13,30 @@ path = "src/webserver.rs"
 required-features = ["webserver"]
 
 [dependencies]
-anyhow = "1"
-deltachat = { path = ".." }
-deltachat-contact-tools = { path = "../deltachat-contact-tools" }
-num-traits = "0.2"
-schemars = "0.8.19"
-serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.10.1"
-log = "0.4"
-async-channel = { version = "2.2.1" }
-futures = { version = "0.3.30" }
-serde_json = "1"
-yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
-typescript-type-def = { version = "0.5.8", features = ["json_value"] }
-tokio = { version = "1.37.0" }
-sanitize-filename = "0.5"
+anyhow = { workspace = true }
+deltachat = { workspace = true }
+deltachat-contact-tools = { workspace = true }
+num-traits = { workspace = true }
+schemars = "0.8.21"
+serde = { workspace = true, features = ["derive"] }
+tempfile = { workspace = true }
+log = { workspace = true }
+async-channel = { workspace = true }
+futures = { workspace = true }
+serde_json = { workspace = true }
+yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
+typescript-type-def = { version = "0.5.13", features = ["json_value"] }
+tokio = { workspace = true }
+sanitize-filename = { workspace = true }
 walkdir = "2.5.0"
-base64 = "0.22"
+base64 = { workspace = true }
 
 # optional dependencies
 axum = { version = "0.7", optional = true, features = ["ws"] }
-env_logger = { version = "0.11.3", optional = true }
+env_logger = { version = "0.11.5", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.37.0", features = ["full", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["full", "rt-multi-thread"] }
 
 
 [features]

deltachat-jsonrpc/src/api.rs

@@ -254,11 +254,12 @@ impl CommandApi {
     /// Process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
     async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
-        self.accounts
-            .write()
-            .await
-            .background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
-            .await;
+        let future = {
+            let lock = self.accounts.read().await;
+            lock.background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
+        };
+        // At this point account manager is not locked anymore.
+        future.await;
         Ok(())
     }
 
@@ -321,12 +322,12 @@ impl CommandApi {
     ) -> Result<Option<ProviderInfo>> {
         let ctx = self.get_context(account_id).await?;
 
-        let socks5_enabled = ctx
-            .get_config_bool(deltachat::config::Config::Socks5Enabled)
+        let proxy_enabled = ctx
+            .get_config_bool(deltachat::config::Config::ProxyEnabled)
             .await?;
 
         let provider_info =
-            get_provider_info(&ctx, email.split('@').last().unwrap_or(""), socks5_enabled).await;
+            get_provider_info(&ctx, email.split('@').last().unwrap_or(""), proxy_enabled).await;
         Ok(ProviderInfo::from_dc_type(provider_info))
     }
 
@@ -707,7 +708,22 @@ impl CommandApi {
         ChatId::new(chat_id).get_encryption_info(&ctx).await
     }
 
-    /// Get QR code (text and SVG) that will offer an Setup-Contact or Verified-Group invitation.
+    /// Get QR code text that will offer a [SecureJoin](https://securejoin.delta.chat/) invitation.
+    ///
+    /// If `chat_id` is a group chat ID, SecureJoin QR code for the group is returned.
+    /// If `chat_id` is unset, setup contact QR code is returned.
+    async fn get_chat_securejoin_qr_code(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+    ) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let chat = chat_id.map(ChatId::new);
+        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
+        Ok(qr)
+    }
+
+    /// Get QR code (text and SVG) that will offer a Setup-Contact or Verified-Group invitation.
     /// The QR code is compatible to the OPENPGP4FPR format
     /// so that a basic fingerprint comparison also works e.g. with OpenKeychain.
     ///
@@ -729,10 +745,9 @@ impl CommandApi {
     ) -> Result<(String, String)> {
         let ctx = self.get_context(account_id).await?;
         let chat = chat_id.map(ChatId::new);
-        Ok((
-            securejoin::get_securejoin_qr(&ctx, chat).await?,
-            get_securejoin_qr_svg(&ctx, chat).await?,
-        ))
+        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
+        let svg = get_securejoin_qr_svg(&ctx, chat).await?;
+        Ok((qr, svg))
     }
 
     /// Continue a Setup-Contact or Verified-Group-Invite protocol
@@ -1120,9 +1135,11 @@ impl CommandApi {
     async fn get_message(&self, account_id: u32, msg_id: u32) -> Result<MessageObject> {
         let ctx = self.get_context(account_id).await?;
         let msg_id = MsgId::new(msg_id);
-        MessageObject::from_msg_id(&ctx, msg_id)
+        let message_object = MessageObject::from_msg_id(&ctx, msg_id)
             .await
-            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))
+            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))?
+            .with_context(|| format!("Message {msg_id} does not exist for account {account_id}"))?;
+        Ok(message_object)
     }
 
     async fn get_message_html(&self, account_id: u32, message_id: u32) -> Result<Option<String>> {
@@ -1146,7 +1163,10 @@ impl CommandApi {
             messages.insert(
                 message_id,
                 match message_result {
-                    Ok(message) => MessageLoadResult::Message(message),
+                    Ok(Some(message)) => MessageLoadResult::Message(message),
+                    Ok(None) => MessageLoadResult::LoadingError {
+                        error: "Message does not exist".to_string(),
+                    },
                     Err(error) => MessageLoadResult::LoadingError {
                         error: format!("{error:#}"),
                     },
@@ -1404,6 +1424,15 @@ impl CommandApi {
         Ok(())
     }
 
+    /// Resets contact encryption.
+    async fn reset_contact_encryption(&self, account_id: u32, contact_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let contact_id = ContactId::new(contact_id);
+
+        contact_id.reset_encryption(&ctx).await?;
+        Ok(())
+    }
+
     async fn change_contact_name(
         &self,
         account_id: u32,
@@ -1447,7 +1476,7 @@ impl CommandApi {
 
     /// Parses a vCard file located at the given path. Returns contacts in their original order.
     async fn parse_vcard(&self, path: String) -> Result<Vec<VcardContact>> {
-        let vcard = tokio::fs::read(Path::new(&path)).await?;
+        let vcard = fs::read(Path::new(&path)).await?;
         let vcard = str::from_utf8(&vcard)?;
         Ok(deltachat_contact_tools::parse_vcard(vcard)
             .into_iter()
@@ -1455,6 +1484,20 @@ impl CommandApi {
             .collect())
     }
 
+    /// Imports contacts from a vCard file located at the given path.
+    ///
+    /// Returns the ids of created/modified contacts in the order they appear in the vCard.
+    async fn import_vcard(&self, account_id: u32, path: String) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let vcard = tokio::fs::read(Path::new(&path)).await?;
+        let vcard = str::from_utf8(&vcard)?;
+        Ok(deltachat::contact::import_vcard(&ctx, vcard)
+            .await?
+            .into_iter()
+            .map(|c| c.to_u32())
+            .collect())
+    }
+
     /// Returns a vCard containing contacts with the given ids.
     async fn make_vcard(&self, account_id: u32, contacts: Vec<u32>) -> Result<String> {
         let ctx = self.get_context(account_id).await?;
@@ -1462,6 +1505,20 @@ impl CommandApi {
         deltachat::contact::make_vcard(&ctx, &contacts).await
     }
 
+    /// Sets vCard containing the given contacts to the message draft.
+    async fn set_draft_vcard(
+        &self,
+        account_id: u32,
+        msg_id: u32,
+        contacts: Vec<u32>,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts: Vec<_> = contacts.iter().map(|&c| ContactId::new(c)).collect();
+        let mut msg = Message::load_from_db(&ctx, MsgId::new(msg_id)).await?;
+        msg.make_vcard(&ctx, &contacts).await?;
+        msg.get_chat_id().set_draft(&ctx, Some(&mut msg)).await
+    }
+
     // ---------------------------------------------
     //                   chat
     // ---------------------------------------------
@@ -1510,55 +1567,6 @@ impl CommandApi {
         Ok(media.iter().map(|msg_id| msg_id.to_u32()).collect())
     }
 
-    /// Search next/previous message based on a given message and a list of types.
-    /// Typically used to implement the "next" and "previous" buttons
-    /// in a gallery or in a media player.
-    ///
-    /// one combined call for getting chat::get_next_media for both directions
-    /// the manual chat::get_next_media in only one direction is not exposed by the jsonrpc yet
-    ///
-    /// Deprecated 2023-10-03, use `get_chat_media` method
-    /// and navigate the returned array instead.
-    #[allow(deprecated)]
-    async fn get_neighboring_chat_media(
-        &self,
-        account_id: u32,
-        msg_id: u32,
-        message_type: MessageViewtype,
-        or_message_type2: Option<MessageViewtype>,
-        or_message_type3: Option<MessageViewtype>,
-    ) -> Result<(Option<u32>, Option<u32>)> {
-        let ctx = self.get_context(account_id).await?;
-
-        let msg_type: Viewtype = message_type.into();
-        let msg_type2: Viewtype = or_message_type2.map(|v| v.into()).unwrap_or_default();
-        let msg_type3: Viewtype = or_message_type3.map(|v| v.into()).unwrap_or_default();
-
-        let prev = chat::get_next_media(
-            &ctx,
-            MsgId::new(msg_id),
-            chat::Direction::Backward,
-            msg_type,
-            msg_type2,
-            msg_type3,
-        )
-        .await?
-        .map(|id| id.to_u32());
-
-        let next = chat::get_next_media(
-            &ctx,
-            MsgId::new(msg_id),
-            chat::Direction::Forward,
-            msg_type,
-            msg_type2,
-            msg_type3,
-        )
-        .await?
-        .map(|id| id.to_u32());
-
-        Ok((prev, next))
-    }
-
     // ---------------------------------------------
     //                   backup
     // ---------------------------------------------
@@ -1630,10 +1638,10 @@ impl CommandApi {
     ///
     /// This call will block until the QR code is ready,
     /// even if there is no concurrent call to [`CommandApi::provide_backup`],
-    /// but will fail after 10 seconds to avoid deadlocks.
+    /// but will fail after 60 seconds to avoid deadlocks.
     async fn get_backup_qr(&self, account_id: u32) -> Result<String> {
         let qr = tokio::time::timeout(
-            Duration::from_secs(10),
+            Duration::from_secs(60),
             self.inner_get_backup_qr(account_id),
         )
         .await
@@ -1649,13 +1657,13 @@ impl CommandApi {
     ///
     /// This call will block until the QR code is ready,
     /// even if there is no concurrent call to [`CommandApi::provide_backup`],
-    /// but will fail after 10 seconds to avoid deadlocks.
+    /// but will fail after 60 seconds to avoid deadlocks.
     ///
     /// Returns the QR code rendered as an SVG image.
     async fn get_backup_qr_svg(&self, account_id: u32) -> Result<String> {
         let ctx = self.get_context(account_id).await?;
         let qr = tokio::time::timeout(
-            Duration::from_secs(10),
+            Duration::from_secs(60),
             self.inner_get_backup_qr(account_id),
         )
         .await
@@ -1953,9 +1961,13 @@ impl CommandApi {
 
     async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
-        let mut message = data.create_message(&ctx).await?;
+        let mut message = data
+            .create_message(&ctx)
+            .await
+            .context("Failed to create message")?;
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
-            .await?
+            .await
+            .context("Failed to send created message")?
             .to_u32();
         Ok(msg_id)
     }
@@ -1992,9 +2004,7 @@ impl CommandApi {
     async fn get_draft(&self, account_id: u32, chat_id: u32) -> Result<Option<MessageObject>> {
         let ctx = self.get_context(account_id).await?;
         if let Some(draft) = ChatId::new(chat_id).get_draft(&ctx).await? {
-            Ok(Some(
-                MessageObject::from_msg_id(&ctx, draft.get_id()).await?,
-            ))
+            Ok(MessageObject::from_msg_id(&ctx, draft.get_id()).await?)
         } else {
             Ok(None)
         }
@@ -2120,8 +2130,7 @@ impl CommandApi {
     ) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
 
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(text);
+        let mut msg = Message::new_text(text);
 
         let message_id = deltachat::chat::send_msg(&ctx, ChatId::new(chat_id), &mut msg).await?;
         Ok(message_id.to_u32())
@@ -2164,7 +2173,9 @@ impl CommandApi {
                 .await?;
         }
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message).await?;
-        let message = MessageObject::from_msg_id(&ctx, msg_id).await?;
+        let message = MessageObject::from_msg_id(&ctx, msg_id)
+            .await?
+            .context("Just sent message does not exist")?;
         Ok((msg_id.to_u32(), message))
     }
 

deltachat-jsonrpc/src/api/types/account.rs

@@ -17,6 +17,9 @@ pub enum Account {
         // size: u32,
         profile_image: Option<String>, // TODO: This needs to be converted to work with blob http server.
         color: String,
+        /// Optional tag as "Work", "Family".
+        /// Meant to help profile owner to differ between profiles with similar names.
+        private_tag: Option<String>,
     },
     #[serde(rename_all = "camelCase")]
     Unconfigured { id: u32 },
@@ -31,12 +34,14 @@ impl Account {
             let color = color_int_to_hex_string(
                 Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
             );
+            let private_tag = ctx.get_config(Config::PrivateTag).await?;
             Ok(Account::Configured {
                 id,
                 display_name,
                 addr,
                 profile_image,
                 color,
+                private_tag,
             })
         } else {
             Ok(Account::Unconfigured { id })

deltachat-jsonrpc/src/api/types/chat.rs

@@ -32,6 +32,7 @@ pub struct FullChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
+    pinned: bool,
     // subtitle  - will be moved to frontend because it uses translation functions
     chat_type: u32,
     is_unpromoted: bool,
@@ -104,6 +105,7 @@ impl FullChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),
@@ -153,6 +155,7 @@ pub struct BasicChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
+    pinned: bool,
     chat_type: u32,
     is_unpromoted: bool,
     is_self_talk: bool,
@@ -180,6 +183,7 @@ impl BasicChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -88,11 +88,17 @@ pub(crate) async fn get_chat_list_item_by_id(
 
     let (last_updated, message_type) = match last_msgid {
         Some(id) => {
-            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
-            (
-                Some(last_message.get_timestamp() * 1000),
-                Some(last_message.get_viewtype().into()),
-            )
+            if let Some(last_message) =
+                deltachat::message::Message::load_from_db_optional(ctx, id).await?
+            {
+                (
+                    Some(last_message.get_timestamp() * 1000),
+                    Some(last_message.get_viewtype().into()),
+                )
+            } else {
+                // Message may be deleted by the time we try to load it.
+                (None, None)
+            }
         }
         None => (None, None),
     };

deltachat-jsonrpc/src/api/types/contact.rs

@@ -19,6 +19,7 @@ pub struct ContactObject {
     profile_image: Option<String>, // BLOBS
     name_and_addr: String,
     is_blocked: bool,
+    e2ee_avail: bool,
 
     /// True if the contact can be added to verified groups.
     ///
@@ -79,6 +80,7 @@ impl ContactObject {
             profile_image, //BLOBS
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
+            e2ee_avail: contact.e2ee_avail(context).await?,
             is_verified,
             is_profile_verified,
             verifier_id,

deltachat-jsonrpc/src/api/types/events.rs

@@ -98,6 +98,14 @@ pub enum EventType {
         contact_id: u32,
     },
 
+    /// Incoming reaction, should be notified.
+    #[serde(rename_all = "camelCase")]
+    IncomingReaction {
+        contact_id: u32,
+        msg_id: u32,
+        reaction: String,
+    },
+
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
@@ -244,6 +252,11 @@ pub enum EventType {
     #[serde(rename_all = "camelCase")]
     WebxdcRealtimeData { msg_id: u32, data: Vec<u8> },
 
+    /// Advertisement received over an ephemeral peer channel.
+    /// This can be used by bots to initiate peer-to-peer communication from their side.
+    #[serde(rename_all = "camelCase")]
+    WebxdcRealtimeAdvertisementReceived { msg_id: u32 },
+
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
     WebxdcInstanceDeleted { msg_id: u32 },
@@ -263,6 +276,9 @@ pub enum EventType {
     /// If `chat_id` is set to None, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
     #[serde(rename_all = "camelCase")]
     ChatlistItemChanged { chat_id: Option<u32> },
+
+    /// Inform than some events have been skipped due to event channel overflow.
+    EventChannelOverflow { n: u64 },
 }
 
 impl From<CoreEventType> for EventType {
@@ -294,6 +310,15 @@ impl From<CoreEventType> for EventType {
                 msg_id: msg_id.to_u32(),
                 contact_id: contact_id.to_u32(),
             },
+            CoreEventType::IncomingReaction {
+                contact_id,
+                msg_id,
+                reaction,
+            } => IncomingReaction {
+                contact_id: contact_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+                reaction: reaction.as_str().to_string(),
+            },
             CoreEventType::IncomingMsg { chat_id, msg_id } => IncomingMsg {
                 chat_id: chat_id.to_u32(),
                 msg_id: msg_id.to_u32(),
@@ -370,6 +395,11 @@ impl From<CoreEventType> for EventType {
                 msg_id: msg_id.to_u32(),
                 data,
             },
+            CoreEventType::WebxdcRealtimeAdvertisementReceived { msg_id } => {
+                WebxdcRealtimeAdvertisementReceived {
+                    msg_id: msg_id.to_u32(),
+                }
+            }
             CoreEventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
                 msg_id: msg_id.to_u32(),
             },
@@ -378,6 +408,10 @@ impl From<CoreEventType> for EventType {
                 chat_id: chat_id.map(|id| id.to_u32()),
             },
             CoreEventType::ChatlistChanged => ChatlistChanged,
+            CoreEventType::EventChannelOverflow { n } => EventChannelOverflow { n },
+            #[allow(unreachable_patterns)]
+            #[cfg(test)]
+            _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
         }
     }
 }

deltachat-jsonrpc/src/api/types/message.rs

@@ -112,8 +112,10 @@ enum MessageQuote {
 }
 
 impl MessageObject {
-    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
-        let message = Message::load_from_db(context, msg_id).await?;
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Option<Self>> {
+        let Some(message) = Message::load_from_db_optional(context, msg_id).await? else {
+            return Ok(None);
+        };
 
         let sender_contact = Contact::get_by_id(context, message.get_from_id())
             .await
@@ -183,7 +185,7 @@ impl MessageObject {
             .map(Into::into)
             .collect();
 
-        Ok(MessageObject {
+        let message_object = MessageObject {
             id: msg_id.to_u32(),
             chat_id: message.get_chat_id().to_u32(),
             from_id: message.get_from_id().to_u32(),
@@ -244,7 +246,8 @@ impl MessageObject {
             reactions,
 
             vcard_contact: vcard_contacts.first().cloned(),
-        })
+        };
+        Ok(Some(message_object))
     }
 }
 
@@ -490,6 +493,7 @@ pub struct MessageSearchResult {
     author_name: String,
     author_color: String,
     author_id: u32,
+    chat_id: u32,
     chat_profile_image: Option<String>,
     chat_color: String,
     chat_name: String,
@@ -529,6 +533,7 @@ impl MessageSearchResult {
             author_name,
             author_color: color_int_to_hex_string(sender.get_color()),
             author_id: sender.id.to_u32(),
+            chat_id: chat.id.to_u32(),
             chat_name: chat.get_name().to_owned(),
             chat_color,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
@@ -577,7 +582,9 @@ pub struct MessageData {
     pub file: Option<String>,
     pub location: Option<(f64, f64)>,
     pub override_sender_name: Option<String>,
+    /// Quoted message id. Takes preference over `quoted_text` (see below).
     pub quoted_message_id: Option<u32>,
+    pub quoted_text: Option<String>,
 }
 
 impl MessageData {
@@ -603,16 +610,16 @@ impl MessageData {
             message.set_location(latitude, longitude);
         }
         if let Some(id) = self.quoted_message_id {
+            let quoted_message = Message::load_from_db(context, MsgId::new(id))
+                .await
+                .context("Failed to load quoted message")?;
             message
-                .set_quote(
-                    context,
-                    Some(
-                        &Message::load_from_db(context, MsgId::new(id))
-                            .await
-                            .context("message to quote could not be loaded")?,
-                    ),
-                )
-                .await?;
+                .set_quote(context, Some(&quoted_message))
+                .await
+                .context("Failed to set quote")?;
+        } else if let Some(text) = self.quoted_text {
+            let protect = false;
+            message.set_quote_text(Some((text, protect)));
         }
         Ok(message)
     }
@@ -635,7 +642,7 @@ pub struct MessageInfo {
     error: Option<String>,
     rfc724_mid: String,
     server_urls: Vec<String>,
-    hop_info: Option<String>,
+    hop_info: String,
 }
 
 impl MessageInfo {

deltachat-jsonrpc/src/api/types/qr.rs

@@ -6,80 +6,161 @@ use typescript_type_def::TypeDef;
 #[serde(rename = "Qr", rename_all = "camelCase")]
 #[serde(tag = "kind")]
 pub enum QrObject {
+    /// Ask the user whether to verify the contact.
+    ///
+    /// If the user agrees, pass this QR code to [`crate::securejoin::join_securejoin`].
     AskVerifyContact {
+        /// ID of the contact.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user whether to join the group.
     AskVerifyGroup {
+        /// Group name.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// ID of the contact.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Contact fingerprint is verified.
+    ///
+    /// Ask the user if they want to start chatting.
     FprOk {
+        /// Contact ID.
         contact_id: u32,
     },
+    /// Scanned fingerprint does not match the last seen fingerprint.
     FprMismatch {
+        /// Contact ID.
         contact_id: Option<u32>,
     },
+    /// The scanned QR code contains a fingerprint but no e-mail address.
     FprWithoutAddr {
+        /// Key fingerprint.
         fingerprint: String,
     },
+    /// Ask the user if they want to create an account on the given domain.
     Account {
+        /// Server domain name.
         domain: String,
     },
-    Backup {
-        ticket: String,
+    /// Provides a backup that can be retrieved using iroh-net based backup transfer protocol.
+    Backup2 {
+        /// Authentication token.
+        auth_token: String,
+        /// Iroh node address.
+        node_addr: String,
     },
+    /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
     },
+    /// Ask the user if they want to use the given proxy.
+    ///
+    /// Note that HTTP(S) URLs without a path
+    /// and query parameters are treated as HTTP(S) proxy URL.
+    /// UI may want to still offer to open the URL
+    /// in the browser if QR code contents
+    /// starts with `http://` or `https://`
+    /// and the QR code was not scanned from
+    /// the proxy configuration screen.
+    Proxy {
+        /// Proxy URL.
+        ///
+        /// This is the URL that is going to be added.
+        url: String,
+        /// Host extracted from the URL to display in the UI.
+        host: String,
+        /// Port extracted from the URL to display in the UI.
+        port: u16,
+    },
+    /// Contact address is scanned.
+    ///
+    /// Optionally, a draft message could be provided.
+    /// Ask the user if they want to start chatting.
     Addr {
+        /// Contact ID.
         contact_id: u32,
+        /// Draft message.
         draft: Option<String>,
     },
-    Url {
-        url: String,
-    },
-    Text {
-        text: String,
-    },
+    /// URL scanned.
+    ///
+    /// Ask the user if they want to open a browser or copy the URL to clipboard.
+    Url { url: String },
+    /// Text scanned.
+    ///
+    /// Ask the user if they want to copy the text to clipboard.
+    Text { text: String },
+    /// Ask the user if they want to withdraw their own QR code.
     WithdrawVerifyContact {
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to withdraw their own group invite QR code.
     WithdrawVerifyGroup {
+        /// Group name.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own QR code.
     ReviveVerifyContact {
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own group invite QR code.
     ReviveVerifyGroup {
+        /// Contact ID.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
-    Login {
-        address: String,
-    },
+    /// `dclogin:` scheme parameters.
+    ///
+    /// Ask the user if they want to login with the email address.
+    Login { address: String },
 }
 
 impl From<Qr> for QrObject {
@@ -129,8 +210,12 @@ impl From<Qr> for QrObject {
             }
             Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint },
             Qr::Account { domain } => QrObject::Account { domain },
-            Qr::Backup { ticket } => QrObject::Backup {
-                ticket: ticket.to_string(),
+            Qr::Backup2 {
+                ref node_addr,
+                auth_token,
+            } => QrObject::Backup2 {
+                node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
+                auth_token,
             },
             Qr::WebrtcInstance {
                 domain,
@@ -139,6 +224,7 @@ impl From<Qr> for QrObject {
                 domain,
                 instance_pattern,
             },
+            Qr::Proxy { url, host, port } => QrObject::Proxy { url, host, port },
             Qr::Addr { contact_id, draft } => {
                 let contact_id = contact_id.to_u32();
                 QrObject::Addr { contact_id, draft }

deltachat-jsonrpc/src/lib.rs

@@ -1,4 +1,6 @@
 #![recursion_limit = "256"]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 pub mod api;
 pub use yerpc;
 
@@ -83,7 +85,7 @@ mod tests {
             assert_eq!(result, response.to_owned());
         }
         {
-            let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
+            let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":""}]}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
             session.handle_incoming(request).await;
             let result = receiver.recv().await?;

deltachat-jsonrpc/typescript/package.json

@@ -3,7 +3,7 @@
   "dependencies": {
     "@deltachat/tiny-emitter": "3.0.0",
     "isomorphic-ws": "^4.0.1",
-    "yerpc": "^0.4.3"
+    "yerpc": "^0.6.2"
   },
   "devDependencies": {
     "@types/chai": "^4.2.21",
@@ -32,6 +32,10 @@
   "license": "MPL-2.0",
   "main": "dist/deltachat.js",
   "name": "@deltachat/jsonrpc-client",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/deltachat/deltachat-core-rust.git"
+  },
   "scripts": {
     "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
     "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
@@ -54,5 +58,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.138.5"
+  "version": "1.150.0"
 }

deltachat-jsonrpc/typescript/test/online.ts

@@ -86,10 +86,7 @@ describe("online tests", function () {
       null
     );
     const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
-    const eventPromise = Promise.race([
-      waitForEvent(dc, "MsgsChanged", accountId2),
-      waitForEvent(dc, "IncomingMsg", accountId2),
-    ]);
+    const eventPromise = waitForEvent(dc, "IncomingMsg", accountId2);
 
     await dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello");
     const { chatId: chatIdOnAccountB } = await eventPromise;
@@ -119,10 +116,7 @@ describe("online tests", function () {
       null
     );
     const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
-    const eventPromise = Promise.race([
-      waitForEvent(dc, "MsgsChanged", accountId2),
-      waitForEvent(dc, "IncomingMsg", accountId2),
-    ]);
+    const eventPromise = waitForEvent(dc, "IncomingMsg", accountId2);
     dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello2");
     // wait for message from A
     console.log("wait for message from A");
@@ -143,10 +137,7 @@ describe("online tests", function () {
     );
     expect(message.text).equal("Hello2");
     // Send message back from B to A
-    const eventPromise2 = Promise.race([
-      waitForEvent(dc, "MsgsChanged", accountId1),
-      waitForEvent(dc, "IncomingMsg", accountId1),
-    ]);
+    const eventPromise2 = waitForEvent(dc, "IncomingMsg", accountId1);
     dc.rpc.miscSendTextMessage(accountId2, chatId, "super secret message");
     // Check if answer arrives at A and if it is encrypted
     await eventPromise2;

deltachat-repl/Cargo.toml

@@ -1,20 +1,21 @@
 [package]
 name = "deltachat-repl"
-version = "1.138.5"
+version = "1.150.0"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [dependencies]
-ansi_term = "0.12.1"
-anyhow = "1"
-deltachat = { path = "..", features = ["internals"]}
+anyhow = { workspace = true }
+deltachat = { workspace = true, features = ["internals"]}
 dirs = "5"
-log = "0.4.21"
-pretty_env_logger = "0.5"
-rusqlite = "0.31"
+log = { workspace = true }
+nu-ansi-term = { workspace = true }
+qr2term = "0.3.3"
+rusqlite = { workspace = true }
 rustyline = "14"
-tokio = { version = "1.37.0", features = ["fs", "rt-multi-thread", "macros"] }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
 
 [features]
 default = ["vendored"]

deltachat-repl/src/cmdline.rs

@@ -19,8 +19,10 @@ use deltachat::location;
 use deltachat::log::LogExt;
 use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::mimeparser::SystemMessage;
+use deltachat::peer_channels::{send_webxdc_realtime_advertisement, send_webxdc_realtime_data};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
+use deltachat::qr_code_generator::create_qr_svg;
 use deltachat::reaction::send_reaction;
 use deltachat::receive_imf::*;
 use deltachat::sql;
@@ -338,7 +340,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  receive-backup <qr>\n\
                  export-keys\n\
                  import-keys\n\
-                 export-setup\n\
                  poke [<eml-file>|<folder>|<addr> <key-file>]\n\
                  reset <flags>\n\
                  stop\n\
@@ -355,6 +356,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  configure\n\
                  connect\n\
                  disconnect\n\
+                 fetch\n\
                  connectivity\n\
                  maybenetwork\n\
                  housekeeping\n\
@@ -424,6 +426,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  checkqr <qr-content>\n\
                  joinqr <qr-content>\n\
                  setqr <qr-content>\n\
+                 createqrsvg <qr-content>\n\
                  providerinfo <addr>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
@@ -486,8 +489,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "send-backup" => {
             let provider = BackupProvider::prepare(&context).await?;
-            let qr = provider.qr();
-            println!("QR code: {}", format_backup(&qr)?);
+            let qr = format_backup(&provider.qr())?;
+            println!("QR code: {}", qr);
+            qr2term::print_qr(qr.as_str())?;
             provider.await?;
         }
         "receive-backup" => {
@@ -503,17 +507,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "import-keys" => {
             imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
         }
-        "export-setup" => {
-            let setup_code = create_setup_code(&context);
-            let file_name = blobdir.join("autocrypt-setup-message.html");
-            let file_content = render_setup_file(&context, &setup_code).await?;
-            fs::write(&file_name, file_content).await?;
-            println!(
-                "Setup message written to: {}\nSetup code: {}",
-                file_name.display(),
-                &setup_code,
-            );
-        }
         "poke" => {
             ensure!(poke_spec(&context, Some(arg1)).await, "Poke failed");
         }
@@ -642,6 +635,30 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             println!("{cnt} chats");
             println!("{time_needed:?} to create this list");
         }
+        "start-realtime" => {
+            if arg1.is_empty() {
+                bail!("missing msgid");
+            }
+            let msg_id = MsgId::new(arg1.parse()?);
+            let res = send_webxdc_realtime_advertisement(&context, msg_id).await?;
+
+            if let Some(res) = res {
+                println!("waiting for peer channel join");
+                res.await?;
+            }
+            println!("joined peer channel");
+        }
+        "send-realtime" => {
+            if arg1.is_empty() {
+                bail!("missing msgid");
+            }
+            if arg2.is_empty() {
+                bail!("no message");
+            }
+            let msg_id = MsgId::new(arg1.parse()?);
+            send_webxdc_realtime_data(&context, msg_id, arg2.as_bytes().to_vec()).await?;
+            println!("sent realtime message");
+        }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
                 bail!("Argument [chat-id] is missing.");
@@ -987,8 +1004,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             if !arg1.is_empty() {
-                let mut draft = Message::new(Viewtype::Text);
-                draft.set_text(arg1.to_string());
+                let mut draft = Message::new_text(arg1.to_string());
                 sel_chat
                     .as_ref()
                     .unwrap()
@@ -1011,8 +1027,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty(),
                 "Please specify text to add as device message."
             );
-            let mut msg = Message::new(Viewtype::Text);
-            msg.set_text(arg1.to_string());
+            let mut msg = Message::new_text(arg1.to_string());
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "listmedia" => {
@@ -1234,12 +1249,19 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Err(err) => println!("Cannot set config from QR code: {err:?}"),
             }
         }
+        "createqrsvg" => {
+            ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
+            let svg = create_qr_svg(arg1)?;
+            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
+            fs::write(&file, svg).await?;
+            println!("{file:#?} written.");
+        }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            let socks5_enabled = context
-                .get_config_bool(config::Config::Socks5Enabled)
+            let proxy_enabled = context
+                .get_config_bool(config::Config::ProxyEnabled)
                 .await?;
-            match provider::get_provider_info(&context, arg1, socks5_enabled).await {
+            match provider::get_provider_info(&context, arg1, proxy_enabled).await {
                 Some(info) => {
                     println!("Information for provider belonging to {arg1}:");
                     println!("status: {}", info.status as u32);

deltachat-repl/src/main.rs

@@ -9,10 +9,7 @@
 extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
-use std::io::{self, Write};
-use std::process::Command;
 
-use ansi_term::Color;
 use anyhow::{bail, Error};
 use deltachat::chat::ChatId;
 use deltachat::config;
@@ -22,6 +19,7 @@ use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
 use deltachat::EventType;
 use log::{error, info, warn};
+use nu_ansi_term::Color;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
@@ -32,6 +30,7 @@ use rustyline::{
 };
 use tokio::fs;
 use tokio::runtime::Handle;
+use tracing_subscriber::EnvFilter;
 
 mod cmdline;
 use self::cmdline::*;
@@ -151,7 +150,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 14] = [
+const IMEX_COMMANDS: [&str; 13] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -162,13 +161,12 @@ const IMEX_COMMANDS: [&str; 14] = [
     "receive-backup",
     "export-keys",
     "import-keys",
-    "export-setup",
     "poke",
     "reset",
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 11] = [
     "info",
     "set",
     "get",
@@ -176,6 +174,7 @@ const DB_COMMANDS: [&str; 10] = [
     "configure",
     "connect",
     "disconnect",
+    "fetch",
     "connectivity",
     "maybenetwork",
     "housekeeping",
@@ -241,12 +240,13 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 11] = [
+const MISC_COMMANDS: [&str; 12] = [
     "getqr",
     "getqrsvg",
     "getbadqr",
     "checkqr",
     "joinqr",
+    "createqrsvg",
     "fileinfo",
     "clear",
     "exit",
@@ -417,6 +417,9 @@ async fn handle_cmd(
         "disconnect" => {
             ctx.stop_io().await;
         }
+        "fetch" => {
+            ctx.background_fetch().await?;
+        }
         "configure" => {
             ctx.configure().await?;
         }
@@ -446,12 +449,7 @@ async fn handle_cmd(
                     qr.replace_range(12..22, "0000000000")
                 }
                 println!("{qr}");
-                let output = Command::new("qrencode")
-                    .args(["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                    .output()
-                    .expect("failed to execute process");
-                io::stdout().write_all(&output.stdout).unwrap();
-                io::stderr().write_all(&output.stderr).unwrap();
+                qr2term::print_qr(qr.as_str())?;
             }
         }
         "getqrsvg" => {
@@ -483,9 +481,10 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    pretty_env_logger::formatted_timed_builder()
-        .parse_default_env()
-        .filter_module("deltachat_repl", log::LevelFilter::Info)
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            EnvFilter::from_default_env().add_directive("deltachat_repl=info".parse()?),
+        )
         .init();
 
     let args = std::env::args().collect();

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.138.5"
+version = "1.150.0"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -13,10 +13,13 @@ classifiers = [
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS :: MacOS X",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Communications :: Chat",
     "Topic :: Communications :: Email"
 ]

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -250,12 +250,16 @@ class Account:
         """
         return Chat(self, self._rpc.secure_join(self.id, qrdata))
 
-    def get_qr_code(self) -> tuple[str, str]:
-        """Get Setup-Contact QR Code text and SVG data.
+    def get_qr_code(self) -> str:
+        """Get Setup-Contact QR Code text.
 
-        this data needs to be transferred to another Delta Chat account
+        This data needs to be transferred to another Delta Chat account
         in a second channel, typically used by mobiles with QRcode-show + scan UX.
         """
+        return self._rpc.get_chat_securejoin_qr_code(self.id, None)
+
+    def get_qr_code_svg(self) -> tuple[str, str]:
+        """Get Setup-Contact QR code text and SVG."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.id, None)
 
     def get_message_by_id(self, msg_id: int) -> Message:
@@ -297,6 +301,12 @@ class Account:
             if event.kind == EventType.INCOMING_MSG:
                 return event
 
+    def wait_for_incoming_msg(self):
+        """Wait for incoming message and return it.
+
+        Consumes all events before the next incoming message event."""
+        return self.get_message_by_id(self.wait_for_incoming_msg_event().msg_id)
+
     def wait_for_securejoin_inviter_success(self):
         while True:
             event = self.wait_for_event()

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import calendar
 from dataclasses import dataclass
+from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Optional, Union
 
 from ._utils import AttrDict
@@ -95,7 +96,11 @@ class Chat:
         """Return encryption info for this chat."""
         return self._rpc.get_chat_encryption_info(self.account.id, self.id)
 
-    def get_qr_code(self) -> tuple[str, str]:
+    def get_qr_code(self) -> str:
+        """Get Join-Group QR code text."""
+        return self._rpc.get_chat_securejoin_qr_code(self.account.id, self.id)
+
+    def get_qr_code_svg(self) -> tuple[str, str]:
         """Get Join-Group QR code text and SVG data."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.account.id, self.id)
 
@@ -265,3 +270,11 @@ class Chat:
             location["message"] = Message(self.account, location.msg_id)
             locations.append(location)
         return locations
+
+    def send_contact(self, contact: Contact):
+        """Send contact to the chat."""
+        vcard = contact.make_vcard()
+        with NamedTemporaryFile(suffix=".vcard") as f:
+            f.write(vcard.encode())
+            f.flush()
+            self._rpc.send_msg(self.account.id, self.id, {"viewtype": ViewType.VCARD, "file": f.name})

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -62,6 +62,8 @@ class EventType(str, Enum):
     CHATLIST_CHANGED = "ChatlistChanged"
     CHATLIST_ITEM_CHANGED = "ChatlistItemChanged"
     CONFIG_SYNCED = "ConfigSynced"
+    WEBXDC_REALTIME_DATA = "WebxdcRealtimeData"
+    WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED = "WebxdcRealtimeAdvertisementReceived"
 
 
 class ChatId(IntEnum):
@@ -114,6 +116,7 @@ class ViewType(str, Enum):
     FILE = "File"
     VIDEOCHAT_INVITATION = "VideochatInvitation"
     WEBXDC = "Webxdc"
+    VCARD = "Vcard"
 
 
 class SystemMessageType(str, Enum):

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -36,6 +36,10 @@ class Contact:
         """Delete contact."""
         self._rpc.delete_contact(self.account.id, self.id)
 
+    def reset_encryption(self) -> None:
+        """Reset contact encryption."""
+        self._rpc.reset_contact_encryption(self.account.id, self.id)
+
     def set_name(self, name: str) -> None:
         """Change the name of this contact."""
         self._rpc.change_contact_name(self.account.id, self.id, name)
@@ -60,3 +64,6 @@ class Contact:
             self.account,
             self._rpc.create_chat_by_contact_id(self.account.id, self.id),
         )
+
+    def make_vcard(self) -> str:
+        return self._rpc.make_vcard(self.account.id, [self.id])

deltachat-rpc-client/src/deltachat_rpc_client/direct_imap.py

@@ -9,18 +9,19 @@ import io
 import pathlib
 import ssl
 from contextlib import contextmanager
+from typing import TYPE_CHECKING
 
 from imap_tools import (
     AND,
     Header,
     MailBox,
-    MailBoxTls,
     MailMessage,
     MailMessageFlags,
     errors,
 )
 
-from . import Account, const
+if TYPE_CHECKING:
+    from . import Account
 
 FLAGS = b"FLAGS"
 FETCH = b"FETCH"
@@ -35,28 +36,15 @@ class DirectImap:
         self.connect()
 
     def connect(self):
+        # Assume the testing server supports TLS on port 993.
         host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
+        port = 993
 
         user = self.account.get_config("addr")
+        host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        if security == const.SocketSecurity.PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.SocketSecurity.STARTTLS:
-            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-        elif security == const.SocketSecurity.PLAIN or security == const.SocketSecurity.SSL:
-            self.conn = MailBox(host, port, ssl_context=ssl_context)
+        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -2,7 +2,7 @@ import json
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Optional, Union
 
-from ._utils import AttrDict
+from ._utils import AttrDict, futuremethod
 from .const import EventType
 from .contact import Contact
 
@@ -70,3 +70,11 @@ class Message:
             event = self.account.wait_for_event()
             if event.kind == EventType.MSG_DELIVERED and event.msg_id == self.id:
                 break
+
+    @futuremethod
+    def send_webxdc_realtime_advertisement(self):
+        yield self._rpc.send_webxdc_realtime_advertisement.future(self.account.id, self.id)
+
+    @futuremethod
+    def send_webxdc_realtime_data(self, data) -> None:
+        yield self._rpc.send_webxdc_realtime_data.future(self.account.id, self.id, list(data))

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -114,13 +114,13 @@ class ACFactory:
         return to_client.run_until(lambda e: e.kind == EventType.INCOMING_MSG)
 
 
-@pytest.fixture()
+@pytest.fixture
 def rpc(tmp_path) -> AsyncGenerator:
     rpc_server = Rpc(accounts_dir=str(tmp_path / "accounts"))
     with rpc_server:
         yield rpc_server
 
 
-@pytest.fixture()
+@pytest.fixture
 def acfactory(rpc) -> AsyncGenerator:
     return ACFactory(DeltaChat(rpc))

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -126,8 +126,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
 
     alice.set_config("download_limit", "1")
 
-    event = bob.wait_for_incoming_msg_event()
-    msg = bob.get_message_by_id(event.msg_id)
+    msg = bob.wait_for_incoming_msg()
     chat_id = msg.get_snapshot().chat_id
     msg.get_snapshot().chat.accept()
     bob.get_chat_by_id(chat_id).send_message(
@@ -135,13 +134,15 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
         html=base64.b64encode(os.urandom(300000)).decode("utf-8"),
     )
 
-    msg_id = alice.wait_for_incoming_msg_event().msg_id
-
-    assert alice.get_message_by_id(msg_id).get_snapshot().download_state == const.DownloadState.AVAILABLE
+    message = alice.wait_for_incoming_msg()
+    snapshot = message.get_snapshot()
+    assert snapshot.download_state == const.DownloadState.AVAILABLE
 
     alice.clear_all_events()
-    chat_id = alice.get_message_by_id(msg_id).get_snapshot().chat_id
-    alice._rpc.download_full_message(alice.id, msg_id)
+
+    snapshot = message.get_snapshot()
+    chat_id = snapshot.chat_id
+    alice._rpc.download_full_message(alice.id, message.id)
 
     wait_for_chatlist_specific_item(alice, chat_id)
 
@@ -177,8 +178,7 @@ def test_imap_sync_seen_msgs(acfactory: ACFactory) -> None:
 
     alice_chat_bob.send_text("hello")
 
-    event = bob.wait_for_incoming_msg_event()
-    msg = bob.get_message_by_id(event.msg_id)
+    msg = bob.wait_for_incoming_msg()
     bob_chat_id = msg.get_snapshot().chat_id
     msg.get_snapshot().chat.accept()
 
@@ -189,8 +189,7 @@ def test_imap_sync_seen_msgs(acfactory: ACFactory) -> None:
     # make sure alice_second_device already received the message
     alice_second_device.wait_for_incoming_msg_event()
 
-    event = alice.wait_for_incoming_msg_event()
-    msg = alice.get_message_by_id(event.msg_id)
+    msg = alice.wait_for_incoming_msg()
     alice_second_device.clear_all_events()
     msg.mark_seen()
 
@@ -211,6 +210,7 @@ def test_multidevice_sync_chat(acfactory: ACFactory) -> None:
     alice_second_device.clear_all_events()
     alice_chat_bob.pin()
     wait_for_chatlist_specific_item(alice_second_device, alice_chat_bob.id)
+    assert alice_second_device.get_chat_by_id(alice_chat_bob.id).get_basic_snapshot().pinned
 
     alice_second_device.clear_all_events()
     alice_chat_bob.mute()

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+"""
+Testing webxdc iroh connectivity
+
+If you want to debug iroh at rust-trace/log level set
+
+    RUST_LOG=iroh_net=trace,iroh_gossip=trace
+"""
+
+import logging
+import os
+import threading
+import time
+
+import pytest
+
+from deltachat_rpc_client import EventType
+
+
+@pytest.fixture
+def path_to_webxdc(request):
+    p = request.path.parent.parent.parent.joinpath("test-data/webxdc/chess.xdc")
+    assert p.exists()
+    return str(p)
+
+
+def log(msg):
+    logging.info(msg)
+
+
+def setup_realtime_webxdc(ac1, ac2, path_to_webxdc):
+    assert ac1.get_config("webxdc_realtime_enabled") == "1"
+    assert ac2.get_config("webxdc_realtime_enabled") == "1"
+    ac1_ac2_chat = ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+
+    # share a webxdc app between ac1 and ac2
+    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="play", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
+    assert ac2_webxdc_msg.get_snapshot().text == "play"
+
+    # send iroh announcements simultaneously
+    log("sending ac1 -> ac2 realtime advertisement and additional message")
+    ac1_webxdc_msg.send_webxdc_realtime_advertisement()
+
+    log("sending ac2 -> ac1 realtime advertisement and additional message")
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+
+    return ac1_webxdc_msg, ac2_webxdc_msg
+
+
+def setup_thread_send_realtime_data(msg, data):
+    def thread_run():
+        for _i in range(10):
+            msg.send_webxdc_realtime_data(data)
+            time.sleep(1)
+
+    threading.Thread(target=thread_run, daemon=True).start()
+
+
+def wait_receive_realtime_data(msg_data_list):
+    account = msg_data_list[0][0].account
+    msg_data_list = msg_data_list[:]
+
+    log(f"account {account.id}: waiting for realtime data {msg_data_list}")
+    while msg_data_list:
+        event = account.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            for i, (msg, data) in enumerate(msg_data_list):
+                if msg.id == event.msg_id:
+                    assert list(data) == event.data
+                    log(f"msg {msg.id}: got correct realtime data {data}")
+                    del msg_data_list[i]
+                    break
+
+
+def test_realtime_sequentially(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection sequentially."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+    ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+
+    # share a webxdc app between ac1 and ac2
+    ac1_webxdc_msg = acfactory.send_message(from_account=ac1, to_account=ac2, text="play", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    snapshot = ac2_webxdc_msg.get_snapshot()
+    assert snapshot.text == "play"
+
+    # send iroh announcements sequentially
+    log("sending ac1 -> ac2 realtime advertisement and additional message")
+    ac1_webxdc_msg.send_webxdc_realtime_advertisement()
+    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping1")
+
+    log("waiting for incoming message on ac2")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping1"
+
+    log("sending ac2 -> ac1 realtime advertisement and additional message")
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+    acfactory.send_message(from_account=ac2, to_account=ac1, text="ping2")
+
+    log("waiting for incoming message on ac1")
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping2"
+
+    log("sending realtime data ac1 -> ac2")
+    # Test that 128 KB of data can be sent in a single message.
+    data = os.urandom(128000)
+    ac1_webxdc_msg.send_webxdc_realtime_data(data)
+
+    log("ac2: waiting for realtime data")
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            assert event.data == list(data)
+            break
+
+
+def test_realtime_simultaneously(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection simultaneously."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, [10])
+    wait_receive_realtime_data([(ac2_webxdc_msg, [10])])
+
+
+def test_two_parallel_realtime_simultaneously(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection simultaneously."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+    ac1_webxdc_msg2, ac2_webxdc_msg2 = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, [10])
+    setup_thread_send_realtime_data(ac1_webxdc_msg2, [20])
+    setup_thread_send_realtime_data(ac2_webxdc_msg, [30])
+    setup_thread_send_realtime_data(ac2_webxdc_msg2, [40])
+
+    wait_receive_realtime_data([(ac1_webxdc_msg, [30]), (ac1_webxdc_msg2, [40])])
+    wait_receive_realtime_data([(ac2_webxdc_msg, [10]), (ac2_webxdc_msg2, [20])])
+
+
+def test_no_duplicate_messages(acfactory, path_to_webxdc):
+    """Test that messages are received only once."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_ac2_chat = ac1.create_chat(ac2)
+
+    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="webxdc", file=path_to_webxdc)
+
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
+    ac2_webxdc_msg.get_snapshot().chat.accept()
+    assert ac2_webxdc_msg.get_snapshot().text == "webxdc"
+
+    # Issue a "send" call in parallel with sending advertisement.
+    # Previously due to a bug this caused subscribing to the channel twice.
+    ac2_webxdc_msg.send_webxdc_realtime_data.future(b"foobar")
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+
+    def thread_run():
+        for i in range(10):
+            data = str(i).encode()
+            ac1_webxdc_msg.send_webxdc_realtime_data(data)
+            time.sleep(1)
+
+    threading.Thread(target=thread_run, daemon=True).start()
+
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            n = int(bytes(event.data).decode())
+            break
+
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            assert int(bytes(event.data).decode()) > n
+            break
+
+
+def test_no_reordering(acfactory, path_to_webxdc):
+    """Test that sending a lot of realtime messages does not result in reordering."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, b"hello")
+    wait_receive_realtime_data([(ac2_webxdc_msg, b"hello")])
+
+    for i in range(200):
+        ac1_webxdc_msg.send_webxdc_realtime_data([i])
+
+    for i in range(200):
+        while 1:
+            event = ac2.wait_for_event()
+            if event.kind == EventType.WEBXDC_REALTIME_DATA and bytes(event.data) != b"hello":
+                if event.data[0] == i:
+                    break
+                pytest.fail("Reordering detected")
+
+
+def test_advertisement_after_chatting(acfactory, path_to_webxdc):
+    """Test that realtime advertisement is assigned to the correct message after chatting."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_ac2_chat = ac1.create_chat(ac2)
+    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="WebXDC", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
+    assert ac2_webxdc_msg.get_snapshot().text == "WebXDC"
+
+    ac1_ac2_chat.send_text("Hello!")
+    ac2_hello_msg = ac2.wait_for_incoming_msg()
+    ac2_hello_msg_snapshot = ac2_hello_msg.get_snapshot()
+    assert ac2_hello_msg_snapshot.text == "Hello!"
+    ac2_hello_msg_snapshot.chat.accept()
+
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+    while 1:
+        event = ac1.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED:
+            assert event.msg_id == ac1_webxdc_msg.id
+            break

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,13 +1,15 @@
 import logging
+import time
 
 import pytest
+
 from deltachat_rpc_client import Chat, EventType, SpecialContactId
 
 
 def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
 
     alice.wait_for_securejoin_inviter_success()
@@ -30,23 +32,45 @@ def test_qr_setup_contact(acfactory, tmp_path) -> None:
     bob2.export_self_keys(tmp_path)
 
     logging.info("Bob imports a key")
-    bob.import_self_keys(tmp_path / "private-key-default.asc")
+    bob.import_self_keys(tmp_path)
 
     assert bob.get_config("key_id") == "2"
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert not bob_contact_alice_snapshot.is_verified
 
 
-@pytest.mark.parametrize("protect", [True, False])
-def test_qr_securejoin(acfactory, protect):
-    alice, bob = acfactory.get_online_accounts(2)
+def test_qr_setup_contact_svg(acfactory) -> None:
+    alice = acfactory.new_configured_account()
+    _, _, domain = alice.get_config("addr").rpartition("@")
 
-    logging.info("Alice creates a verified group")
-    alice_chat = alice.create_group("Verified group", protect=protect)
+    _qr_code, svg = alice.get_qr_code_svg()
+
+    alice.set_config("displayname", "Alice")
+
+    # Test that display name is used
+    # in SVG and no address is visible.
+    _qr_code, svg = alice.get_qr_code_svg()
+    assert domain not in svg
+    assert "Alice" in svg
+
+
+@pytest.mark.parametrize("protect", [True, False])
+def test_qr_securejoin(acfactory, protect, tmp_path):
+    alice, bob, fiona = acfactory.get_online_accounts(3)
+
+    # Setup second device for Alice
+    # to test observing securejoin protocol.
+    alice.export_backup(tmp_path)
+    files = list(tmp_path.glob("*.tar"))
+    alice2 = acfactory.get_unconfigured_account()
+    alice2.import_backup(files[0])
+
+    logging.info("Alice creates a group")
+    alice_chat = alice.create_group("Group", protect=protect)
     assert alice_chat.get_basic_snapshot().is_protected == protect
 
-    logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
+    logging.info("Bob joins the group")
+    qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
 
     # Check that at least some of the handshake messages are deleted.
@@ -74,6 +98,21 @@ def test_qr_securejoin(acfactory, protect):
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
+    # Start second Alice device.
+    # Alice observes securejoin protocol and verifies Bob on second device.
+    alice2.start_io()
+    alice2.wait_for_securejoin_inviter_success()
+    alice2_contact_bob = alice2.get_contact_by_addr(bob.get_config("addr"))
+    alice2_contact_bob_snapshot = alice2_contact_bob.get_snapshot()
+    assert alice2_contact_bob_snapshot.is_verified
+
+    # The QR code token is synced, so alice2 must be able to handle join requests.
+    logging.info("Fiona joins the group via alice2")
+    alice.stop_io()
+    fiona.secure_join(qr_code)
+    alice2.wait_for_securejoin_inviter_success()
+    fiona.wait_for_securejoin_joiner_success()
+
 
 def test_qr_securejoin_contact_request(acfactory) -> None:
     """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
@@ -91,7 +130,7 @@ def test_qr_securejoin_contact_request(acfactory) -> None:
 
     alice_chat = alice.create_group("Verified group", protect=True)
     logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
     while True:
         event = bob.wait_for_event()
@@ -106,7 +145,7 @@ def test_qr_readreceipt(acfactory) -> None:
     alice, bob, charlie = acfactory.get_online_accounts(3)
 
     logging.info("Bob and Charlie setup contact with Alice")
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
 
     bob.secure_join(qr_code)
     charlie.secure_join(qr_code)
@@ -168,13 +207,13 @@ def test_setup_contact_resetup(acfactory) -> None:
     """Tests that setup contact works after Alice resets the device and changes the key."""
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     alice = acfactory.resetup_account(alice)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -188,7 +227,7 @@ def test_verified_group_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -205,7 +244,7 @@ def test_verified_group_recovery(acfactory) -> None:
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
+    qr_code = ac3.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -252,7 +291,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -269,7 +308,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
+    qr_code = ac3.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -287,7 +326,6 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
 
     ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
     ac3_chat.remove_contact(ac3_contact_ac2)
-    ac3_chat.add_contact(ac3_contact_ac2)
 
     msg_id = ac2.wait_for_incoming_msg_event().msg_id
     message = ac2.get_message_by_id(msg_id)
@@ -297,6 +335,8 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert "removed" in snapshot.text
 
+    ac3_chat.add_contact(ac3_contact_ac2)
+
     event = ac2.wait_for_incoming_msg_event()
     msg_id = event.msg_id
     chat_id = event.chat_id
@@ -336,7 +376,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
     ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
 
     logging.info("ac3: verify with ac2")
-    qr_code, _svg = ac2.get_qr_code()
+    qr_code = ac2.get_qr_code()
     ac3.secure_join(qr_code)
     ac2.wait_for_securejoin_inviter_success()
 
@@ -346,7 +386,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
 
     logging.info("ac1: create verified group that ac2 fully joins")
     ch1 = ac1.create_group("Group", protect=True)
-    qr_code, _svg = ch1.get_qr_code()
+    qr_code = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
 
@@ -359,7 +399,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
-    qr_code, _svg = ch1.get_qr_code()
+    qr_code = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.remove()
     logging.info("ac2 now has pending bobstate but ac1 is shutoff")
@@ -381,7 +421,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
-    qr_code, _svg = vg.get_qr_code()
+    qr_code = vg.get_qr_code()
     ac4.secure_join(qr_code)
     ac3.wait_for_securejoin_inviter_success()
     while 1:
@@ -402,7 +442,7 @@ def test_qr_new_group_unblocked(acfactory):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_chat = ac1.create_group("Group for joining", protect=True)
-    qr_code, _svg = ac1_chat.get_qr_code()
+    qr_code = ac1_chat.get_qr_code()
     ac2.secure_join(qr_code)
 
     ac1.wait_for_securejoin_inviter_success()
@@ -420,15 +460,19 @@ def test_qr_new_group_unblocked(acfactory):
 
 def test_aeap_flow_verified(acfactory):
     """Test that a new address is added to a contact when it changes its address."""
-    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    # ac1new is only used to get a new address.
+    ac1new = acfactory.new_preconfigured_account()
 
     logging.info("ac1: create verified-group QR, ac2 scans and joins")
     chat = ac1.create_group("hello", protect=True)
     assert chat.get_basic_snapshot().is_protected
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     logging.info("ac2: start QR-code based join-group protocol")
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("sending first message")
     msg_out = chat.send_text("old address").get_snapshot()
@@ -464,12 +508,12 @@ def test_gossip_verification(acfactory) -> None:
     alice, bob, carol = acfactory.get_online_accounts(3)
 
     # Bob verifies Alice.
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     # Bob verifies Carol.
-    qr_code, _svg = carol.get_qr_code()
+    qr_code = carol.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -520,16 +564,17 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     ac3_chat = ac3.create_group("Verified group", protect=True)
 
     # ac1 joins ac3 group.
-    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac3_qr_code = ac3_chat.get_qr_code()
     ac1.secure_join(ac3_qr_code)
     ac1.wait_for_securejoin_joiner_success()
 
     # ac1 waits for member added message and creates a QR code.
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+    assert snapshot.text == "Member Me ({}) added by {}.".format(ac1.get_config("addr"), ac3.get_config("addr"))
+    ac1_qr_code = snapshot.chat.get_qr_code()
 
     # ac2 verifies ac1
-    qr_code, _svg = ac1.get_qr_code()
+    qr_code = ac1.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -540,17 +585,29 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     # ac1 resetups the account.
     ac1 = acfactory.resetup_account(ac1)
 
-    # ac1 sends a message to ac2.
-    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
-    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
-    ac1_chat_ac2.send_text("Hello!")
+    # Loop sending message from ac1 to ac2
+    # until ac2 accepts new ac1 key.
+    #
+    # This may not happen immediately because resetup of ac1
+    # rewinds "smeared timestamp" so Date: header for messages
+    # sent by new ac1 are in the past compared to the last Date:
+    # header sent by old ac1.
+    while True:
+        # ac1 sends a message to ac2.
+        ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
+        ac1_chat_ac2 = ac1_contact_ac2.create_chat()
+        ac1_chat_ac2.send_text("Hello!")
 
-    # ac2 receives a message.
-    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Hello!"
+        # ac2 receives a message.
+        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        assert snapshot.text == "Hello!"
+        logging.info("ac2 received Hello!")
 
-    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
-    assert not ac2_contact_ac1.get_snapshot().is_verified
+        # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
+        logging.info("ac2 addr={}, ac1 addr={}".format(ac2.get_config("addr"), ac1.get_config("addr")))
+        if not ac2_contact_ac1.get_snapshot().is_verified:
+            break
+        time.sleep(1)
 
     # ac1 goes offline.
     ac1.remove()
@@ -589,7 +646,7 @@ def test_withdraw_securejoin_qr(acfactory):
     assert alice_chat.get_basic_snapshot().is_protected
     logging.info("Bob joins verified group")
 
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob_chat = bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -612,7 +669,8 @@ def test_withdraw_securejoin_qr(acfactory):
     logging.info("Bob scanned withdrawn QR code")
     while True:
         event = alice.wait_for_event()
-        if event.kind == EventType.MSGS_CHANGED and event.chat_id != 0:
+        if (
+            event.kind == EventType.WARNING
+            and "Ignoring vg-request-with-auth message because of invalid auth code." in event.msg
+        ):
             break
-    snapshot = alice.get_message_by_id(event.msg_id).get_snapshot()
-    assert snapshot.text == "Cannot establish guaranteed end-to-end encryption with {}".format(bob.get_config("addr"))

deltachat-rpc-client/tests/test_something.py

@@ -1,12 +1,15 @@
+import base64
 import concurrent.futures
 import json
 import logging
 import os
+import socket
 import subprocess
 import time
 from unittest.mock import MagicMock
 
 import pytest
+
 from deltachat_rpc_client import Contact, EventType, Message, events
 from deltachat_rpc_client.const import DownloadState, MessageState
 from deltachat_rpc_client.direct_imap import DirectImap
@@ -54,8 +57,8 @@ def test_acfactory(acfactory) -> None:
             if event.progress == 1000:  # Success
                 break
         else:
-            print(event)
-    print("Successful configuration")
+            logging.info(event)
+    logging.info("Successful configuration")
 
 
 def test_configure_starttls(acfactory) -> None:
@@ -68,6 +71,38 @@ def test_configure_starttls(acfactory) -> None:
     assert account.is_configured()
 
 
+def test_configure_ip(acfactory) -> None:
+    account = acfactory.new_preconfigured_account()
+
+    domain = account.get_config("addr").rsplit("@")[-1]
+    ip_address = socket.gethostbyname(domain)
+
+    # This should fail TLS check.
+    account.set_config("mail_server", ip_address)
+    with pytest.raises(JsonRpcError):
+        account.configure()
+
+
+def test_configure_alternative_port(acfactory) -> None:
+    """Test that configuration with alternative port 443 works."""
+    account = acfactory.new_preconfigured_account()
+
+    account.set_config("mail_port", "443")
+    account.set_config("send_port", "443")
+
+    account.configure()
+
+
+def test_configure_username(acfactory) -> None:
+    account = acfactory.new_preconfigured_account()
+
+    addr = account.get_config("addr")
+    account.set_config("mail_user", addr)
+    account.configure()
+
+    assert account.get_config("configured_mail_user") == addr
+
+
 def test_account(acfactory) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
@@ -103,12 +138,12 @@ def test_account(acfactory) -> None:
     assert alice.get_chatlist(snapshot=True)
     assert alice.get_qr_code()
     assert alice.get_fresh_messages()
-    assert alice.get_next_messages()
 
     # Test sending empty message.
     assert len(bob.wait_next_messages()) == 0
     alice_chat_bob.send_text("")
     messages = bob.wait_next_messages()
+    assert bob.get_next_messages() == messages
     assert len(messages) == 1
     message = messages[0]
     snapshot = message.get_snapshot()
@@ -211,6 +246,7 @@ def test_contact(acfactory) -> None:
     assert repr(alice_contact_bob)
     alice_contact_bob.block()
     alice_contact_bob.unblock()
+    alice_contact_bob.reset_encryption()
     alice_contact_bob.set_name("new name")
     alice_contact_bob.get_encryption_info()
     snapshot = alice_contact_bob.get_snapshot()
@@ -398,7 +434,7 @@ def test_provider_info(rpc) -> None:
     assert provider_info["id"] == "gmail"
 
     # Disable MX record resolution.
-    rpc.set_config(account_id, "socks5_enabled", "1")
+    rpc.set_config(account_id, "proxy_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
 
@@ -613,3 +649,31 @@ def test_markseen_contact_request(acfactory, tmp_path):
         if event.kind == EventType.MSGS_NOTICED:
             break
     assert message2.get_snapshot().state == MessageState.IN_SEEN
+
+
+def test_get_http_response(acfactory):
+    alice = acfactory.new_configured_account()
+    http_response = alice._rpc.get_http_response(alice.id, "https://example.org")
+    assert http_response["mimetype"] == "text/html"
+    assert b"<title>Example Domain</title>" in base64.b64decode((http_response["blob"] + "==").encode())
+
+
+def test_configured_imap_certificate_checks(acfactory):
+    alice = acfactory.new_configured_account()
+    configured_certificate_checks = alice.get_config("configured_imap_certificate_checks")
+
+    # Certificate checks should be configured (not None)
+    assert configured_certificate_checks
+
+    # 0 is the value old Delta Chat core versions used
+    # to mean user entered "imap_certificate_checks=0" (Automatic)
+    # and configuration failed to use strict TLS checks
+    # so it switched strict TLS checks off.
+    #
+    # New versions of Delta Chat are not disabling TLS checks
+    # unless users explicitly disables them
+    # or provider database says provider has invalid certificates.
+    #
+    # Core 1.142.4, 1.142.5 and 1.142.6 saved this value due to bug.
+    # This test is a regression test to prevent this happening again.
+    assert configured_certificate_checks != "0"

deltachat-rpc-client/tests/test_vcard.py

@@ -0,0 +1,15 @@
+def test_vcard(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_contact_charlie = alice.create_contact("charlie@example.org", "Charlie")
+
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_contact(alice_contact_charlie)
+
+    event = bob.wait_for_incoming_msg_event()
+    message = bob.get_message_by_id(event.msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.vcard_contact
+    assert snapshot.vcard_contact.addr == "charlie@example.org"

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.138.5"
+version = "1.150.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -10,18 +10,18 @@ keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
 
 [dependencies]
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
-deltachat = { path = "..", default-features = false }
+deltachat-jsonrpc = { workspace = true }
+deltachat = { workspace = true }
 
-anyhow = "1"
-env_logger = { version = "0.11.3" }
-futures-lite = "2.3.0"
-log = "0.4"
-serde_json = "1"
-serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.37.0", features = ["io-std"] }
-tokio-util = "0.7.9"
-yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+anyhow = { workspace = true }
+futures-lite = { workspace = true }
+log = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+tokio = { workspace = true, features = ["io-std"] }
+tokio-util = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
 
 [features]
 default = ["vendored"]

deltachat-rpc-server/npm-package/README.md

@@ -7,7 +7,7 @@ This simplifies cross-compilation and even reduces binary size (no CFFI layer an
 
 ## Usage
 
-> The **minimum** nodejs version for this package is `20.11`
+> The **minimum** nodejs version for this package is `16`
 
 ```
 npm i @deltachat/stdio-rpc-server @deltachat/jsonrpc-client
@@ -20,7 +20,9 @@ import { C } from "@deltachat/jsonrpc-client";
 async function main() {
     const dc = await startDeltaChat("deltachat-data");
     console.log(await dc.rpc.getSystemInfo());
+    dc.close()
 }
+main()
 ```
 
 For a more complete example refer to https://github.com/deltachat-bot/echo/pull/69/files (TODO change link when pr is merged).
@@ -44,12 +46,11 @@ references:
 When you import this package it searches for the rpc server in the following locations and order:
 
 1. `DELTA_CHAT_RPC_SERVER` environment variable
-2. in PATH
-   - unless `DELTA_CHAT_SKIP_PATH=1` is specified
-   - searches in .cargo/bin directory first
-   - but there an additional version check is performed
+2. use the PATH when `{takeVersionFromPATH: true}` is supplied in the options. 
 3. prebuilds in npm packages
 
+so by default it uses the prebuilds.
+
 ## How do you built this package in CI
 
 - To build platform packages, run the `build_platform_package.py` script:

deltachat-rpc-server/npm-package/index.d.ts

@@ -1,8 +1,8 @@
 import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
 
 export interface SearchOptions {
-  /** whether to disable looking for deltachat-rpc-server inside of $PATH */
-  skipSearchInPath: boolean;
+  /** whether take deltachat-rpc-server inside of $PATH*/
+  takeVersionFromPATH: boolean;
 
   /** whether to disable the DELTA_CHAT_RPC_SERVER environment variable */
   disableEnvPath: boolean;

deltachat-rpc-server/npm-package/index.js

@@ -1,15 +1,9 @@
 //@ts-check
-import { execFile, spawn } from "node:child_process";
-import { stat, readdir } from "node:fs/promises";
+import { spawn } from "node:child_process";
+import { stat } from "node:fs/promises";
 import os from "node:os";
-import { join, basename } from "node:path";
 import process from "node:process";
-import { promisify } from "node:util";
-import {
-  ENV_VAR_NAME,
-  PATH_EXECUTABLE_NAME,
-  SKIP_SEARCH_IN_PATH,
-} from "./src/const.js";
+import { ENV_VAR_NAME, PATH_EXECUTABLE_NAME } from "./src/const.js";
 import {
   ENV_VAR_LOCATION_NOT_FOUND,
   FAILED_TO_START_SERVER_EXECUTABLE,
@@ -17,9 +11,6 @@ import {
   NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR,
 } from "./src/errors.js";
 
-// Because this is not compiled by typescript, esm needs this stuff (` with { type: "json" };`,
-// nodejs still complains about it being experimental, but deno also uses it, so treefit bets taht it will become standard)
-import package_json from "./package.json" with { type: "json" };
 import { createRequire } from "node:module";
 
 function findRPCServerInNodeModules() {
@@ -31,7 +22,12 @@ function findRPCServerInNodeModules() {
     return resolve(package_name);
   } catch (error) {
     console.debug("findRpcServerInNodeModules", error);
-    if (Object.keys(package_json.optionalDependencies).includes(package_name)) {
+    const require = createRequire(import.meta.url);
+    if (
+      Object.keys(require("./package.json").optionalDependencies).includes(
+        package_name
+      )
+    ) {
       throw new Error(NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name));
     } else {
       throw new Error(NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR());
@@ -39,38 +35,13 @@ function findRPCServerInNodeModules() {
   }
 }
 
-/**
- * @returns {Promise<string>}
- */
-async function getLocationInPath() {
-  const exec = promisify(execFile);
-
-  if (os.platform() === "win32") {
-    const { stdout: executable } = await exec("where", [PATH_EXECUTABLE_NAME], {
-      shell: true,
-    });
-    return executable;
-  }
-
-  try {
-    const { stdout: executable } = await exec(
-      "command",
-      ["-v", PATH_EXECUTABLE_NAME],
-      { shell: true }
-    );
-    return executable;
-  } catch (error) {
-    if (error.code > 0) return "";
-    else throw error;
-  }
-}
-
 /** @type {import("./index").FnTypes.getRPCServerPath} */
-export async function getRPCServerPath(
-  options = { skipSearchInPath: false, disableEnvPath: false }
-) {
-  // @TODO: improve confusing naming of these options
-  const { skipSearchInPath, disableEnvPath } = options;
+export async function getRPCServerPath(options = {}) {
+  const { takeVersionFromPATH, disableEnvPath } = {
+    takeVersionFromPATH: false,
+    disableEnvPath: false,
+    ...options,
+  };
   // 1. check if it is set as env var
   if (process.env[ENV_VAR_NAME] && !disableEnvPath) {
     try {
@@ -85,35 +56,9 @@ export async function getRPCServerPath(
     return process.env[ENV_VAR_NAME];
   }
 
-  // 2. check if it can be found in PATH
-  if (!process.env[SKIP_SEARCH_IN_PATH] && !skipSearchInPath) {
-    const executable = await getLocationInPath();
-
-    // by just trying to execute it and then use "command -v deltachat-rpc-server" (unix) or "where deltachat-rpc-server" (windows) to get the path to the executable
-    if (executable.length > 1) {
-      // test if it is the right version
-      try {
-        // for some unknown reason it is in stderr and not in stdout
-        const { stderr } = await promisify(execFile)(
-          executable,
-          ["--version"],
-          { shell: true }
-        );
-        const version = stderr.slice(0, stderr.indexOf("\n"));
-        if (package_json.version !== version) {
-          throw new Error(
-            `version mismatch: (npm package: ${package_json.version})  (installed ${PATH_EXECUTABLE_NAME} version: ${version})`
-          );
-        } else {
-          return executable;
-        }
-      } catch (error) {
-        console.error(
-          "Found executable in PATH, but there was an error: " + error
-        );
-        console.error("So falling back to using prebuild...");
-      }
-    }
+  // 2. check if PATH should be used
+  if (takeVersionFromPATH) {
+    return PATH_EXECUTABLE_NAME;
   }
   // 3. check for prebuilds
 
@@ -123,11 +68,11 @@ export async function getRPCServerPath(
 import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
 
 /** @type {import("./index").FnTypes.startDeltaChat} */
-export async function startDeltaChat(directory, options) {
+export async function startDeltaChat(directory, options = {}) {
   const pathToServerBinary = await getRPCServerPath(options);
   const server = spawn(pathToServerBinary, {
     env: {
-      RUST_LOG: process.env.RUST_LOG || "info",
+      RUST_LOG: process.env.RUST_LOG,
       DC_ACCOUNTS_PATH: directory,
     },
     stdio: ["pipe", "pipe", options.muteStdErr ? "ignore" : "inherit"],

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "1.138.5"
+  "version": "1.150.0"
 }

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -55,9 +55,10 @@ for (const { folder_name, package_name } of platform_package_names) {
 }
 
 if (is_local) {
-  package_json.peerDependencies["@deltachat/jsonrpc-client"] = 'file:../../deltachat-jsonrpc/typescript'
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] =
+    `file:${join(expected_cwd, "/../../deltachat-jsonrpc/typescript")}`;
 } else {
-  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*"
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*";
 }
 
 await fs.writeFile("./package.json", JSON.stringify(package_json, null, 4));

deltachat-rpc-server/npm-package/src/const.js

@@ -2,5 +2,4 @@
 
 export const PATH_EXECUTABLE_NAME = 'deltachat-rpc-server'
 
-export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"
-export const SKIP_SEARCH_IN_PATH = "DELTA_CHAT_SKIP_PATH"
+export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"

deltachat-rpc-server/src/main.rs

@@ -11,6 +11,7 @@ use deltachat::constants::DC_VERSION_STR;
 use deltachat_jsonrpc::api::{Accounts, CommandApi};
 use futures_lite::stream::StreamExt;
 use tokio::io::{self, AsyncBufReadExt, BufReader};
+use tracing_subscriber::EnvFilter;
 use yerpc::RpcServer as _;
 
 #[cfg(target_family = "unix")]
@@ -28,6 +29,9 @@ async fn main() {
     // "For technical reasons, stdin is implemented by using an ordinary blocking read on a separate
     // thread, and it is impossible to cancel that read. This can make shutdown of the runtime hang
     // until the user presses enter."
+    if let Err(error) = &r {
+        log::error!("Fatal error: {error:#}.")
+    }
     std::process::exit(if r.is_ok() { 0 } else { 1 });
 }
 
@@ -60,7 +64,13 @@ async fn main_impl() -> Result<()> {
     #[cfg(target_family = "unix")]
     let mut sigterm = signal_unix::signal(signal_unix::SignalKind::terminate())?;
 
-    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+    // Logs from `log` crate and traces from `tracing` crate
+    // are configurable with `RUST_LOG` environment variable
+    // and go to stderr to avoid interferring with JSON-RPC using stdout.
+    tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::from_default_env())
+        .with_writer(std::io::stderr)
+        .init();
 
     let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "accounts".to_string());
     log::info!("Starting with accounts directory `{}`.", path);

deny.toml

@@ -1,7 +1,6 @@
 [advisories]
 ignore = [
     "RUSTSEC-2020-0071",
-    "RUSTSEC-2022-0093",
 
     # Timing attack on RSA.
     # Delta Chat does not use RSA for new keys
@@ -10,11 +9,15 @@ ignore = [
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
 
-    # Unmaintained ansi_term
-    "RUSTSEC-2021-0139",
-
     # Unmaintained encoding
     "RUSTSEC-2021-0153",
+
+    # Unmaintained proc-macro-error
+    # <https://rustsec.org/advisories/RUSTSEC-2024-0370>
+    "RUSTSEC-2024-0370",
+
+    # Unmaintained instant
+    "RUSTSEC-2024-0384",
 ]
 
 [bans]
@@ -23,86 +26,46 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
-     { name = "asn1-rs-derive", version = "0.4.0" },
-     { name = "asn1-rs-impl", version = "0.1.0" },
-     { name = "asn1-rs", version = "0.5.2" },
      { name = "async-channel", version = "1.9.0" },
-     { name = "base16ct", version = "0.1.1" },
      { name = "base64", version = "<0.21" },
      { name = "base64", version = "0.21.7" },
      { name = "bitflags", version = "1.3.2" },
-     { name = "block-buffer", version = "<0.10" },
-     { name = "convert_case", version = "0.4.0" },
-     { name = "curve25519-dalek", version = "3.2.0" },
-     { name = "darling_core", version = "<0.14" },
-     { name = "darling_macro", version = "<0.14" },
-     { name = "darling", version = "<0.14" },
-     { name = "der_derive", version = "0.6.1" },
-     { name = "derive_more", version = "0.99.17" },
-     { name = "der-parser", version = "8.2.0" },
-     { name = "der", version = "0.6.1" },
-     { name = "digest", version = "<0.10" },
-     { name = "dlopen2", version = "0.4.1" },
-     { name = "ed25519-dalek", version = "1.0.1" },
-     { name = "ed25519", version = "1.5.3" },
-     { name = "env_logger", version = "0.10.2" },
      { name = "event-listener", version = "2.5.3" },
      { name = "event-listener", version = "4.0.3" },
      { name = "fastrand", version = "1.9.0" },
+     { name = "fiat-crypto", version = "0.1.20" },
      { name = "futures-lite", version = "1.13.0" },
      { name = "getrandom", version = "<0.2" },
-     { name = "http-body", version = "0.4.6" },
      { name = "http", version = "0.2.12" },
-     { name = "hyper", version = "0.14.28" },
-     { name = "idna", version = "0.4.0" },
-     { name = "netlink-packet-core", version = "0.5.0" },
-     { name = "netlink-packet-route", version = "0.15.0" },
      { name = "nix", version = "0.26.4" },
-     { name = "oid-registry", version = "0.6.1" },
-     { name = "pem-rfc7468", version = "0.6.0" },
-     { name = "pem", version = "1.1.1" },
-     { name = "pkcs8", version = "0.9.0" },
-     { name = "proc-macro-error-attr", version = "0.4.12" },
-     { name = "proc-macro-error", version = "0.4.12" },
+     { name = "num_enum_derive", version = "0.5.11" },
+     { name = "num_enum", version = "0.5.11" },
+     { name = "proc-macro-crate", version = "1.3.1" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
-     { name = "rcgen", version = "<0.12.1" },
      { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
-     { name = "ring", version = "0.16.20" },
-     { name = "sec1", version = "0.3.0" },
-     { name = "sha2", version = "<0.10" },
-     { name = "signature", version = "1.6.4" },
-     { name = "spin", version = "<0.9.6" },
-     { name = "spki", version = "0.6.0" },
-     { name = "ssh-encoding", version = "0.1.0" },
-     { name = "ssh-key", version = "0.5.1" },
      { name = "sync_wrapper", version = "0.1.2" },
-     { name = "synstructure", version = "0.12.6" },
      { name = "syn", version = "1.0.109" },
-     { name = "system-configuration-sys", version = "0.5.0" },
-     { name = "system-configuration", version = "0.5.1" },
      { name = "time", version = "<0.3" },
-     { name = "toml_edit", version = "0.21.1" },
-     { name = "untrusted", version = "0.7.1" },
+     { name = "toml_edit", version = "0.19.15" },
      { name = "wasi", version = "<0.11" },
      { name = "windows_aarch64_gnullvm", version = "<0.52" },
      { name = "windows_aarch64_msvc", version = "<0.52" },
      { name = "windows-core", version = "<0.54.0" },
      { name = "windows_i686_gnu", version = "<0.52" },
      { name = "windows_i686_msvc", version = "<0.52" },
-     { name = "windows-sys", version = "<0.52" },
+     { name = "windows-sys", version = "<0.59" },
      { name = "windows-targets", version = "<0.52" },
-     { name = "windows", version = "0.32.0" },
      { name = "windows", version = "<0.54.0" },
      { name = "windows_x86_64_gnullvm", version = "<0.52" },
      { name = "windows_x86_64_gnu", version = "<0.52" },
      { name = "windows_x86_64_msvc", version = "<0.52" },
      { name = "winnow", version = "0.5.40" },
-     { name = "x509-parser", version = "<0.16.0" },
+     { name = "winreg", version = "0.50.0" },
 ]
 
 

flake.lock

@@ -3,15 +3,15 @@
     "android": {
       "inputs": {
         "devshell": "devshell",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1712088936,
-        "narHash": "sha256-mVjeSWQiR/t4UZ9fUawY9OEPAhY1R3meYG+0oh8DUBs=",
+        "lastModified": 1731356359,
+        "narHash": "sha256-vYqJnu6jotmWpPT4DgzHVdvNIZcKZCIUqS8QaptsZA0=",
         "owner": "tadfisher",
         "repo": "android-nixpkgs",
-        "rev": "2d8181caef279f19c4a33dc694723f89ffc195d4",
+        "rev": "c028ead7e88edb2e94cd7c90ee37593f63ae494a",
         "type": "github"
       },
       "original": {
@@ -22,18 +22,17 @@
     },
     "devshell": {
       "inputs": {
-        "flake-utils": "flake-utils",
         "nixpkgs": [
           "android",
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1711099426,
-        "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=",
+        "lastModified": 1728330715,
+        "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
         "owner": "numtide",
         "repo": "devshell",
-        "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8",
+        "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
         "type": "github"
       },
       "original": {
@@ -48,11 +47,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1714112748,
-        "narHash": "sha256-jq6Cpf/pQH85p+uTwPPrGG8Ky/zUOTwMJ7mcqc5M4So=",
+        "lastModified": 1731393059,
+        "narHash": "sha256-rmzi0GHEwpzg1LGfGPO4SRD7D6QGV3UYGQxkJvn+J5U=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "3ae4b908a795b6a3824d401a0702e11a7157d7e1",
+        "rev": "fda8d5b59bb0dc0021ad3ba1d722f9ef6d36e4d9",
         "type": "github"
       },
       "original": {
@@ -66,11 +65,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1701680307,
-        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -84,29 +83,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_3": {
-      "inputs": {
-        "systems": "systems_3"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -120,11 +101,11 @@
         "nixpkgs": "nixpkgs_3"
       },
       "locked": {
-        "lastModified": 1713520724,
-        "narHash": "sha256-CO8MmVDmqZX2FovL75pu5BvwhW+Vugc7Q6ze7Hj8heI=",
+        "lastModified": 1721727458,
+        "narHash": "sha256-r/xppY958gmZ4oTfLiHN0ZGuQ+RSTijDblVgVLFi1mw=",
         "owner": "nix-community",
         "repo": "naersk",
-        "rev": "c5037590290c6c7dae2e42e7da1e247e54ed2d49",
+        "rev": "3fb418eaf352498f6b6c30592e3beb63df42ef11",
         "type": "github"
       },
       "original": {
@@ -135,11 +116,11 @@
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1710156097,
-        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "lastModified": 1730207686,
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
         "type": "github"
       },
       "original": {
@@ -150,11 +131,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1711703276,
-        "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=",
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -166,11 +147,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1713895582,
-        "narHash": "sha256-cfh1hi+6muQMbi9acOlju3V1gl8BEaZBXBR9jQfQi4U=",
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "572af610f6151fd41c212f897c71f7056e3fb518",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -182,10 +163,9 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1711668574,
-        "narHash": "sha256-u1dfs0ASQIEr1icTVrsKwg2xToIpn7ZXxW3RHfHxshg=",
-        "path": "/nix/store/9fpv0kjq9a80isa1wkkvrdqsh9dpcn05-source",
-        "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659",
+        "lastModified": 0,
+        "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=",
+        "path": "/nix/store/zq2axpgzd5kykk1v446rkffj3bxa2m2h-source",
         "type": "path"
       },
       "original": {
@@ -195,11 +175,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1714076141,
-        "narHash": "sha256-Drmja/f5MRHZCskS6mvzFqxEaZMeciScCTFxWVLqWEY=",
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "7bb2ccd8cdc44c91edba16c48d2c8f331fb3d856",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -213,7 +193,7 @@
       "inputs": {
         "android": "android",
         "fenix": "fenix",
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_2",
         "naersk": "naersk",
         "nix-filter": "nix-filter",
         "nixpkgs": "nixpkgs_4"
@@ -222,11 +202,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1714031783,
-        "narHash": "sha256-xS/niQsq1CQPOe4M4jvVPO2cnXS/EIeRG5gIopUbk+Q=",
+        "lastModified": 1731342671,
+        "narHash": "sha256-36eYDHoPzjavnpmEpc2MXdzMk557S0YooGms07mDuKk=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "56bee2ddafa6177b19c631eedc88d43366553223",
+        "rev": "fc98e0657abf3ce07eed513e38274c89bbb2f8ad",
         "type": "github"
       },
       "original": {
@@ -265,21 +245,6 @@
         "repo": "default",
         "type": "github"
       }
-    },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -18,9 +18,9 @@
         manifest = (pkgs.lib.importTOML ./Cargo.toml).package;
         androidSdk = android.sdk.${system} (sdkPkgs:
           builtins.attrValues {
-            inherit (sdkPkgs) ndk-24-0-8215888 cmdline-tools-latest;
+            inherit (sdkPkgs) ndk-27-0-11902837 cmdline-tools-latest;
           });
-        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/24.0.8215888";
+        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/27.0.11902837";
 
         rustSrc = nix-filter.lib {
           root = ./.;
@@ -257,13 +257,21 @@
 
         androidAttrs = {
           armeabi-v7a = {
-            cc = "armv7a-linux-androideabi19-clang";
+            cc = "armv7a-linux-androideabi21-clang";
             rustTarget = "armv7-linux-androideabi";
           };
           arm64-v8a = {
             cc = "aarch64-linux-android21-clang";
             rustTarget = "aarch64-linux-android";
           };
+          x86 = {
+            cc = "i686-linux-android21-clang";
+            rustTarget = "i686-linux-android";
+          };
+          x86_64 = {
+            cc = "x86_64-linux-android21-clang";
+            rustTarget = "x86_64-linux-android";
+          };
         };
 
         mkAndroidRustPackage = arch: packageName:
@@ -525,27 +533,30 @@
               };
           };
 
-        devShells.default = let 
-          pkgs = import nixpkgs {
-            system = system;
-            overlays = [ fenix.overlays.default ];
-          };
-          in pkgs.mkShell {
+        devShells.default =
+          let
+            pkgs = import nixpkgs {
+              system = system;
+              overlays = [ fenix.overlays.default ];
+            };
+          in
+          pkgs.mkShell {
 
-          buildInputs = with pkgs; [
-            (fenix.packages.${system}.complete.withComponents [
-              "cargo"
-              "clippy"
-              "rust-src"
-              "rustc"
-              "rustfmt"
-            ])
-            cargo-deny
-            rust-analyzer-nightly
-            perl # needed to build vendored OpenSSL
-            git-cliff
-          ];
-        };
+            buildInputs = with pkgs; [
+              (fenix.packages.${system}.complete.withComponents [
+                "cargo"
+                "clippy"
+                "rust-src"
+                "rustc"
+                "rustfmt"
+              ])
+              cargo-deny
+              rust-analyzer-nightly
+              cargo-nextest
+              perl # needed to build vendored OpenSSL
+              git-cliff
+            ];
+          };
       }
     );
 }

format-flowed/src/lib.rs

@@ -8,6 +8,8 @@
 //! is assumed to be set to "no".
 //!
 //! For received messages, DelSp parameter is honoured.
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///

node/constants.js

@@ -1,6 +1,7 @@
 // Generated!
 
 module.exports = {
+  DC_CERTCK_ACCEPT_INVALID: 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES: 3,
   DC_CERTCK_AUTO: 0,
   DC_CERTCK_STRICT: 1,
@@ -30,6 +31,7 @@ module.exports = {
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
+  DC_EVENT_CHANNEL_OVERFLOW: 2400,
   DC_EVENT_CHATLIST_CHANGED: 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED: 2301,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
@@ -49,6 +51,7 @@ module.exports = {
   DC_EVENT_IMEX_PROGRESS: 2051,
   DC_EVENT_INCOMING_MSG: 2005,
   DC_EVENT_INCOMING_MSG_BUNCH: 2006,
+  DC_EVENT_INCOMING_REACTION: 2002,
   DC_EVENT_INFO: 100,
   DC_EVENT_LOCATION_CHANGED: 2035,
   DC_EVENT_MSGS_CHANGED: 2000,
@@ -66,6 +69,8 @@ module.exports = {
   DC_EVENT_SMTP_MESSAGE_SENT: 103,
   DC_EVENT_WARNING: 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED: 2121,
+  DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT: 2151,
+  DC_EVENT_WEBXDC_REALTIME_DATA: 2150,
   DC_EVENT_WEBXDC_STATUS_UPDATE: 2120,
   DC_GCL_ADD_ALLDONE_HINT: 4,
   DC_GCL_ADD_SELF: 2,
@@ -126,11 +131,13 @@ module.exports = {
   DC_QR_ASK_VERIFYCONTACT: 200,
   DC_QR_ASK_VERIFYGROUP: 202,
   DC_QR_BACKUP: 251,
+  DC_QR_BACKUP2: 252,
   DC_QR_ERROR: 400,
   DC_QR_FPR_MISMATCH: 220,
   DC_QR_FPR_OK: 210,
   DC_QR_FPR_WITHOUT_ADDR: 230,
   DC_QR_LOGIN: 520,
+  DC_QR_PROXY: 271,
   DC_QR_REVIVE_VERIFYCONTACT: 510,
   DC_QR_REVIVE_VERIFYGROUP: 512,
   DC_QR_TEXT: 330,

node/events.js

@@ -16,6 +16,7 @@ module.exports = {
   410: 'DC_EVENT_ERROR_SELF_NOT_IN_GROUP',
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
+  2002: 'DC_EVENT_INCOMING_REACTION',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -37,7 +38,10 @@ module.exports = {
   2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2150: 'DC_EVENT_WEBXDC_REALTIME_DATA',
+  2151: 'DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT',
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
-  2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED'
+  2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2400: 'DC_EVENT_CHANNEL_OVERFLOW'
 }

node/lib/constants.ts

@@ -1,6 +1,7 @@
 // Generated!
 
 export enum C {
+  DC_CERTCK_ACCEPT_INVALID = 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3,
   DC_CERTCK_AUTO = 0,
   DC_CERTCK_STRICT = 1,
@@ -30,6 +31,7 @@ export enum C {
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
+  DC_EVENT_CHANNEL_OVERFLOW = 2400,
   DC_EVENT_CHATLIST_CHANGED = 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED = 2301,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
@@ -49,6 +51,7 @@ export enum C {
   DC_EVENT_IMEX_PROGRESS = 2051,
   DC_EVENT_INCOMING_MSG = 2005,
   DC_EVENT_INCOMING_MSG_BUNCH = 2006,
+  DC_EVENT_INCOMING_REACTION = 2002,
   DC_EVENT_INFO = 100,
   DC_EVENT_LOCATION_CHANGED = 2035,
   DC_EVENT_MSGS_CHANGED = 2000,
@@ -66,6 +69,8 @@ export enum C {
   DC_EVENT_SMTP_MESSAGE_SENT = 103,
   DC_EVENT_WARNING = 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED = 2121,
+  DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT = 2151,
+  DC_EVENT_WEBXDC_REALTIME_DATA = 2150,
   DC_EVENT_WEBXDC_STATUS_UPDATE = 2120,
   DC_GCL_ADD_ALLDONE_HINT = 4,
   DC_GCL_ADD_SELF = 2,
@@ -126,11 +131,13 @@ export enum C {
   DC_QR_ASK_VERIFYCONTACT = 200,
   DC_QR_ASK_VERIFYGROUP = 202,
   DC_QR_BACKUP = 251,
+  DC_QR_BACKUP2 = 252,
   DC_QR_ERROR = 400,
   DC_QR_FPR_MISMATCH = 220,
   DC_QR_FPR_OK = 210,
   DC_QR_FPR_WITHOUT_ADDR = 230,
   DC_QR_LOGIN = 520,
+  DC_QR_PROXY = 271,
   DC_QR_REVIVE_VERIFYCONTACT = 510,
   DC_QR_REVIVE_VERIFYGROUP = 512,
   DC_QR_TEXT = 330,
@@ -317,6 +324,7 @@ export const EventId2EventName: { [key: number]: string } = {
   410: 'DC_EVENT_ERROR_SELF_NOT_IN_GROUP',
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
+  2002: 'DC_EVENT_INCOMING_REACTION',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -338,7 +346,10 @@ export const EventId2EventName: { [key: number]: string } = {
   2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2150: 'DC_EVENT_WEBXDC_REALTIME_DATA',
+  2151: 'DC_EVENT_WEBXDC_REALTIME_ADVERTISEMENT',
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
   2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2400: 'DC_EVENT_CHANNEL_OVERFLOW',
 }

node/lib/context.ts

@@ -475,47 +475,6 @@ export class Context extends EventEmitter {
     return binding.dcn_get_msg_html(this.dcn_context, Number(messageId))
   }
 
-  getNextMediaMessage(
-    messageId: number,
-    msgType1: number,
-    msgType2: number,
-    msgType3: number
-  ) {
-    debug(
-      `getNextMediaMessage ${messageId} ${msgType1} ${msgType2} ${msgType3}`
-    )
-    return this._getNextMedia(messageId, 1, msgType1, msgType2, msgType3)
-  }
-
-  getPreviousMediaMessage(
-    messageId: number,
-    msgType1: number,
-    msgType2: number,
-    msgType3: number
-  ) {
-    debug(
-      `getPreviousMediaMessage ${messageId} ${msgType1} ${msgType2} ${msgType3}`
-    )
-    return this._getNextMedia(messageId, -1, msgType1, msgType2, msgType3)
-  }
-
-  _getNextMedia(
-    messageId: number,
-    dir: number,
-    msgType1: number,
-    msgType2: number,
-    msgType3: number
-  ): number {
-    return binding.dcn_get_next_media(
-      this.dcn_context,
-      Number(messageId),
-      dir,
-      msgType1 || 0,
-      msgType2 || 0,
-      msgType3 || 0
-    )
-  }
-
   getSecurejoinQrCode(chatId: number): string {
     debug(`getSecurejoinQrCode ${chatId}`)
     return binding.dcn_get_securejoin_qr(this.dcn_context, Number(chatId))

node/src/module.c

@@ -1053,27 +1053,6 @@ NAPI_METHOD(dcn_get_msg_html) {
   NAPI_RETURN_AND_UNREF_STRING(msg_html);
 }
 
-NAPI_METHOD(dcn_get_next_media) {
-  NAPI_ARGV(6);
-  NAPI_DCN_CONTEXT();
-  NAPI_ARGV_UINT32(msg_id, 1);
-  NAPI_ARGV_INT32(dir, 2);
-  NAPI_ARGV_INT32(msg_type1, 3);
-  NAPI_ARGV_INT32(msg_type2, 4);
-  NAPI_ARGV_INT32(msg_type3, 5);
-
-  //TRACE("calling..");
-  uint32_t next_id = dc_get_next_media(dcn_context->dc_context,
-                                       msg_id,
-                                       dir,
-                                       msg_type1,
-                                       msg_type2,
-                                       msg_type3);
-  //TRACE("result %d", next_id);
-
-  NAPI_RETURN_UINT32(next_id);
-}
-
 NAPI_METHOD(dcn_set_chat_visibility) {
   NAPI_ARGV(3);
   NAPI_DCN_CONTEXT();
@@ -3443,7 +3422,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_get_msg_cnt);
   NAPI_EXPORT_FUNCTION(dcn_get_msg_info);
   NAPI_EXPORT_FUNCTION(dcn_get_msg_html);
-  NAPI_EXPORT_FUNCTION(dcn_get_next_media);
   NAPI_EXPORT_FUNCTION(dcn_set_chat_visibility);
   NAPI_EXPORT_FUNCTION(dcn_get_securejoin_qr);
   NAPI_EXPORT_FUNCTION(dcn_get_securejoin_qr_svg);

node/test/test.mjs

@@ -271,7 +271,7 @@ describe('Basic offline Tests', function () {
       'sync_msgs',
       'sentbox_watch',
       'show_emails',
-      'socks5_enabled',
+      'proxy_enabled',
       'sqlite_version',
       'uptime',
       'used_account_settings',

package.json

@@ -11,7 +11,7 @@
     "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
     "mocha": "^8.2.1",
-    "node-gyp": "^10.0.0",
+    "node-gyp": "~10.1.0",
     "prebuildify": "^5.0.1",
     "prebuildify-ci": "^1.0.5",
     "prettier": "^3.0.3",
@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.138.5"
+  "version": "1.150.0"
 }

python/examples/test_examples.py

@@ -44,11 +44,6 @@ def test_group_tracking_plugin(acfactory, lp):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
 
-    botproc.fnmatch_lines(
-        """
-        *ac_configure_completed*
-    """,
-    )
     ac1.add_account_plugin(FFIEventLogger(ac1))
     ac2.add_account_plugin(FFIEventLogger(ac2))
 

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.138.5"
+version = "1.150.0"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.7"

python/src/deltachat/account.py

@@ -194,15 +194,13 @@ class Account:
         assert res != ffi.NULL, f"config value not found for: {name!r}"
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr: str, secret: str) -> None:
+    def _preconfigure_keypair(self, secret: str) -> None:
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
         """
         res = lib.dc_preconfigure_keypair(
             self._dc_context,
-            as_dc_charpointer(addr),
-            ffi.NULL,
             as_dc_charpointer(secret),
         )
         if res == 0:

python/src/deltachat/chat.py

@@ -308,7 +308,7 @@ class Chat:
         msg = as_dc_charpointer(text)
         msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
         if msg_id == 0:
-            raise ValueError("message could not be send, does chat exist?")
+            raise ValueError("The message could not be sent. Does the chat exist?")
         return Message.from_db(self.account, msg_id)
 
     def send_file(self, path, mime_type="application/octet-stream"):

python/src/deltachat/direct_imap.py

@@ -8,19 +8,19 @@ import io
 import pathlib
 import ssl
 from contextlib import contextmanager
-from typing import List
+from typing import List, TYPE_CHECKING
 
 from imap_tools import (
     AND,
     Header,
     MailBox,
-    MailBoxTls,
     MailMessage,
     MailMessageFlags,
     errors,
 )
 
-from deltachat import Account, const
+if TYPE_CHECKING:
+    from deltachat import Account
 
 FLAGS = b"FLAGS"
 FETCH = b"FETCH"
@@ -28,7 +28,7 @@ ALL = "1:*"
 
 
 class DirectImap:
-    def __init__(self, account: Account) -> None:
+    def __init__(self, account: "Account") -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False
@@ -36,27 +36,13 @@ class DirectImap:
 
     def connect(self):
         host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
+        port = 993
 
         user = self.account.get_config("addr")
+        host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
-            self.conn = MailBox(host, port, ssl_context=ssl_context)
+        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

python/src/deltachat/testplugin.py

@@ -462,7 +462,7 @@ class ACFactory:
     def remove_preconfigured_keys(self) -> None:
         self._preconfigured_keys = []
 
-    def _preconfigure_key(self, account, addr):
+    def _preconfigure_key(self, account):
         # Only set a preconfigured key if we haven't used it yet for another account.
         try:
             keyname = self._preconfigured_keys.pop(0)
@@ -471,9 +471,9 @@ class ACFactory:
         else:
             fname_sec = self.data.read_path(f"key/{keyname}-secret.asc")
             if fname_sec:
-                account._preconfigure_keypair(addr, fname_sec)
+                account._preconfigure_keypair(fname_sec)
                 return True
-            print(f"WARN: could not use preconfigured keys for {addr!r}")
+            print("WARN: could not use preconfigured keys")
 
     def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account:
         # do a pseudo-configured account
@@ -492,7 +492,7 @@ class ACFactory:
                 "configured": "1",
             },
         )
-        self._preconfigure_key(ac, addr)
+        self._preconfigure_key(ac)
         self._acsetup.init_logging(ac)
         return ac
 
@@ -525,9 +525,10 @@ class ACFactory:
         configdict.setdefault("mvbox_move", False)
         configdict.setdefault("sentbox_watch", False)
         configdict.setdefault("sync_msgs", False)
+        configdict.setdefault("delete_server_after", 0)
         ac.update_config(configdict)
         self._acsetup._account2config[ac] = configdict
-        self._preconfigure_key(ac, configdict["addr"])
+        self._preconfigure_key(ac)
         return ac
 
     def wait_configured(self, account) -> None:
@@ -552,6 +553,15 @@ class ACFactory:
 
         bot_cfg = self.get_next_liveconfig()
         bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
+        self._acsetup.start_configure(bot_ac)
+        self.wait_configured(bot_ac)
+        bot_ac.start_io()
+        # Wait for DC_EVENT_IMAP_INBOX_IDLE so that all emails appeared in the bot's Inbox later are
+        # considered new and not existing ones, and thus processed by the bot.
+        print(bot_ac._logid, "waiting for inbox IDLE to become ready")
+        bot_ac._evtracker.wait_idle_inbox_ready()
+        bot_ac.stop_io()
+        self._acsetup._account2state[bot_ac] = self._acsetup.IDLEREADY
 
         # Forget ac as it will be opened by the bot subprocess
         # but keep something in the list to not confuse account generation

python/tests/test_0_complex_or_slow.py

@@ -1,4 +1,5 @@
 import sys
+import time
 
 import pytest
 import deltachat as dc
@@ -675,3 +676,17 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     assert msg_in.chat == chat2_offl
     assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
     assert ac2_offl_ac1_contact.is_verified()
+
+
+def test_deleted_msgs_dont_reappear(acfactory):
+    ac1 = acfactory.new_online_configuring_account()
+    acfactory.bring_accounts_online()
+    ac1.set_config("bcc_self", "1")
+    chat = ac1.get_self_contact().create_chat()
+    msg = chat.send_text("hello")
+    ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+    ac1.delete_messages([msg])
+    ac1._evtracker.get_matching("DC_EVENT_MSG_DELETED")
+    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+    time.sleep(5)
+    assert len(chat.get_messages()) == 0

python/tests/test_1_online.py

@@ -484,6 +484,24 @@ def test_move_works_on_self_sent(acfactory):
     ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
 
+def test_move_sync_msgs(acfactory):
+    ac1 = acfactory.new_online_configuring_account(bcc_self=True, sync_msgs=True, fix_is_chatmail=True)
+    acfactory.bring_accounts_online()
+
+    ac1.direct_imap.select_folder("DeltaChat")
+    # Sync messages may also be sent during the configuration.
+    mvbox_msg_cnt = len(ac1.direct_imap.get_all_messages())
+
+    ac1.set_config("displayname", "Alice")
+    ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+    ac1.set_config("displayname", "Bob")
+    ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+    ac1.direct_imap.select_folder("Inbox")
+    assert len(ac1.direct_imap.get_all_messages()) == 0
+    ac1.direct_imap.select_folder("DeltaChat")
+    assert len(ac1.direct_imap.get_all_messages()) == mvbox_msg_cnt + 2
+
+
 def test_forward_messages(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     chat = ac1.create_chat(ac2)
@@ -610,7 +628,7 @@ def test_long_group_name(acfactory, lp):
 
 
 def test_send_self_message(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=True)
+    ac1 = acfactory.new_online_configuring_account(mvbox_move=True, bcc_self=True)
     acfactory.bring_accounts_online()
     lp.sec("ac1: create self chat")
     chat = ac1.get_self_contact().create_chat()
@@ -1562,8 +1580,6 @@ def test_import_export_online_all(acfactory, tmp_path, data, lp):
 
         # check progress events for import
         assert imex_tracker.wait_progress(1, progress_upper_limit=249)
-        assert imex_tracker.wait_progress(500, progress_upper_limit=749)
-        assert imex_tracker.wait_progress(750, progress_upper_limit=999)
         assert imex_tracker.wait_progress(1000)
 
     assert_account_is_proper(ac1)
@@ -2068,12 +2084,11 @@ def test_send_receive_locations(acfactory, lp):
 def test_immediate_autodelete(acfactory, lp):
     ac1 = acfactory.new_online_configuring_account()
     ac2 = acfactory.new_online_configuring_account()
+    acfactory.bring_accounts_online()
 
     # "1" means delete immediately, while "0" means do not delete
     ac2.set_config("delete_server_after", "1")
 
-    acfactory.bring_accounts_online()
-
     lp.sec("ac1: create chat with ac2")
     chat1 = ac1.create_chat(ac2)
     ac2.create_chat(ac1)
@@ -2194,6 +2209,19 @@ def test_configure_error_msgs_wrong_pw(acfactory):
     # Password is wrong so it definitely has to say something about "password"
     assert "password" in ev.data2
 
+    ac1.stop_io()
+    ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+    ac1.configure()
+    while True:
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        print(f"Configuration progress: {ev.data1}")
+        if ev.data1 == 0:
+            break
+    assert "password" in ev.data2
+    # Account will continue to work with the old password, so if it becomes wrong, a notification
+    # must be shown.
+    assert ac1.get_config("notify_about_wrong_pw") == "1"
+
 
 def test_configure_error_msgs_invalid_server(acfactory):
     ac2 = acfactory.get_unconfigured_account()

python/tests/test_3_offline.py

@@ -67,7 +67,7 @@ class TestOfflineAccountBasic:
         ac = acfactory.get_unconfigured_account()
         alice_secret = data.read_path("key/alice-secret.asc")
         assert alice_secret
-        ac._preconfigure_keypair("alice@example.org", alice_secret)
+        ac._preconfigure_keypair(alice_secret)
 
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -705,7 +705,7 @@ class TestOfflineChat:
         ac1 = acfactory.get_pseudo_configured_account()
         ac2 = acfactory.get_pseudo_configured_account()
         qr = ac1.get_setup_contact_qr()
-        assert qr.startswith("OPENPGP4FPR:")
+        assert qr.startswith("https://i.delta.chat")
         res = ac2.check_qr(qr)
         assert res.is_ask_verifycontact()
         assert not res.is_ask_verifygroup()

release-date.in

@@ -1 +1 @@
-2024-05-16
+2024-11-21

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.78.0
+RUST_VERSION=1.82.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/create-provider-data-rs.py

@@ -149,7 +149,7 @@ def process_data(data, file):
     oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
 
     provider = ""
-    before_login_hint = cleanstr(data.get("before_login_hint", ""))
+    before_login_hint = cleanstr(data.get("before_login_hint", "") or "")
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
         provider += (

scripts/make-rpc-testenv.sh

@@ -3,4 +3,4 @@ set -euo pipefail
 
 tox -c deltachat-rpc-client -e py --devenv venv
 venv/bin/pip install --upgrade pip
-cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+cargo install --locked --path deltachat-rpc-server/ --root "$PWD/venv" --debug

scripts/run-rpc-test.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
-cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+cargo install --locked --path deltachat-rpc-server/ --root "$PWD/venv" --debug
 PATH="$PWD/venv/bin:$PATH" tox -c deltachat-rpc-client

scripts/run_all.sh

@@ -31,6 +31,6 @@ unset CHATMAIL_DOMAIN
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,py313,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=2f3db24107e4802c2df0aa0a40f0e144006c0a9b
+REV=77cbf92a8565fdf1bcaba10fa93c1455c750a1e9
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.34.0
+Version: 0.35.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -22,7 +22,13 @@ to implement typical messenger functions.
 - [Locations](#locations)
     - [User locations](#user-locations)
     - [Points of interest](#points-of-interest)
+- [Stickers](#stickers)
+- [Voice messages](#voice-messages)
+- [Reactions](#reactions)
+- [Attaching a contact to a message](#attaching-a-contact-to-a-message)
+- [Transitioning to a new e-mail address (AEAP)](#transitioning-to-a-new-e-mail-address-aeap)
 - [Miscellaneous](#miscellaneous)
+    - [Sync messages](#sync-messages)
 
 
 # Encryption
@@ -461,6 +467,58 @@ As an extension to RFC 9078, it is allowed to send empty reaction message,
 in which case all previously sent reactions are retracted.
 
 
+# Attaching a contact to a message
+
+Messengers MAY allow the user to attach a contact to a message
+in order to share it with the chat partner.
+The contact MUST be sent as a [vCard](https://datatracker.ietf.org/doc/html/rfc6350).
+
+The vCard MUST contain `EMAIL`,
+`FN` (display name),
+and `VERSION` (which version of the vCard standard you're using).
+If available, it SHOULD contain
+`REV` (current timestamp),
+`PHOTO` (avatar), and
+`KEY` (OpenPGP public key,
+in binary format,
+encoded with vanilla base64;
+note that this is different from the OpenPGP 'ASCII Armor' format).
+
+Example vCard:
+
+```
+BEGIN:VCARD
+VERSION:4.0
+EMAIL:alice@example.org
+FN:Alice Wonderland
+KEY:data:application/pgp-keys;base64,[Base64-data]
+PHOTO:data:image/jpeg;base64,[image in Base64]
+REV:20240418T184242Z
+END:VCARD
+```
+
+It is fine if messengers do include a full vCard parser
+and e.g. simply search for the line starting with `EMAIL`
+in order to get the email address.
+
+
+# Transitioning to a new e-mail address (AEAP)
+
+When receiving a message:
+- If the key exists, but belongs to another address
+- AND there is a `Chat-Version` header
+- AND the message is signed correctly
+- AND the From address is (also) in the encrypted (and therefore signed) headers 
+- AND the message timestamp is newer than the contact's `lastseen`
+  (to prevent changing the address back when messages arrive out of order)
+  (this condition is not that important
+  since we will have eventual consistency even without it):
+
+  Replace the contact in _all_ groups,
+  possibly deduplicate the members list,
+  and add a system message to all of these chats.
+
+
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -484,21 +542,4 @@ We define the effective date of a message
 as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
-
-# Transitioning to a new e-mail address (AEAP)
-
-When receiving a message:
-- If the key exists, but belongs to another address
-- AND there is a `Chat-Version` header
-- AND the message is signed correctly
-- AND the From address is (also) in the encrypted (and therefore signed) headers 
-- AND the message timestamp is newer than the contact's `lastseen`
-  (to prevent changing the address back when messages arrive out of order)
-  (this condition is not that important
-  since we will have eventual consistency even without it):
-
-  Replace the contact in _all_ groups,
-  possibly deduplicate the members list,
-  and add a system message to all of these chats.
-
 Copyright © 2017-2021 Delta Chat contributors.

src/accounts.rs

@@ -5,7 +5,8 @@ use std::future::Future;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
-use futures::future::join_all;
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
@@ -166,6 +167,19 @@ impl Accounts {
             .remove(&id)
             .with_context(|| format!("no account with id {id}"))?;
         ctx.stop_io().await;
+
+        // Explicitly close the database
+        // to make sure the database file is closed
+        // and can be removed on Windows.
+        // If some spawned task tries to use the database afterwards,
+        // it will fail.
+        //
+        // Previously `stop_io()` aborted the tasks without awaiting them
+        // and this resulted in keeping `Context` clones inside
+        // `Future`s that were not dropped. This bug is fixed now,
+        // but explicitly closing the database ensures that file is freed
+        // even if not all `Context` references are dropped.
+        ctx.sql.close().await;
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id) {
@@ -288,20 +302,48 @@ impl Accounts {
     ///
     /// This is an auxiliary function and not part of public API.
     /// Use [Accounts::background_fetch] instead.
-    async fn background_fetch_without_timeout(&self) {
+    async fn background_fetch_no_timeout(accounts: Vec<Context>, events: Events) {
         async fn background_fetch_and_log_error(account: Context) {
             if let Err(error) = account.background_fetch().await {
                 warn!(account, "{error:#}");
             }
         }
 
-        join_all(
-            self.accounts
-                .values()
-                .cloned()
-                .map(background_fetch_and_log_error),
+        events.emit(Event {
+            id: 0,
+            typ: EventType::Info(format!(
+                "Starting background fetch for {} accounts.",
+                accounts.len()
+            )),
+        });
+        let mut futures_unordered: FuturesUnordered<_> = accounts
+            .into_iter()
+            .map(background_fetch_and_log_error)
+            .collect();
+        while futures_unordered.next().await.is_some() {}
+    }
+
+    /// Auxiliary function for [Accounts::background_fetch].
+    async fn background_fetch_with_timeout(
+        accounts: Vec<Context>,
+        events: Events,
+        timeout: std::time::Duration,
+    ) {
+        if let Err(_err) = tokio::time::timeout(
+            timeout,
+            Self::background_fetch_no_timeout(accounts, events.clone()),
         )
-        .await;
+        .await
+        {
+            events.emit(Event {
+                id: 0,
+                typ: EventType::Warning("Background fetch timed out.".to_string()),
+            });
+        }
+        events.emit(Event {
+            id: 0,
+            typ: EventType::AccountsBackgroundFetchDone,
+        });
     }
 
     /// Performs a background fetch for all accounts in parallel with a timeout.
@@ -309,15 +351,13 @@ impl Accounts {
     /// The `AccountsBackgroundFetchDone` event is emitted at the end,
     /// process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
-    pub async fn background_fetch(&self, timeout: std::time::Duration) {
-        if let Err(_err) =
-            tokio::time::timeout(timeout, self.background_fetch_without_timeout()).await
-        {
-            self.emit_event(EventType::Warning(
-                "Background fetch timed out.".to_string(),
-            ));
-        }
-        self.emit_event(EventType::AccountsBackgroundFetchDone);
+    ///
+    /// Returns a future that resolves when background fetch is done,
+    /// but does not capture `&self`.
+    pub fn background_fetch(&self, timeout: std::time::Duration) -> impl Future<Output = ()> {
+        let accounts: Vec<Context> = self.accounts.values().cloned().collect();
+        let events = self.events.clone();
+        Self::background_fetch_with_timeout(accounts, events, timeout)
     }
 
     /// Emits a single event.
@@ -331,7 +371,7 @@ impl Accounts {
     }
 
     /// Sets notification token for Apple Push Notification service.
-    pub async fn set_push_device_token(&mut self, token: &str) -> Result<()> {
+    pub async fn set_push_device_token(&self, token: &str) -> Result<()> {
         self.push_subscriber.set_device_token(token).await;
         Ok(())
     }
@@ -485,10 +525,6 @@ impl Config {
 
     /// Read a configuration from the given file into memory.
     pub async fn from_file(file: PathBuf, writable: bool) -> Result<Self> {
-        let dir = file
-            .parent()
-            .context("Cannot get config file directory")?
-            .to_path_buf();
         let mut config = Self::new_nosync(file, writable).await?;
         let bytes = fs::read(&config.file)
             .await
@@ -500,9 +536,13 @@ impl Config {
         // Convert them to relative paths.
         let mut modified = false;
         for account in &mut config.inner.accounts {
-            if let Ok(new_dir) = account.dir.strip_prefix(&dir) {
-                account.dir = new_dir.to_path_buf();
-                modified = true;
+            if account.dir.is_absolute() {
+                if let Some(old_path_parent) = account.dir.parent() {
+                    if let Ok(new_path) = account.dir.strip_prefix(old_path_parent) {
+                        account.dir = new_path.to_path_buf();
+                        modified = true;
+                    }
+                }
             }
         }
         if modified && writable {

src/authres.rs

@@ -260,7 +260,6 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
     use tokio::fs;
     use tokio::io::AsyncReadExt;
 
@@ -520,8 +519,13 @@ Authentication-Results: dkim=";
         handle_authres(&t, &mail, "invalid@rom.com").await.unwrap();
     }
 
+    // Test that Autocrypt works with mailing list.
+    //
+    // Previous versions of Delta Chat ignored Autocrypt based on the List-Post header.
+    // This is not needed: comparing of the From address to Autocrypt header address is enough.
+    // If the mailing list is not rewriting the From header, Autocrypt should be applied.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_autocrypt_in_mailinglist_ignored() -> Result<()> {
+    async fn test_autocrypt_in_mailinglist_not_ignored() -> Result<()> {
         let mut tcm = TestContextManager::new();
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
@@ -533,28 +537,18 @@ Authentication-Results: dkim=";
             .insert_str(0, "List-Post: <mailto:deltachat-community.example.net>\n");
         bob.recv_msg(&sent).await;
         let peerstate = Peerstate::from_addr(&bob, "alice@example.org").await?;
-        assert!(peerstate.is_none());
-
-        // Do the same without the mailing list header, this time the peerstate should be accepted
-        let sent = alice
-            .send_text(alice_bob_chat.id, "hellooo without mailing list")
-            .await;
-        bob.recv_msg(&sent).await;
-        let peerstate = Peerstate::from_addr(&bob, "alice@example.org").await?;
         assert!(peerstate.is_some());
 
-        // This also means that Bob can now write encrypted to Alice:
+        // Bob can now write encrypted to Alice:
         let mut sent = bob
             .send_text(bob_alice_chat.id, "hellooo in the mailinglist again")
             .await;
         assert!(sent.load_from_db().await.get_showpadlock());
 
-        // But if Bob writes to a mailing list, Alice doesn't show a padlock
-        // since she can't verify the signature without accepting Bob's key:
         sent.payload
             .insert_str(0, "List-Post: <mailto:deltachat-community.example.net>\n");
         let rcvd = alice.recv_msg(&sent).await;
-        assert!(!rcvd.get_showpadlock());
+        assert!(rcvd.get_showpadlock());
         assert_eq!(&rcvd.text, "hellooo in the mailinglist again");
 
         Ok(())

src/blob.rs

@@ -3,7 +3,7 @@
 use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
-use std::io::Cursor;
+use std::io::{Cursor, Seek};
 use std::iter::FusedIterator;
 use std::mem;
 use std::path::{Path, PathBuf};
@@ -12,6 +12,7 @@ use anyhow::{format_err, Context as _, Result};
 use base64::Engine as _;
 use futures::StreamExt;
 use image::codecs::jpeg::JpegEncoder;
+use image::ImageReader;
 use image::{DynamicImage, GenericImage, GenericImageView, ImageFormat, Pixel, Rgba};
 use num_traits::FromPrimitive;
 use tokio::io::AsyncWriteExt;
@@ -252,16 +253,16 @@ impl<'a> BlobObject<'a> {
     ///
     /// The extension part will always be lowercased.
     fn sanitise_name(name: &str) -> (String, String) {
-        let mut name = name.to_string();
+        let mut name = name;
         for part in name.rsplit('/') {
             if !part.is_empty() {
-                name = part.to_string();
+                name = part;
                 break;
             }
         }
         for part in name.rsplit('\\') {
             if !part.is_empty() {
-                name = part.to_string();
+                name = part;
                 break;
             }
         }
@@ -271,32 +272,39 @@ impl<'a> BlobObject<'a> {
             replacement: "",
         };
 
-        let clean = sanitize_filename::sanitize_with_options(name, opts);
-        // Let's take the tricky filename
+        let name = sanitize_filename::sanitize_with_options(name, opts);
+        // Let's take a tricky filename,
         // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
-        // Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
-        let mut iter = clean.splitn(2, '.');
-
-        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        // stem == "file"
-
-        let ext_chars = iter.next().unwrap_or_default().chars();
-        let ext: String = ext_chars
+        // Assume that the extension is 32 chars maximum.
+        let ext: String = name
+            .chars()
             .rev()
-            .take(32)
+            .take_while(|c| !c.is_whitespace())
+            .take(33)
             .collect::<Vec<_>>()
             .iter()
             .rev()
             .collect();
-        // ext == "d_point_and_double_ending.tar.gz"
+        // ext == "nd_point_and_double_ending.tar.gz"
 
-        if ext.is_empty() {
-            (stem, "".to_string())
+        // Split it into "nd_point_and_double_ending" and "tar.gz":
+        let mut iter = ext.splitn(2, '.');
+        iter.next();
+
+        let ext = iter.next().unwrap_or_default();
+        let ext = if ext.is_empty() {
+            String::new()
         } else {
-            (stem, format!(".{ext}").to_lowercase())
-            // Return ("file", ".d_point_and_double_ending.tar.gz")
-            // which is not perfect but acceptable.
-        }
+            format!(".{ext}")
+            // ".tar.gz"
+        };
+        let stem = name
+            .strip_suffix(&ext)
+            .unwrap_or_default()
+            .chars()
+            .take(64)
+            .collect();
+        (stem, ext.to_lowercase())
     }
 
     /// Checks whether a name is a valid blob name.
@@ -426,9 +434,25 @@ impl<'a> BlobObject<'a> {
         let mut no_exif = false;
         let no_exif_ref = &mut no_exif;
         let res = tokio::task::block_in_place(move || {
-            let (nr_bytes, exif) = self.metadata()?;
+            let mut file = std::fs::File::open(self.to_abs_path())?;
+            let (nr_bytes, exif) = image_metadata(&file)?;
             *no_exif_ref = exif.is_none();
-            let mut img = image::open(&blob_abs).context("image decode failure")?;
+            // It's strange that BufReader modifies a file position while it takes a non-mut
+            // reference. Ok, just rewind it.
+            file.rewind()?;
+            let imgreader = ImageReader::new(std::io::BufReader::new(&file)).with_guessed_format();
+            let imgreader = match imgreader {
+                Ok(ir) => ir,
+                _ => {
+                    file.rewind()?;
+                    ImageReader::with_format(
+                        std::io::BufReader::new(&file),
+                        ImageFormat::from_path(&blob_abs)?,
+                    )
+                }
+            };
+            let fmt = imgreader.format().context("No format??")?;
+            let mut img = imgreader.decode().context("image decode failure")?;
             let orientation = exif.as_ref().map(|exif| exif_orientation(exif, context));
             let mut encoded = Vec::new();
             let mut changed_name = None;
@@ -457,10 +481,9 @@ impl<'a> BlobObject<'a> {
             let exceeds_max_bytes = nr_bytes > max_bytes as u64;
 
             let jpeg_quality = 75;
-            let fmt = ImageFormat::from_path(&blob_abs);
             let ofmt = match fmt {
-                Ok(ImageFormat::Png) if !exceeds_max_bytes => ImageOutputFormat::Png,
-                Ok(ImageFormat::Jpeg) => {
+                ImageFormat::Png if !exceeds_max_bytes => ImageOutputFormat::Png,
+                ImageFormat::Jpeg => {
                     add_white_bg = false;
                     ImageOutputFormat::Jpeg {
                         quality: jpeg_quality,
@@ -497,7 +520,7 @@ impl<'a> BlobObject<'a> {
                     img_wh = max(img.width(), img.height());
                     // PNGs and WebPs may be huge because of animation, which is lost by the `image`
                     // crate when recoding, so don't scale them down.
-                    if matches!(fmt, Ok(ImageFormat::Jpeg)) || !encoded.is_empty() {
+                    if matches!(fmt, ImageFormat::Jpeg) || !encoded.is_empty() {
                         img_wh = img_wh * 2 / 3;
                     }
                 }
@@ -538,7 +561,7 @@ impl<'a> BlobObject<'a> {
 
             if do_scale || exif.is_some() {
                 // The file format is JPEG/PNG now, we may have to change the file extension
-                if !matches!(fmt, Ok(ImageFormat::Jpeg))
+                if !matches!(fmt, ImageFormat::Jpeg)
                     && matches!(ofmt, ImageOutputFormat::Jpeg { .. })
                 {
                     blob_abs = blob_abs.with_extension("jpg");
@@ -575,15 +598,14 @@ impl<'a> BlobObject<'a> {
             }
         }
     }
+}
 
-    /// Returns image file size and Exif.
-    pub fn metadata(&self) -> Result<(u64, Option<exif::Exif>)> {
-        let file = std::fs::File::open(self.to_abs_path())?;
-        let len = file.metadata()?.len();
-        let mut bufreader = std::io::BufReader::new(&file);
-        let exif = exif::Reader::new().read_from_container(&mut bufreader).ok();
-        Ok((len, exif))
-    }
+/// Returns image file size and Exif.
+pub fn image_metadata(file: &std::fs::File) -> Result<(u64, Option<exif::Exif>)> {
+    let len = file.metadata()?.len();
+    let mut bufreader = std::io::BufReader::new(file);
+    let exif = exif::Reader::new().read_from_container(&mut bufreader).ok();
+    Ok((len, exif))
 }
 
 fn exif_orientation(exif: &exif::Exif, context: &Context) -> i32 {
@@ -600,7 +622,7 @@ fn exif_orientation(exif: &exif::Exif, context: &Context) -> i32 {
     0
 }
 
-impl<'a> fmt::Display for BlobObject<'a> {
+impl fmt::Display for BlobObject<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "$BLOBDIR/{}", self.name)
     }
@@ -651,10 +673,6 @@ impl<'a> BlobDirContents<'a> {
     pub(crate) fn iter(&self) -> BlobDirIter<'_> {
         BlobDirIter::new(self.context, self.inner.iter())
     }
-
-    pub(crate) fn len(&self) -> usize {
-        self.inner.len()
-    }
 }
 
 /// A iterator over all the [`BlobObject`]s in the blobdir.
@@ -952,6 +970,19 @@ mod tests {
         assert!(!stem.contains(':'));
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
+
+        let (stem, ext) = BlobObject::sanitise_name(
+            "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz",
+        );
+        assert_eq!(
+            stem,
+            "file.with_lots_of_characters_behind_point_and_double_ending"
+        );
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name("a. tar.tar.gz");
+        assert_eq!(stem, "a. tar");
+        assert_eq!(ext, ".tar.gz");
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1086,32 +1117,34 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "jpg",
-            true, // has Exif
-            1000,
-            1000,
-            0,
-            1000,
-            1000,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 1000,
+            original_height: 1000,
+            compressed_width: 1000,
+            compressed_height: 1000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
             bytes,
-            "jpg",
-            true, // has Exif
-            1000,
-            1000,
-            0,
-            1000,
-            1000,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 1000,
+            original_height: 1000,
+            compressed_width: 1000,
+            compressed_height: 1000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1120,18 +1153,20 @@ mod tests {
     async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        let img_rotated = SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "jpg",
-            true, // has Exif
-            2000,
-            1800,
-            270,
-            1800,
-            2000,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 2000,
+            original_height: 1800,
+            orientation: 270,
+            compressed_width: 1800,
+            compressed_height: 2000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
         assert_correct_rotation(&img_rotated);
@@ -1140,18 +1175,18 @@ mod tests {
         img_rotated.write_to(&mut buf, ImageFormat::Jpeg).unwrap();
         let bytes = buf.into_inner();
 
-        let img_rotated = send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
-            &bytes,
-            "jpg",
-            false, // no Exif
-            1800,
-            2000,
-            0,
-            1800,
-            2000,
-        )
+        let img_rotated = SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
+            bytes: &bytes,
+            extension: "jpg",
+            original_width: 1800,
+            original_height: 2000,
+            compressed_width: 1800,
+            compressed_height: 2000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
         assert_correct_rotation(&img_rotated);
@@ -1161,49 +1196,80 @@ mod tests {
     async fn test_recode_image_balanced_png() {
         let bytes = include_bytes!("../test-data/image/screenshot.png");
 
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            1920,
-            1080,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
 
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            constants::WORSE_IMAGE_SIZE,
-            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: constants::WORSE_IMAGE_SIZE,
+            compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+            ..Default::default()
+        }
+        .test()
+        .await
+        .unwrap();
+
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::File,
+            media_quality_config: "1",
+            bytes,
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            ..Default::default()
+        }
+        .test()
+        .await
+        .unwrap();
+
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::File,
+            media_quality_config: "1",
+            bytes,
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            set_draft: true,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
 
         // This will be sent as Image, see [`BlobObject::maybe_sticker`] for explanation.
-        send_image_check_mediaquality(
-            Viewtype::Sticker,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Sticker,
+            media_quality_config: "0",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            1920,
-            1080,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1214,18 +1280,18 @@ mod tests {
     async fn test_recode_image_rgba_png_to_jpeg() {
         let bytes = include_bytes!("../test-data/image/screenshot-rgba.png");
 
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            constants::WORSE_IMAGE_SIZE,
-            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: constants::WORSE_IMAGE_SIZE,
+            compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1233,18 +1299,19 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_huge_jpg() {
         let bytes = include_bytes!("../test-data/image/screenshot.jpg");
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "jpg",
-            true, // has Exif
-            1920,
-            1080,
-            0,
-            constants::BALANCED_IMAGE_SIZE,
-            constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: constants::BALANCED_IMAGE_SIZE,
+            compressed_height: constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1266,68 +1333,93 @@ mod tests {
         assert_eq!(luma, 0);
     }
 
-    #[allow(clippy::too_many_arguments)]
-    async fn send_image_check_mediaquality(
-        viewtype: Viewtype,
-        media_quality_config: Option<&str>,
-        bytes: &[u8],
-        extension: &str,
-        has_exif: bool,
-        original_width: u32,
-        original_height: u32,
-        orientation: i32,
-        compressed_width: u32,
-        compressed_height: u32,
-    ) -> anyhow::Result<DynamicImage> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-        alice
-            .set_config(Config::MediaQuality, media_quality_config)
-            .await?;
-        let file = alice.get_blobdir().join("file").with_extension(extension);
+    #[derive(Default)]
+    struct SendImageCheckMediaquality<'a> {
+        pub(crate) viewtype: Viewtype,
+        pub(crate) media_quality_config: &'a str,
+        pub(crate) bytes: &'a [u8],
+        pub(crate) extension: &'a str,
+        pub(crate) has_exif: bool,
+        pub(crate) original_width: u32,
+        pub(crate) original_height: u32,
+        pub(crate) orientation: i32,
+        pub(crate) compressed_width: u32,
+        pub(crate) compressed_height: u32,
+        pub(crate) set_draft: bool,
+    }
 
-        fs::write(&file, &bytes)
-            .await
-            .context("failed to write file")?;
-        check_image_size(&file, original_width, original_height);
+    impl SendImageCheckMediaquality<'_> {
+        pub(crate) async fn test(self) -> anyhow::Result<DynamicImage> {
+            let viewtype = self.viewtype;
+            let media_quality_config = self.media_quality_config;
+            let bytes = self.bytes;
+            let extension = self.extension;
+            let has_exif = self.has_exif;
+            let original_width = self.original_width;
+            let original_height = self.original_height;
+            let orientation = self.orientation;
+            let compressed_width = self.compressed_width;
+            let compressed_height = self.compressed_height;
+            let set_draft = self.set_draft;
 
-        let blob = BlobObject::new_from_path(&alice, &file).await?;
-        let (_, exif) = blob.metadata()?;
-        if has_exif {
-            let exif = exif.unwrap();
-            assert_eq!(exif_orientation(&exif, &alice), orientation);
-        } else {
+            let alice = TestContext::new_alice().await;
+            let bob = TestContext::new_bob().await;
+            alice
+                .set_config(Config::MediaQuality, Some(media_quality_config))
+                .await?;
+            let file = alice.get_blobdir().join("file").with_extension(extension);
+
+            fs::write(&file, &bytes)
+                .await
+                .context("failed to write file")?;
+            check_image_size(&file, original_width, original_height);
+
+            let (_, exif) = image_metadata(&std::fs::File::open(&file)?)?;
+            if has_exif {
+                let exif = exif.unwrap();
+                assert_eq!(exif_orientation(&exif, &alice), orientation);
+            } else {
+                assert!(exif.is_none());
+            }
+
+            let mut msg = Message::new(viewtype);
+            msg.set_file(file.to_str().unwrap(), None);
+            let chat = alice.create_chat(&bob).await;
+            if set_draft {
+                chat.id.set_draft(&alice, Some(&mut msg)).await.unwrap();
+                msg = chat.id.get_draft(&alice).await.unwrap().unwrap();
+                assert_eq!(msg.get_viewtype(), Viewtype::File);
+            }
+            let sent = alice.send_msg(chat.id, &mut msg).await;
+            let alice_msg = alice.get_last_msg().await;
+            assert_eq!(alice_msg.get_width() as u32, compressed_width);
+            assert_eq!(alice_msg.get_height() as u32, compressed_height);
+            let file_saved = alice
+                .get_blobdir()
+                .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
+            alice_msg.save_file(&alice, &file_saved).await?;
+            check_image_size(file_saved, compressed_width, compressed_height);
+
+            let bob_msg = bob.recv_msg(&sent).await;
+            assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
+            assert_eq!(bob_msg.get_width() as u32, compressed_width);
+            assert_eq!(bob_msg.get_height() as u32, compressed_height);
+            let file_saved = bob
+                .get_blobdir()
+                .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
+            bob_msg.save_file(&bob, &file_saved).await?;
+            if viewtype == Viewtype::File {
+                assert_eq!(file_saved.extension().unwrap(), extension);
+                let bytes1 = fs::read(&file_saved).await?;
+                assert_eq!(&bytes1, bytes);
+            }
+
+            let (_, exif) = image_metadata(&std::fs::File::open(&file_saved)?)?;
             assert!(exif.is_none());
+
+            let img = check_image_size(file_saved, compressed_width, compressed_height);
+            Ok(img)
         }
-
-        let mut msg = Message::new(viewtype);
-        msg.set_file(file.to_str().unwrap(), None);
-        let chat = alice.create_chat(&bob).await;
-        let sent = alice.send_msg(chat.id, &mut msg).await;
-        let alice_msg = alice.get_last_msg().await;
-        assert_eq!(alice_msg.get_width() as u32, compressed_width);
-        assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let file_saved = alice
-            .get_blobdir()
-            .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
-        alice_msg.save_file(&alice, &file_saved).await?;
-        check_image_size(file_saved, compressed_width, compressed_height);
-
-        let bob_msg = bob.recv_msg(&sent).await;
-        assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
-        assert_eq!(bob_msg.get_width() as u32, compressed_width);
-        assert_eq!(bob_msg.get_height() as u32, compressed_height);
-        let file_saved = bob
-            .get_blobdir()
-            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
-        bob_msg.save_file(&bob, &file_saved).await?;
-
-        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
-        let (_, exif) = blob.metadata()?;
-        assert!(exif.is_none());
-
-        let img = check_image_size(file_saved, compressed_width, compressed_height);
-        Ok(img)
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1359,8 +1451,7 @@ mod tests {
             .get_blobdir()
             .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
         bob_msg.save_file(&bob, &file_saved).await?;
-        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
-        let (file_size, _) = blob.metadata()?;
+        let (file_size, _) = image_metadata(&std::fs::File::open(&file_saved)?)?;
         assert_eq!(file_size, bytes.len() as u64);
         check_image_size(file_saved, width, height);
         Ok(())

src/chatlist.rs

@@ -82,11 +82,13 @@ impl Chatlist {
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
     /// - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
     ///   is added as needed.
+    ///
     /// `query`: An optional query for filtering the list. Only chats matching this query
-    ///     are returned. When `is:unread` is contained in the query, the chatlist is
-    ///     filtered such that only chats with unread messages show up.
+    /// are returned. When `is:unread` is contained in the query, the chatlist is
+    /// filtered such that only chats with unread messages show up.
+    ///
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
-    ///     are returned.
+    /// are returned.
     pub async fn try_load(
         context: &Context,
         listflags: usize,
@@ -392,25 +394,32 @@ impl Chatlist {
             &chat_loaded
         };
 
-        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
-            let lastmsg = Message::load_from_db(context, lastmsg_id)
+        let lastmsg = if let Some(lastmsg_id) = lastmsg_id {
+            // Message may be deleted by the time we try to load it,
+            // so use `load_from_db_optional` instead of `load_from_db`.
+            Message::load_from_db_optional(context, lastmsg_id)
                 .await
-                .context("loading message failed")?;
+                .context("Loading message failed")?
+        } else {
+            None
+        };
+
+        let lastcontact = if let Some(lastmsg) = &lastmsg {
             if lastmsg.from_id == ContactId::SELF {
-                (Some(lastmsg), None)
+                None
             } else {
                 match chat.typ {
                     Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                         let lastcontact = Contact::get_by_id(context, lastmsg.from_id)
                             .await
                             .context("loading contact failed")?;
-                        (Some(lastmsg), Some(lastcontact))
+                        Some(lastcontact)
                     }
-                    Chattype::Single => (Some(lastmsg), None),
+                    Chattype::Single => None,
                 }
             }
         } else {
-            (None, None)
+            None
         };
 
         if chat.id.is_archived_link() {
@@ -474,7 +483,6 @@ mod tests {
         add_contact_to_chat, create_group_chat, get_chat_contacts, remove_contact_from_chat,
         send_text_msg, ProtectionStatus,
     };
-    use crate::message::Viewtype;
     use crate::receive_imf::receive_imf;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
@@ -508,8 +516,7 @@ mod tests {
         // Instead of setting drafts for chat_id1 and chat_id3, we could also sleep
         // 2s here.
         for chat_id in &[chat_id1, chat_id3, chat_id2] {
-            let mut msg = Message::new(Viewtype::Text);
-            msg.set_text("hello".to_string());
+            let mut msg = Message::new_text("hello".to_string());
             chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
         }
 
@@ -753,8 +760,7 @@ mod tests {
             .await
             .unwrap();
 
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text("foo:\nbar \r\n test".to_string());
+        let mut msg = Message::new_text("foo:\nbar \r\n test".to_string());
         chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
@@ -762,6 +768,25 @@ mod tests {
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
 
+    /// Tests that summary does not fail to load
+    /// if the draft was deleted after loading the chatlist.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_summary_deleted_draft() {
+        let t = TestContext::new().await;
+
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+            .await
+            .unwrap();
+        let mut msg = Message::new_text("Foobar".to_string());
+        chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
+
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        chat_id.set_draft(&t, None).await.unwrap();
+
+        let summary_res = chats.get_summary(&t, 0, None).await;
+        assert!(summary_res.is_ok());
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_broken() {
         let t = TestContext::new_bob().await;

src/config.rs

@@ -6,21 +6,21 @@ use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
 use base64::Engine as _;
-use deltachat_contact_tools::addr_cmp;
+use deltachat_contact_tools::{addr_cmp, sanitize_single_line};
 use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumString};
 use tokio::fs;
 
 use crate::blob::BlobObject;
-use crate::constants::{self, DC_VERSION_STR};
+use crate::constants;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 use crate::sync::{self, Sync::*, SyncData};
-use crate::tools::{get_abs_path, improve_single_line_input};
+use crate::tools::get_abs_path;
 
 /// The available configuration keys.
 #[derive(
@@ -59,7 +59,10 @@ pub enum Config {
     /// IMAP server security (e.g. TLS, STARTTLS).
     MailSecurity,
 
-    /// How to check IMAP server TLS certificates.
+    /// How to check TLS certificates.
+    ///
+    /// "IMAP" in the name is for compatibility,
+    /// this actually applies to both IMAP and SMTP connections.
     ImapCertificateChecks,
 
     /// SMTP server hostname.
@@ -77,7 +80,9 @@ pub enum Config {
     /// SMTP server security (e.g. TLS, STARTTLS).
     SendSecurity,
 
-    /// How to check SMTP server TLS certificates.
+    /// Deprecated option for backwards compatibilty.
+    ///
+    /// Certificate checks for SMTP are actually controlled by `imap_certificate_checks` config.
     SmtpCertificateChecks,
 
     /// Whether to use OAuth 2.
@@ -86,21 +91,44 @@ pub enum Config {
     /// Should not be extended in the future, create new config keys instead.
     ServerFlags,
 
+    /// True if proxy is enabled.
+    ///
+    /// Can be used to disable proxy without erasing known URLs.
+    ProxyEnabled,
+
+    /// Proxy URL.
+    ///
+    /// Supported URLs schemes are `http://` (HTTP), `https://` (HTTPS),
+    /// `socks5://` (SOCKS5) and `ss://` (Shadowsocks).
+    ///
+    /// May contain multiple URLs separated by newline, in which case the first one is used.
+    ProxyUrl,
+
     /// True if SOCKS5 is enabled.
     ///
     /// Can be used to disable SOCKS5 without erasing SOCKS5 configuration.
+    ///
+    /// Deprecated in favor of `ProxyEnabled`.
     Socks5Enabled,
 
     /// SOCKS5 proxy server hostname or address.
+    ///
+    /// Deprecated in favor of `ProxyUrl`.
     Socks5Host,
 
     /// SOCKS5 proxy server port.
+    ///
+    /// Deprecated in favor of `ProxyUrl`.
     Socks5Port,
 
     /// SOCKS5 proxy server username.
+    ///
+    /// Deprecated in favor of `ProxyUrl`.
     Socks5User,
 
     /// SOCKS5 proxy server password.
+    ///
+    /// Deprecated in favor of `ProxyUrl`.
     Socks5Password,
 
     /// Own name to use in the `From:` field when sending messages.
@@ -131,7 +159,8 @@ pub enum Config {
     #[strum(props(default = "0"))]
     SentboxWatch,
 
-    /// True if chat messages should be moved to a separate folder.
+    /// True if chat messages should be moved to a separate folder. Auto-sent messages like sync
+    /// ones are moved there anyway.
     #[strum(props(default = "1"))]
     MvboxMove,
 
@@ -168,12 +197,12 @@ pub enum Config {
     /// Timer in seconds after which the message is deleted from the
     /// server.
     ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
+    /// 0 means messages are never deleted by Delta Chat.
     ///
     /// Value 1 is treated as "delete at once": messages are deleted
     /// immediately, without moving to DeltaChat folder.
-    #[strum(props(default = "0"))]
+    ///
+    /// Default is 1 for chatmail accounts before a backup export, 0 otherwise.
     DeleteServerAfter,
 
     /// Timer in seconds after which the message is deleted from the
@@ -194,45 +223,74 @@ pub enum Config {
     /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
 
+    /// List of configured IMAP servers as a JSON array.
+    ConfiguredImapServers,
+
     /// Configured IMAP server hostname.
+    ///
+    /// This is replaced by `configured_imap_servers` for new configurations.
     ConfiguredMailServer,
 
+    /// Configured IMAP server port.
+    ///
+    /// This is replaced by `configured_imap_servers` for new configurations.
+    ConfiguredMailPort,
+
+    /// Configured IMAP server security (e.g. TLS, STARTTLS).
+    ///
+    /// This is replaced by `configured_imap_servers` for new configurations.
+    ConfiguredMailSecurity,
+
     /// Configured IMAP server username.
+    ///
+    /// This is set if user has configured username manually.
     ConfiguredMailUser,
 
     /// Configured IMAP server password.
     ConfiguredMailPw,
 
-    /// Configured IMAP server port.
-    ConfiguredMailPort,
-
-    /// Configured IMAP server security (e.g. TLS, STARTTLS).
-    ConfiguredMailSecurity,
-
-    /// How to check IMAP server TLS certificates.
+    /// Configured TLS certificate checks.
+    /// This option is saved on successful configuration
+    /// and should not be modified manually.
+    ///
+    /// This actually applies to both IMAP and SMTP connections,
+    /// but has "IMAP" in the name for backwards compatibility.
     ConfiguredImapCertificateChecks,
 
+    /// List of configured SMTP servers as a JSON array.
+    ConfiguredSmtpServers,
+
     /// Configured SMTP server hostname.
+    ///
+    /// This is replaced by `configured_smtp_servers` for new configurations.
     ConfiguredSendServer,
 
+    /// Configured SMTP server port.
+    ///
+    /// This is replaced by `configured_smtp_servers` for new configurations.
+    ConfiguredSendPort,
+
+    /// Configured SMTP server security (e.g. TLS, STARTTLS).
+    ///
+    /// This is replaced by `configured_smtp_servers` for new configurations.
+    ConfiguredSendSecurity,
+
     /// Configured SMTP server username.
+    ///
+    /// This is set if user has configured username manually.
     ConfiguredSendUser,
 
     /// Configured SMTP server password.
     ConfiguredSendPw,
 
-    /// Configured SMTP server port.
-    ConfiguredSendPort,
-
-    /// How to check SMTP server TLS certificates.
+    /// Deprecated, stored for backwards compatibility.
+    ///
+    /// ConfiguredImapCertificateChecks is actually used.
     ConfiguredSmtpCertificateChecks,
 
     /// Whether OAuth 2 is used with configured provider.
     ConfiguredServerFlags,
 
-    /// Configured SMTP server security (e.g. TLS, STARTTLS).
-    ConfiguredSendSecurity,
-
     /// Configured folder for incoming messages.
     ConfiguredInboxFolder,
 
@@ -257,6 +315,16 @@ pub enum Config {
     /// True if account is a chatmail account.
     IsChatmail,
 
+    /// True if `IsChatmail` mustn't be autoconfigured. For tests.
+    FixIsChatmail,
+
+    /// True if account is muted.
+    IsMuted,
+
+    /// Optional tag as "Work", "Family".
+    /// Meant to help profile owner to differ between profiles with similar names.
+    PrivateTag,
+
     /// All secondary self addresses separated by spaces
     /// (`addr1@example.org addr2@example.org addr3@example.org`)
     SecondaryAddrs,
@@ -314,7 +382,8 @@ pub enum Config {
     #[strum(props(default = "0"))]
     DownloadLimit,
 
-    /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set.
+    /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set
+    /// and `Bot` unset.
     #[strum(props(default = "1"))]
     SyncMsgs,
 
@@ -327,6 +396,12 @@ pub enum Config {
     /// Make all outgoing messages with Autocrypt header "multipart/signed".
     SignUnencrypted,
 
+    /// Enable header protection for `Autocrypt` header.
+    ///
+    /// This is an experimental setting not compatible to other MUAs
+    /// and older Delta Chat versions (core version <= v1.149.0).
+    ProtectAutocrypt,
+
     /// Let the core save all events to the database.
     /// This value is used internally to remember the MsgId of the logging xdc
     #[strum(props(default = "0"))]
@@ -363,8 +438,9 @@ pub enum Config {
     /// MsgId of webxdc map integration.
     WebxdcIntegration,
 
-    /// Iroh secret key.
-    IrohSecretKey,
+    /// Enable webxdc realtime features.
+    #[strum(props(default = "1"))]
+    WebxdcRealtimeEnabled,
 }
 
 impl Config {
@@ -378,14 +454,11 @@ impl Config {
     /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
     /// mustn't be controlled by other devices.
     pub(crate) fn is_synced(&self) -> bool {
-        // We don't restart IO from the synchronisation code, so this is to be on the safe side.
-        if self.needs_io_restart() {
-            return false;
-        }
         matches!(
             self,
             Self::Displayname
                 | Self::MdnsEnabled
+                | Self::MvboxMove
                 | Self::ShowEmails
                 | Self::Selfavatar
                 | Self::Selfstatus,
@@ -394,21 +467,21 @@ impl Config {
 
     /// Whether the config option needs an IO scheduler restart to take effect.
     pub(crate) fn needs_io_restart(&self) -> bool {
-        matches!(
-            self,
-            Config::MvboxMove | Config::OnlyFetchMvbox | Config::SentboxWatch
-        )
+        matches!(self, Config::OnlyFetchMvbox | Config::SentboxWatch)
     }
 }
 
 impl Context {
-    /// Returns true if configuration value is set for the given key.
-    pub async fn config_exists(&self, key: Config) -> Result<bool> {
+    /// Returns true if configuration value is set in the db for the given key.
+    ///
+    /// NB: Don't use this to check if the key is configured because this doesn't look into
+    /// environment. The proper use of this function is e.g. checking a key before setting it.
+    pub(crate) async fn config_exists(&self, key: Config) -> Result<bool> {
         Ok(self.sql.get_raw_config(key.as_ref()).await?.is_some())
     }
 
-    /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
+    /// Get a config key value. Returns `None` if no value is set.
+    pub(crate) async fn get_config_opt(&self, key: Config) -> Result<Option<String>> {
         let env_key = format!("DELTACHAT_{}", key.as_ref().to_uppercase());
         if let Ok(value) = env::var(env_key) {
             return Ok(Some(value));
@@ -423,24 +496,43 @@ impl Context {
                         .into_owned()
                 })
             }
-            Config::SysVersion => Some((*DC_VERSION_STR).clone()),
+            Config::SysVersion => Some((*constants::DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{RECOMMENDED_FILE_SIZE}")),
             Config::SysConfigKeys => Some(get_config_keys_string()),
             _ => self.sql.get_raw_config(key.as_ref()).await?,
         };
+        Ok(value)
+    }
 
+    /// Get a config key value if set, or a default value. Returns `None` if no value exists.
+    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
+        let value = self.get_config_opt(key).await?;
         if value.is_some() {
             return Ok(value);
         }
 
         // Default values
-        match key {
-            Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
-            _ => Ok(key.get_str("default").map(|s| s.to_string())),
-        }
+        let val = match key {
+            Config::ConfiguredInboxFolder => Some("INBOX"),
+            Config::DeleteServerAfter => match Box::pin(self.is_chatmail()).await? {
+                false => Some("0"),
+                true => Some("1"),
+            },
+            _ => key.get_str("default"),
+        };
+        Ok(val.map(|s| s.to_string()))
     }
 
-    /// Returns Some(T) if a value for the given key exists and was successfully parsed.
+    /// Returns Some(T) if a value for the given key is set and was successfully parsed.
+    /// Returns None if could not parse.
+    pub(crate) async fn get_config_opt_parsed<T: FromStr>(&self, key: Config) -> Result<Option<T>> {
+        self.get_config_opt(key)
+            .await
+            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()))
+    }
+
+    /// Returns Some(T) if a value for the given key exists (incl. default value) and was
+    /// successfully parsed.
     /// Returns None if could not parse.
     pub async fn get_config_parsed<T: FromStr>(&self, key: Config) -> Result<Option<T>> {
         self.get_config(key)
@@ -468,20 +560,28 @@ impl Context {
         Ok(self.get_config_parsed(key).await?.unwrap_or_default())
     }
 
-    /// Returns boolean configuration value (if any) for the given key.
-    pub async fn get_config_bool_opt(&self, key: Config) -> Result<Option<bool>> {
-        Ok(self.get_config_parsed::<i32>(key).await?.map(|x| x != 0))
+    /// Returns boolean configuration value (if set) for the given key.
+    pub(crate) async fn get_config_bool_opt(&self, key: Config) -> Result<Option<bool>> {
+        Ok(self
+            .get_config_opt_parsed::<i32>(key)
+            .await?
+            .map(|x| x != 0))
     }
 
     /// Returns boolean configuration value for the given key.
     pub async fn get_config_bool(&self, key: Config) -> Result<bool> {
-        Ok(self.get_config_bool_opt(key).await?.unwrap_or_default())
+        Ok(self
+            .get_config_parsed::<i32>(key)
+            .await?
+            .map(|x| x != 0)
+            .unwrap_or_default())
     }
 
     /// Returns true if movebox ("DeltaChat" folder) should be watched.
     pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
         Ok(self.get_config_bool(Config::MvboxMove).await?
-            || self.get_config_bool(Config::OnlyFetchMvbox).await?)
+            || self.get_config_bool(Config::OnlyFetchMvbox).await?
+            || !self.get_config_bool(Config::IsChatmail).await?)
     }
 
     /// Returns true if sentbox ("Sent" folder) should be watched.
@@ -493,16 +593,47 @@ impl Context {
                 .is_some())
     }
 
+    /// Returns true if sync messages should be sent.
+    pub(crate) async fn should_send_sync_msgs(&self) -> Result<bool> {
+        Ok(self.get_config_bool(Config::SyncMsgs).await?
+            && self.get_config_bool(Config::BccSelf).await?
+            && !self.get_config_bool(Config::Bot).await?)
+    }
+
+    /// Returns whether sync messages should be uploaded to the mvbox.
+    pub(crate) async fn should_move_sync_msgs(&self) -> Result<bool> {
+        Ok(self.get_config_bool(Config::MvboxMove).await?
+            || !self.get_config_bool(Config::IsChatmail).await?)
+    }
+
+    /// Returns whether MDNs should be requested.
+    pub(crate) async fn should_request_mdns(&self) -> Result<bool> {
+        match self.get_config_bool_opt(Config::MdnsEnabled).await? {
+            Some(val) => Ok(val),
+            None => Ok(!self.get_config_bool(Config::Bot).await?),
+        }
+    }
+
+    /// Returns whether MDNs should be sent.
+    pub(crate) async fn should_send_mdns(&self) -> Result<bool> {
+        self.get_config_bool(Config::MdnsEnabled).await
+    }
+
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
     pub async fn get_config_delete_server_after(&self) -> Result<Option<i64>> {
-        match self.get_config_int(Config::DeleteServerAfter).await? {
-            0 => Ok(None),
-            1 => Ok(Some(0)),
-            x => Ok(Some(i64::from(x))),
-        }
+        let val = match self
+            .get_config_parsed::<i64>(Config::DeleteServerAfter)
+            .await?
+            .unwrap_or(0)
+        {
+            0 => None,
+            1 => Some(0),
+            x => Some(x),
+        };
+        Ok(val)
     }
 
     /// Gets the configured provider, as saved in the `configured_provider` value.
@@ -547,6 +678,7 @@ impl Context {
     fn check_config(key: Config, value: Option<&str>) -> Result<()> {
         match key {
             Config::Socks5Enabled
+            | Config::ProxyEnabled
             | Config::BccSelf
             | Config::E2eeEnabled
             | Config::MdnsEnabled
@@ -600,7 +732,7 @@ impl Context {
         mut value: Option<&str>,
     ) -> Result<()> {
         Self::check_config(key, value)?;
-        let sync = sync == Sync && key.is_synced();
+        let sync = sync == Sync && key.is_synced() && self.is_configured().await?;
         let better_value;
 
         match key {
@@ -639,7 +771,7 @@ impl Context {
             }
             Config::Displayname => {
                 if let Some(v) = value {
-                    better_value = improve_single_line_input(v);
+                    better_value = sanitize_single_line(v);
                     value = Some(&better_value);
                 }
                 self.sql.set_raw_config(key.as_ref(), value).await?;
@@ -677,7 +809,7 @@ impl Context {
         {
             return Ok(());
         }
-        Box::pin(self.send_sync_msg()).await.log_err(self).ok();
+        self.scheduler.interrupt_inbox().await;
         Ok(())
     }
 
@@ -736,6 +868,8 @@ impl Context {
     ///
     /// This should only be used by test code and during configure.
     pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
+        self.quota.write().await.take();
+
         // add old primary address (if exists) to secondary addresses
         let mut secondary_addrs = self.get_all_self_addrs().await?;
         // never store a primary address also as a secondary
@@ -748,7 +882,7 @@ impl Context {
 
         self.set_config_internal(Config::ConfiguredAddr, Some(primary_new))
             .await?;
-
+        self.emit_event(EventType::ConnectivityChanged);
         Ok(())
     }
 
@@ -942,6 +1076,23 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mdns_default_behaviour() -> Result<()> {
+        let t = &TestContext::new_alice().await;
+        assert!(t.should_request_mdns().await?);
+        assert!(t.should_send_mdns().await?);
+        assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
+        // The setting should be displayed correctly.
+        assert!(t.get_config_bool(Config::MdnsEnabled).await?);
+
+        t.set_config_bool(Config::Bot, true).await?;
+        assert!(!t.should_request_mdns().await?);
+        assert!(t.should_send_mdns().await?);
+        assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
+        assert!(t.get_config_bool(Config::MdnsEnabled).await?);
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sync() -> Result<()> {
         let alice0 = TestContext::new_alice().await;
@@ -968,20 +1119,16 @@ mod tests {
         // Reset to default. Test that it's not synced because defaults may differ across client
         // versions.
         alice0.set_config(Config::MdnsEnabled, None).await?;
-        assert_eq!(alice0.get_config_bool(Config::MdnsEnabled).await?, true);
         alice0.set_config_bool(Config::MdnsEnabled, false).await?;
         sync(&alice0, &alice1).await;
         assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
 
-        let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
-        alice0
-            .set_config_bool(Config::ShowEmails, !show_emails)
-            .await?;
-        sync(&alice0, &alice1).await;
-        assert_eq!(
-            alice1.get_config_bool(Config::ShowEmails).await?,
-            !show_emails
-        );
+        for key in [Config::ShowEmails, Config::MvboxMove] {
+            let val = alice0.get_config_bool(key).await?;
+            alice0.set_config_bool(key, !val).await?;
+            sync(&alice0, &alice1).await;
+            assert_eq!(alice1.get_config_bool(key).await?, !val);
+        }
 
         // `Config::SyncMsgs` mustn't be synced.
         alice0.set_config_bool(Config::SyncMsgs, false).await?;
@@ -1046,7 +1193,8 @@ mod tests {
 
         let status = "Synced via usual message";
         alice0.set_config(Config::Selfstatus, Some(status)).await?;
-        alice0.pop_sent_msg().await; // Sync message
+        alice0.send_sync_msg().await?;
+        alice0.pop_sent_sync_msg().await;
         let status1 = "Synced via sync message";
         alice1.set_config(Config::Selfstatus, Some(status1)).await?;
         tcm.send_recv(alice0, alice1, "hi Alice!").await;
@@ -1069,7 +1217,8 @@ mod tests {
         alice0
             .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
             .await?;
-        alice0.pop_sent_msg().await; // Sync message
+        alice0.send_sync_msg().await?;
+        alice0.pop_sent_sync_msg().await;
         let file = alice1.dir.path().join("avatar.jpg");
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         tokio::fs::write(&file, bytes).await?;

src/configure.rs

@@ -11,28 +11,31 @@
 
 mod auto_mozilla;
 mod auto_outlook;
-mod server_params;
+pub(crate) mod server_params;
 
-use anyhow::{bail, ensure, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
 use deltachat_contact_tools::EmailAddress;
 use futures::FutureExt;
 use futures_lite::FutureExt as _;
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use percent_encoding::utf8_percent_encode;
 use server_params::{expand_param_vector, ServerParams};
 use tokio::task;
 
 use crate::config::{self, Config};
+use crate::constants::NON_ALPHANUMERIC_WITHOUT_DOT;
 use crate::context::Context;
-use crate::imap::{session::Session as ImapSession, Imap};
+use crate::imap::Imap;
 use crate::log::LogExt;
-use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
-use crate::message::{Message, Viewtype};
+use crate::login_param::{
+    ConfiguredCertificateChecks, ConfiguredLoginParam, ConfiguredServerLoginParam,
+    ConnectionCandidate, EnteredCertificateChecks, EnteredLoginParam,
+};
+use crate::message::Message;
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::socks::Socks5Config;
 use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::tools::time;
@@ -78,10 +81,7 @@ impl Context {
 
         let res = self
             .inner_configure()
-            .race(cancel_channel.recv().map(|_| {
-                progress!(self, 0);
-                Ok(())
-            }))
+            .race(cancel_channel.recv().map(|_| Err(format_err!("Cancelled"))))
             .await;
 
         self.free_ongoing().await;
@@ -110,29 +110,19 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::load_candidate_params(self).await?;
+        let param = EnteredLoginParam::load(self).await?;
         let old_addr = self.get_config(Config::ConfiguredAddr).await?;
-
-        // Reset our knowledge about whether the server is a chatmail server.
-        // We will update it when we connect to IMAP.
-        self.set_config_internal(Config::IsChatmail, None).await?;
-
-        let success = configure(self, &mut param).await;
-        self.set_config_internal(Config::NotifyAboutWrongPw, None)
-            .await?;
-
-        on_configure_completed(self, param, old_addr).await?;
-
-        success?;
+        let configured_param = configure(self, &param).await?;
         self.set_config_internal(Config::NotifyAboutWrongPw, Some("1"))
             .await?;
+        on_configure_completed(self, configured_param, old_addr).await?;
         Ok(())
     }
 }
 
 async fn on_configure_completed(
     context: &Context,
-    param: LoginParam,
+    param: ConfiguredLoginParam,
     old_addr: Option<String>,
 ) -> Result<()> {
     if let Some(provider) = param.provider {
@@ -153,8 +143,7 @@ async fn on_configure_completed(
         }
 
         if !provider.after_login_hint.is_empty() {
-            let mut msg = Message::new(Viewtype::Text);
-            msg.text = provider.after_login_hint.to_string();
+            let mut msg = Message::new_text(provider.after_login_hint.to_string());
             if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg))
                 .await
                 .is_err()
@@ -167,9 +156,9 @@ async fn on_configure_completed(
     if let Some(new_addr) = context.get_config(Config::ConfiguredAddr).await? {
         if let Some(old_addr) = old_addr {
             if !addr_cmp(&new_addr, &old_addr) {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text =
-                    stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await;
+                let mut msg = Message::new_text(
+                    stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await,
+                );
                 chat::add_device_msg(context, None, Some(&mut msg))
                     .await
                     .context("Cannot add AEAP explanation")
@@ -182,21 +171,28 @@ async fn on_configure_completed(
     Ok(())
 }
 
-async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
-    progress!(ctx, 1);
+/// Retrieves data from autoconfig and provider database
+/// to transform user-entered login parameters into complete configuration.
+async fn get_configured_param(
+    ctx: &Context,
+    param: &EnteredLoginParam,
+) -> Result<ConfiguredLoginParam> {
+    ensure!(!param.addr.is_empty(), "Missing email address.");
 
-    let socks5_config = param.socks5_config.clone();
-    let socks5_enabled = socks5_config.is_some();
+    ensure!(!param.imap.password.is_empty(), "Missing (IMAP) password.");
 
-    let ctx2 = ctx.clone();
-    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+    // SMTP password is an "advanced" setting. If unset, use the same password as for IMAP.
+    let smtp_password = if param.smtp.password.is_empty() {
+        param.imap.password.clone()
+    } else {
+        param.smtp.password.clone()
+    };
 
-    // Step 1: Load the parameters and check email-address and password
+    let proxy_config = param.proxy_config.clone();
+    let proxy_enabled = proxy_config.is_some();
 
-    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
-    // socks5 requests, this can work with socks5 too.  OAuth is always set either for both
-    // IMAP and SMTP or not at all.
-    if param.imap.oauth2 && !socks5_enabled {
+    let mut addr = param.addr.clone();
+    if param.oauth2 {
         // the used oauth2 addr may differ, check this.
         // if get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -205,7 +201,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             .and_then(|e| e.parse().ok())
         {
             info!(ctx, "Authorized address is {}", oauth2_addr);
-            param.addr = oauth2_addr;
+            addr = oauth2_addr;
             ctx.sql
                 .set_raw_config("addr", Some(param.addr.as_str()))
                 .await?;
@@ -216,11 +212,10 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     let parsed = EmailAddress::new(&param.addr).context("Bad email-address")?;
     let param_domain = parsed.domain;
-    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
 
-    // Step 2: Autoconfig
     progress!(ctx, 200);
 
+    let provider;
     let param_autoconfig;
     if param.imap.server.is_empty()
         && param.imap.port == 0
@@ -232,66 +227,48 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         && param.smtp.user.is_empty()
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
-
         info!(
             ctx,
             "checking internal provider-info for offline autoconfig"
         );
 
-        if let Some(provider) =
-            provider::get_provider_info(ctx, &param_domain, socks5_enabled).await
-        {
-            param.provider = Some(provider);
-            match provider.status {
-                provider::Status::Ok | provider::Status::Preparation => {
-                    if provider.server.is_empty() {
-                        info!(ctx, "offline autoconfig found, but no servers defined");
-                        param_autoconfig = None;
-                    } else {
-                        info!(ctx, "offline autoconfig found");
-                        let servers = provider
-                            .server
-                            .iter()
-                            .map(|s| ServerParams {
-                                protocol: s.protocol,
-                                socket: s.socket,
-                                hostname: s.hostname.to_string(),
-                                port: s.port,
-                                username: match s.username_pattern {
-                                    UsernamePattern::Email => param.addr.to_string(),
-                                    UsernamePattern::Emaillocalpart => {
-                                        if let Some(at) = param.addr.find('@') {
-                                            param.addr.split_at(at).0.to_string()
-                                        } else {
-                                            param.addr.to_string()
-                                        }
-                                    }
-                                },
-                                strict_tls: Some(provider.opt.strict_tls),
-                            })
-                            .collect();
+        provider = provider::get_provider_info(ctx, &param_domain, proxy_enabled).await;
+        if let Some(provider) = provider {
+            if provider.server.is_empty() {
+                info!(ctx, "Offline autoconfig found, but no servers defined.");
+                param_autoconfig = None;
+            } else {
+                info!(ctx, "Offline autoconfig found.");
+                let servers = provider
+                    .server
+                    .iter()
+                    .map(|s| ServerParams {
+                        protocol: s.protocol,
+                        socket: s.socket,
+                        hostname: s.hostname.to_string(),
+                        port: s.port,
+                        username: match s.username_pattern {
+                            UsernamePattern::Email => param.addr.to_string(),
+                            UsernamePattern::Emaillocalpart => {
+                                if let Some(at) = param.addr.find('@') {
+                                    param.addr.split_at(at).0.to_string()
+                                } else {
+                                    param.addr.to_string()
+                                }
+                            }
+                        },
+                    })
+                    .collect();
 
-                        param_autoconfig = Some(servers)
-                    }
-                }
-                provider::Status::Broken => {
-                    info!(ctx, "offline autoconfig found, provider is broken");
-                    param_autoconfig = None;
-                }
+                param_autoconfig = Some(servers)
             }
         } else {
             // Try receiving autoconfig
-            info!(ctx, "no offline autoconfig found");
-            param_autoconfig = if socks5_enabled {
-                // Currently we can't do http requests through socks5, to not leak
-                // the ip, just don't do online autoconfig
-                info!(ctx, "socks5 enabled, skipping autoconfig");
-                None
-            } else {
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
-            }
+            info!(ctx, "No offline autoconfig found.");
+            param_autoconfig = get_autoconfig(ctx, param, &param_domain).await;
         }
     } else {
+        provider = None;
         param_autoconfig = None;
     }
 
@@ -308,7 +285,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.imap.port,
             socket: param.imap.security,
             username: param.imap.user.clone(),
-            strict_tls: None,
         })
     }
     if !servers
@@ -321,144 +297,150 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.smtp.port,
             socket: param.smtp.security,
             username: param.smtp.user.clone(),
-            strict_tls: None,
         })
     }
 
-    // respect certificate setting from function parameters
-    for server in &mut servers {
-        let certificate_checks = match server.protocol {
-            Protocol::Imap => param.imap.certificate_checks,
-            Protocol::Smtp => param.smtp.certificate_checks,
-        };
-        server.strict_tls = match certificate_checks {
-            CertificateChecks::AcceptInvalidCertificates
-            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
-            CertificateChecks::Strict => Some(true),
-            CertificateChecks::Automatic => server.strict_tls,
-        };
-    }
-
     let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
+    let configured_login_param = ConfiguredLoginParam {
+        addr,
+        imap: servers
+            .iter()
+            .filter_map(|params| {
+                let Ok(security) = params.socket.try_into() else {
+                    return None;
+                };
+                if params.protocol == Protocol::Imap {
+                    Some(ConfiguredServerLoginParam {
+                        connection: ConnectionCandidate {
+                            host: params.hostname.clone(),
+                            port: params.port,
+                            security,
+                        },
+                        user: params.username.clone(),
+                    })
+                } else {
+                    None
+                }
+            })
+            .collect(),
+        imap_user: param.imap.user.clone(),
+        imap_password: param.imap.password.clone(),
+        smtp: servers
+            .iter()
+            .filter_map(|params| {
+                let Ok(security) = params.socket.try_into() else {
+                    return None;
+                };
+                if params.protocol == Protocol::Smtp {
+                    Some(ConfiguredServerLoginParam {
+                        connection: ConnectionCandidate {
+                            host: params.hostname.clone(),
+                            port: params.port,
+                            security,
+                        },
+                        user: params.username.clone(),
+                    })
+                } else {
+                    None
+                }
+            })
+            .collect(),
+        smtp_user: param.smtp.user.clone(),
+        smtp_password,
+        proxy_config: param.proxy_config.clone(),
+        provider,
+        certificate_checks: match param.certificate_checks {
+            EnteredCertificateChecks::Automatic => ConfiguredCertificateChecks::Automatic,
+            EnteredCertificateChecks::Strict => ConfiguredCertificateChecks::Strict,
+            EnteredCertificateChecks::AcceptInvalidCertificates
+            | EnteredCertificateChecks::AcceptInvalidCertificates2 => {
+                ConfiguredCertificateChecks::AcceptInvalidCertificates
+            }
+        },
+        oauth2: param.oauth2,
+    };
+    Ok(configured_login_param)
+}
+
+async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<ConfiguredLoginParam> {
+    progress!(ctx, 1);
+
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+
+    let configured_param = get_configured_param(ctx, param).await?;
+    let strict_tls = configured_param.strict_tls();
+
     progress!(ctx, 550);
 
     // Spawn SMTP configuration task
-    let mut smtp = Smtp::new();
-
+    // to try SMTP while connecting to IMAP.
     let context_smtp = ctx.clone();
-    let mut smtp_param = param.smtp.clone();
-    let smtp_addr = param.addr.clone();
-    let smtp_servers: Vec<ServerParams> = servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::Smtp)
-        .cloned()
-        .collect();
-    let provider_strict_tls = param
-        .provider
-        .map_or(socks5_config.is_some(), |provider| provider.opt.strict_tls);
+    let smtp_param = configured_param.smtp.clone();
+    let smtp_password = configured_param.smtp_password.clone();
+    let smtp_addr = configured_param.addr.clone();
+    let proxy_config = configured_param.proxy_config.clone();
 
     let smtp_config_task = task::spawn(async move {
-        let mut smtp_configured = false;
-        let mut errors = Vec::new();
-        for smtp_server in smtp_servers {
-            smtp_param.user.clone_from(&smtp_server.username);
-            smtp_param.server.clone_from(&smtp_server.hostname);
-            smtp_param.port = smtp_server.port;
-            smtp_param.security = smtp_server.socket;
-            smtp_param.certificate_checks = match smtp_server.strict_tls {
-                Some(true) => CertificateChecks::Strict,
-                Some(false) => CertificateChecks::AcceptInvalidCertificates,
-                None => CertificateChecks::Automatic,
-            };
+        let mut smtp = Smtp::new();
+        smtp.connect(
+            &context_smtp,
+            &smtp_param,
+            &smtp_password,
+            &proxy_config,
+            &smtp_addr,
+            strict_tls,
+            configured_param.oauth2,
+        )
+        .await?;
 
-            match try_smtp_one_param(
-                &context_smtp,
-                &smtp_param,
-                &socks5_config,
-                &smtp_addr,
-                provider_strict_tls,
-                &mut smtp,
-            )
-            .await
-            {
-                Ok(_) => {
-                    smtp_configured = true;
-                    break;
-                }
-                Err(e) => errors.push(e),
-            }
-        }
-
-        if smtp_configured {
-            Ok(smtp_param)
-        } else {
-            Err(errors)
-        }
+        Ok::<(), anyhow::Error>(())
     });
 
     progress!(ctx, 600);
 
     // Configure IMAP
 
-    let mut imap: Option<(Imap, ImapSession)> = None;
-    let imap_servers: Vec<&ServerParams> = servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::Imap)
-        .collect();
-    let imap_servers_count = imap_servers.len();
-    let mut errors = Vec::new();
-    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
-        param.imap.user.clone_from(&imap_server.username);
-        param.imap.server.clone_from(&imap_server.hostname);
-        param.imap.port = imap_server.port;
-        param.imap.security = imap_server.socket;
-        param.imap.certificate_checks = match imap_server.strict_tls {
-            Some(true) => CertificateChecks::Strict,
-            Some(false) => CertificateChecks::AcceptInvalidCertificates,
-            None => CertificateChecks::Automatic,
-        };
-
-        match try_imap_one_param(
-            ctx,
-            &param.imap,
-            &param.socks5_config,
-            &param.addr,
-            provider_strict_tls,
-        )
-        .await
-        {
-            Ok(configured_imap) => {
-                imap = Some(configured_imap);
-                break;
-            }
-            Err(e) => errors.push(e),
-        }
-        progress!(
-            ctx,
-            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
-        );
-    }
-    let (mut imap, mut imap_session) = match imap {
-        Some(imap) => imap,
-        None => bail!(nicer_configuration_error(ctx, errors).await),
+    let (_s, r) = async_channel::bounded(1);
+    let mut imap = Imap::new(
+        configured_param.imap.clone(),
+        configured_param.imap_password.clone(),
+        configured_param.proxy_config.clone(),
+        &configured_param.addr,
+        strict_tls,
+        configured_param.oauth2,
+        r,
+    );
+    let configuring = true;
+    let mut imap_session = match imap.connect(ctx, configuring).await {
+        Ok(session) => session,
+        Err(err) => bail!("{}", nicer_configuration_error(ctx, err.to_string()).await),
     };
 
     progress!(ctx, 850);
 
     // Wait for SMTP configuration
-    match smtp_config_task.await.unwrap() {
-        Ok(smtp_param) => {
-            param.smtp = smtp_param;
-        }
-        Err(errors) => {
-            bail!(nicer_configuration_error(ctx, errors).await);
-        }
-    }
+    smtp_config_task.await.unwrap()?;
 
     progress!(ctx, 900);
 
-    if imap_session.is_chatmail() {
+    let is_chatmail = match ctx.get_config_bool(Config::FixIsChatmail).await? {
+        false => {
+            let is_chatmail = imap_session.is_chatmail();
+            ctx.set_config(
+                Config::IsChatmail,
+                Some(match is_chatmail {
+                    false => "0",
+                    true => "1",
+                }),
+            )
+            .await?;
+            is_chatmail
+        }
+        true => ctx.get_config_bool(Config::IsChatmail).await?,
+    };
+    if is_chatmail {
         ctx.set_config(Config::SentboxWatch, None).await?;
         ctx.set_config(Config::MvboxMove, Some("0")).await?;
         ctx.set_config(Config::OnlyFetchMvbox, None).await?;
@@ -466,8 +448,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         ctx.set_config(Config::E2eeEnabled, Some("1")).await?;
     }
 
-    let create_mvbox = ctx.should_watch_mvbox().await?;
-
+    let create_mvbox = !is_chatmail;
     imap.configure_folders(ctx, &mut imap_session, create_mvbox)
         .await?;
 
@@ -488,8 +469,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         }
     }
 
-    // the trailing underscore is correct
-    param.save_as_configured_params(ctx).await?;
+    configured_param.save_as_configured_params(ctx).await?;
     ctx.set_config_internal(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -507,25 +487,34 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     ctx.sql.set_raw_config_bool("configured", true).await?;
 
-    Ok(())
+    Ok(configured_param)
 }
 
 /// Retrieve available autoconfigurations.
 ///
-/// A Search configurations from the domain used in the email-address, prefer encrypted
-/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
+/// A. Search configurations from the domain used in the email-address
+/// B. If we have no configuration yet, search configuration in Thunderbird's central database
 async fn get_autoconfig(
     ctx: &Context,
-    param: &LoginParam,
+    param: &EnteredLoginParam,
     param_domain: &str,
-    param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
+    // Make sure to not encode `.` as `%2E` here.
+    // Some servers like murena.io on 2024-11-01 produce incorrect autoconfig XML
+    // when address is encoded.
+    // E.g.
+    // <https://autoconfig.murena.io/mail/config-v1.1.xml?emailaddress=foobar%40example%2Eorg>
+    // produced XML file with `<username>foobar@example%2Eorg</username>`
+    // resulting in failure to log in.
+    let param_addr_urlencoded =
+        utf8_percent_encode(&param.addr, NON_ALPHANUMERIC_WITHOUT_DOT).to_string();
+
     if let Ok(res) = moz_autoconfigure(
         ctx,
         &format!(
             "https://autoconfig.{param_domain}/mail/config-v1.1.xml?emailaddress={param_addr_urlencoded}"
         ),
-        param,
+        &param.addr,
     )
     .await
     {
@@ -540,7 +529,7 @@ async fn get_autoconfig(
             "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
             &param_domain, &param_addr_urlencoded
         ),
-        param,
+        &param.addr,
     )
     .await
     {
@@ -576,7 +565,7 @@ async fn get_autoconfig(
     if let Ok(res) = moz_autoconfigure(
         ctx,
         &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
-        param,
+        &param.addr,
     )
     .await
     {
@@ -586,140 +575,19 @@ async fn get_autoconfig(
     None
 }
 
-async fn try_imap_one_param(
-    context: &Context,
-    param: &ServerLoginParam,
-    socks5_config: &Option<Socks5Config>,
-    addr: &str,
-    provider_strict_tls: bool,
-) -> Result<(Imap, ImapSession), ConfigurationError> {
-    let inf = format!(
-        "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
-        param.user,
-        param.server,
-        param.port,
-        param.security,
-        param.certificate_checks,
-        param.oauth2,
-        if let Some(socks5_config) = socks5_config {
-            socks5_config.to_string()
-        } else {
-            "None".to_string()
-        }
-    );
-    info!(context, "Trying: {}", inf);
-
-    let (_s, r) = async_channel::bounded(1);
-
-    let mut imap = match Imap::new(param, socks5_config.clone(), addr, provider_strict_tls, r) {
-        Err(err) => {
-            info!(context, "failure: {:#}", err);
-            return Err(ConfigurationError {
-                config: inf,
-                msg: format!("{err:#}"),
-            });
-        }
-        Ok(imap) => imap,
-    };
-
-    match imap.connect(context).await {
-        Err(err) => {
-            info!(context, "failure: {:#}", err);
-            Err(ConfigurationError {
-                config: inf,
-                msg: format!("{err:#}"),
-            })
-        }
-        Ok(session) => {
-            info!(context, "success: {}", inf);
-            Ok((imap, session))
-        }
-    }
-}
-
-async fn try_smtp_one_param(
-    context: &Context,
-    param: &ServerLoginParam,
-    socks5_config: &Option<Socks5Config>,
-    addr: &str,
-    provider_strict_tls: bool,
-    smtp: &mut Smtp,
-) -> Result<(), ConfigurationError> {
-    let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
-        param.user,
-        param.server,
-        param.port,
-        param.security,
-        param.certificate_checks,
-        param.oauth2,
-        if let Some(socks5_config) = socks5_config {
-            socks5_config.to_string()
-        } else {
-            "None".to_string()
-        }
-    );
-    info!(context, "Trying: {}", inf);
-
-    if let Err(err) = smtp
-        .connect(context, param, socks5_config, addr, provider_strict_tls)
-        .await
+async fn nicer_configuration_error(context: &Context, e: String) -> String {
+    if e.to_lowercase().contains("could not resolve")
+        || e.to_lowercase().contains("connection attempts")
+        || e.to_lowercase()
+            .contains("temporary failure in name resolution")
+        || e.to_lowercase().contains("name or service not known")
+        || e.to_lowercase()
+            .contains("failed to lookup address information")
     {
-        info!(context, "failure: {}", err);
-        Err(ConfigurationError {
-            config: inf,
-            msg: format!("{err:#}"),
-        })
-    } else {
-        info!(context, "success: {}", inf);
-        smtp.disconnect();
-        Ok(())
-    }
-}
-
-/// Failure to connect and login with email client configuration.
-#[derive(Debug, thiserror::Error)]
-#[error("Trying {config}…\nError: {msg}")]
-pub struct ConfigurationError {
-    /// Tried configuration description.
-    config: String,
-
-    /// Error message.
-    msg: String,
-}
-
-async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
-    let first_err = if let Some(f) = errors.first() {
-        f
-    } else {
-        // This means configuration failed but no errors have been captured. This should never
-        // happen, but if it does, the user will see classic "Error: no error".
-        return "no error".to_string();
-    };
-
-    if errors.iter().all(|e| {
-        e.msg.to_lowercase().contains("could not resolve")
-            || e.msg.to_lowercase().contains("no dns resolution results")
-            || e.msg
-                .to_lowercase()
-                .contains("temporary failure in name resolution")
-            || e.msg.to_lowercase().contains("name or service not known")
-            || e.msg
-                .to_lowercase()
-                .contains("failed to lookup address information")
-    }) {
         return stock_str::error_no_network(context).await;
     }
 
-    if errors.iter().all(|e| e.msg == first_err.msg) {
-        return first_err.msg.to_string();
-    }
-
-    errors
-        .iter()
-        .map(|e| e.to_string())
-        .collect::<Vec<String>>()
-        .join("\n\n")
+    e
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -729,7 +597,7 @@ pub enum Error {
 
     #[error("XML error at position {position}: {error}")]
     InvalidXml {
-        position: usize,
+        position: u64,
         #[source]
         error: quick_xml::Error,
     },
@@ -743,9 +611,9 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
+    use super::*;
     use crate::config::Config;
+    use crate::login_param::EnteredServerLoginParam;
     use crate::test_utils::TestContext;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -757,4 +625,24 @@ mod tests {
         t.set_config(Config::MailPw, Some("123456")).await.unwrap();
         assert!(t.configure().await.is_err());
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_configured_param() -> Result<()> {
+        let t = &TestContext::new().await;
+        let entered_param = EnteredLoginParam {
+            addr: "alice@example.org".to_string(),
+
+            imap: EnteredServerLoginParam {
+                user: "alice@example.net".to_string(),
+                password: "foobar".to_string(),
+                ..Default::default()
+            },
+
+            ..Default::default()
+        };
+        let configured_param = get_configured_param(t, &entered_param).await?;
+        assert_eq!(configured_param.imap_user, "alice@example.net");
+        assert_eq!(configured_param.smtp_user, "");
+        Ok(())
+    }
 }

src/configure/auto_mozilla.rs

@@ -9,7 +9,6 @@ use quick_xml::events::{BytesStart, Event};
 
 use super::{Error, ServerParams};
 use crate::context::Context;
-use crate::login_param::LoginParam;
 use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
@@ -80,7 +79,7 @@ fn parse_server<B: BufRead>(
         })
         .map(|typ| {
             typ.unwrap()
-                .decode_and_unescape_value(reader)
+                .decode_and_unescape_value(reader.decoder())
                 .unwrap_or_default()
                 .to_lowercase()
         })
@@ -191,7 +190,7 @@ fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoco
     };
 
     let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+    reader.config_mut().trim_text(true);
 
     let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
         position: reader.buffer_position(),
@@ -248,7 +247,6 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
                 hostname: server.hostname,
                 port: server.port,
                 username: server.username,
-                strict_tls: None,
             })
         })
         .collect();
@@ -258,11 +256,11 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
-    param_in: &LoginParam,
+    addr: &str,
 ) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_serverparams(&param_in.addr, &xml_raw);
+    let res = parse_serverparams(addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,
@@ -274,8 +272,6 @@ pub(crate) async fn moz_autoconfigure(
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
 
     #[test]

src/configure/auto_outlook.rs

@@ -162,7 +162,7 @@ fn parse_xml_reader<B: BufRead>(
 
 fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+    reader.config_mut().trim_text(true);
 
     parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
         position: reader.buffer_position(),
@@ -187,7 +187,6 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
                 hostname: protocol.server,
                 port: protocol.port,
                 username: String::new(),
-                strict_tls: None,
             })
         })
         .collect()
@@ -216,8 +215,6 @@ pub(crate) async fn outlk_autodiscover(
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
 
     #[test]

src/configure/server_params.rs

@@ -22,31 +22,18 @@ pub(crate) struct ServerParams {
 
     /// Username, empty if unknown.
     pub username: String,
-
-    /// Whether TLS certificates should be strictly checked or not, `None` for automatic.
-    pub strict_tls: Option<bool>,
 }
 
 impl ServerParams {
     fn expand_usernames(self, addr: &str) -> Vec<ServerParams> {
-        let mut res = Vec::new();
-
         if self.username.is_empty() {
-            res.push(Self {
+            vec![Self {
                 username: addr.to_string(),
                 ..self.clone()
-            });
-
-            if let Some(at) = addr.find('@') {
-                res.push(Self {
-                    username: addr.split_at(at).0.to_string(),
-                    ..self
-                });
-            }
+            }]
         } else {
-            res.push(self)
+            vec![self]
         }
-        res
     }
 
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
@@ -135,14 +122,6 @@ impl ServerParams {
             vec![self]
         }
     }
-
-    fn expand_strict_tls(self) -> Vec<ServerParams> {
-        vec![Self {
-            // Strict if not set by the user or provider database.
-            strict_tls: Some(self.strict_tls.unwrap_or(true)),
-            ..self
-        }]
-    }
 }
 
 /// Expands vector of `ServerParams`, replacing placeholders with
@@ -155,9 +134,7 @@ pub(crate) fn expand_param_vector(
     v.into_iter()
         // The order of expansion is important.
         //
-        // Ports are expanded the last, so they are changed the first.  Username is only changed if
-        // default value (address with domain) didn't work for all available hosts and ports.
-        .flat_map(|params| params.expand_strict_tls().into_iter())
+        // Ports are expanded the last, so they are changed the first.
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
         .flat_map(|params| params.expand_ports().into_iter())
@@ -177,7 +154,6 @@ mod tests {
                 port: 0,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -191,7 +167,6 @@ mod tests {
                 port: 993,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true)
             }],
         );
 
@@ -202,7 +177,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
-                strict_tls: None,
             }],
             "foobar@example.net",
             "example.net",
@@ -217,7 +191,6 @@ mod tests {
                     port: 123,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true),
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
@@ -225,12 +198,10 @@ mod tests {
                     port: 123,
                     socket: Socket::Starttls,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
             ],
         );
 
-        // Test that strict_tls is not expanded for plaintext connections.
         let v = expand_param_vector(
             vec![ServerParams {
                 protocol: Protocol::Smtp,
@@ -238,7 +209,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Plain,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -251,7 +221,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Plain,
                 username: "foobar".to_string(),
-                strict_tls: Some(true)
             }],
         );
 
@@ -263,7 +232,6 @@ mod tests {
                 port: 10480,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -277,7 +245,6 @@ mod tests {
                     port: 10480,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
                 ServerParams {
                     protocol: Protocol::Imap,
@@ -285,7 +252,6 @@ mod tests {
                     port: 10480,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
                 ServerParams {
                     protocol: Protocol::Imap,
@@ -293,7 +259,6 @@ mod tests {
                     port: 10480,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 }
             ],
         );
@@ -307,7 +272,6 @@ mod tests {
                 port: 0,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -321,7 +285,6 @@ mod tests {
                     port: 465,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
@@ -329,7 +292,45 @@ mod tests {
                     port: 587,
                     socket: Socket::Starttls,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
+                },
+            ],
+        );
+
+        // Test that email address is used as the default username.
+        // We do not try other usernames
+        // such as the local part of the address
+        // as this is very uncommon configuration
+        // and not worth doubling the number of candidates to try.
+        // If such configuration is used, email provider
+        // should provide XML autoconfig or
+        // be added to the provider database as an exception.
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Imap,
+                hostname: "example.net".to_string(),
+                port: 0,
+                socket: Socket::Automatic,
+                username: "".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 993,
+                    socket: Socket::Ssl,
+                    username: "foobar@example.net".to_string(),
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 143,
+                    socket: Socket::Starttls,
+                    username: "foobar@example.net".to_string(),
                 },
             ],
         );