chore(release): prepare for 1.132.1

We do not want all_work_done() to return true immediately
after calling start_io(), but only when connection goes idle.

"Connected" state is set immediately after connecting to the server,
but it does not mean there is nothing to do.

This change make all_work_done() return false
from the Connected state and introduces a new Idle
connectivity state that is only set before connection
actually goes idle. For idle state all_work_done() returns true.

From the user point of view both old Connected state
and new Idle state look the same.

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.73.0
+      RUSTUP_TOOLCHAIN: 1.74.1
     steps:
       - uses: actions/checkout@v3
       - name: Install rustfmt and clippy
@@ -76,11 +76,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.73.0
+            rust: 1.74.1
           - os: windows-latest
-            rust: 1.73.0
+            rust: 1.74.1
           - os: macos-latest
-            rust: 1.73.0
+            rust: 1.74.1
 
           # Minimum Supported Rust Version = 1.70.0
           - os: ubuntu-latest

.github/workflows/jsonrpc-client-npm-package.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/jsonrpc.yml

@@ -15,10 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
       - name: Add Rust cache
         uses: Swatinem/rust-cache@v2
       - name: npm install

.github/workflows/node-docs.yml

@@ -16,10 +16,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
 
       - name: npm install and generate documentation
         working-directory: node

.github/workflows/node-package.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: System info
         run: |
           rustc -vV
@@ -66,10 +66,8 @@ jobs:
     runs-on: ubuntu-latest
 
     # Build Linux prebuilds inside a container with old glibc for backwards compatibility.
-    # Debian 10 contained glibc 2.28 at the time of the writing (2023-06-04): https://packages.debian.org/buster/libc6
-    # Ubuntu 18.04 is at the End of Standard Support since June 2023, but it contains glibc 2.27,
-    # so we are using it to support Ubuntu 18.04 setups that are still not upgraded.
-    container: ubuntu:18.04
+    # Debian 10 contained glibc 2.28: https://packages.debian.org/buster/libc6
+    container: debian:10
     steps:
       # Working directory is owned by 1001:1001 by default.
       # Change it to our user.
@@ -80,7 +78,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - run: apt-get update
 
       # Python is needed for node-gyp
@@ -143,7 +141,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v2
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/node-tests.yml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: System info
         run: |
           rustc -vV

.github/workflows/upload-python-docs.yml

@@ -0,0 +1,25 @@
+name: Build & Deploy Documentation on py.delta.chat
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - name: Build Python documentation
+        run: scripts/build-python-docs.sh
+      - name: Upload to py.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: delta
+          KEY: ${{ secrets.CODESPEAK_KEY }}
+          HOST: "lists.codespeak.net"
+          SOURCE: "dist/html/"
+          TARGET: "/home/delta/build/master"

CHANGELOG.md

@@ -1,5 +1,295 @@
 # Changelog
 
+## [1.132.1] - 2023-12-12
+
+### Features / Changes
+
+- Add "From:" to protected headers for signed-only messages.
+- Sync user actions for ad-hoc groups across devices ([#5065](https://github.com/deltachat/deltachat-core-rust/pull/5065)).
+
+### Fixes
+
+- Add padlock to empty part if the whole message is empty.
+- Renew IDLE timeout on keepalives and reduce it to 5 minutes.
+- connectivity: Return false from `all_work_done()` immediately after connecting (iOS notification fix).
+
+### API-Changes
+
+- deltachat-jsonrpc-client: add `Account.{import,export}_self_keys`.
+
+### CI
+
+- Update to Rust 1.74.1.
+
+## [1.132.0] - 2023-12-06
+
+### Features / Changes
+
+- Increase TCP timeouts from 30 to 60 seconds.
+
+### Fixes
+
+- Don't sort message creating a protected group over a protection message ([#4963](https://github.com/deltachat/deltachat-core-rust/pull/4963)).
+- Do not lock accounts.toml on iOS.
+- Protect groups even if some members are not verified and add `test_securejoin_after_contact_resetup` regression test.
+
+## [1.131.9] - 2023-12-02
+
+### API-Changes
+
+- Remove `dc_get_http_response()`, `dc_http_response_get_mimetype()`, `dc_http_response_get_encoding()`, `dc_http_response_get_blob()`, `dc_http_response_get_size()`, `dc_http_response_unref()` and `dc_http_response_t` from cffi.
+- Deprecate CFFI APIs `dc_send_reaction()`, `dc_get_msg_reactions()`, `dc_reactions_get_contacts()`, `dc_reactions_get_by_contact_id()`, `dc_reactions_unref` and `dc_reactions_t`.
+- Make `Contact.is_verified()` return bool.
+
+### Build system
+
+- Switch from fork of iroh to iroh 0.4.2 pre-release.
+
+### Features / Changes
+
+- Send `Chat-Verified` headers in 1:1 chats.
+- Ratelimit IMAP connections ([#4940](https://github.com/deltachat/deltachat-core-rust/pull/4940)).
+- Remove receiver limit on `.xdc` size.
+- Don't affect MimeMessage with "From" and secured headers from encrypted unsigned messages.
+- Sync `Config::{MdnsEnabled,ShowEmails}` across devices ([#4954](https://github.com/deltachat/deltachat-core-rust/pull/4954)).
+- Sync `Config::Displayname` across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- `Chat::rename_ex`: Don't send sync message if usual message is sent.
+
+### Fixes
+
+- Lock the database when INSERTing a webxdc update, avoid "Database is locked" errors.
+- Use keyring with all private keys when decrypting a message ([#5046](https://github.com/deltachat/deltachat-core-rust/pull/5046)).
+
+### Tests
+
+- Make Result-returning tests produce a line number.
+- Add `test_utils::sync()`.
+- Test inserting lots of webxdc updates.
+- Split `test_sync_alter_chat()` into smaller tests.
+
+## [1.131.8] - 2023-11-27
+
+### Features / Changes
+
+- webxdc: Add unique IDs to status updates sent outside and deduplicate based on IDs.
+
+### Fixes
+
+- Allow IMAP servers not returning UIDNEXT on SELECT and STATUS such as mail.163.com.
+- Use the correct securejoin strings used in the UI, remove old TODO ([#5047](https://github.com/deltachat/deltachat-core-rust/pull/5047)).
+- Do not emit events about webxdc update events logged into debug log webxdc.
+
+### Tests
+
+- Check that `receive_status_update` has forward compatibility and unique webxdc IDs will be ignored by previous Delta Chat versions.
+
+## [1.131.7] - 2023-11-24
+
+### Fixes
+
+- Revert "fix: check UIDNEXT with a STATUS command before going IDLE". This attempts to fix mail.163.com which has broken STATUS command.
+
+## [1.131.6] - 2023-11-21
+
+### Fixes
+
+- Fail fast if IMAP FETCH cannot be parsed instead of getting stuck in infinite loop.
+
+### Documentation
+
+- Generate deltachat-rpc-client documentation and publish it to <https://py.delta.chat>.
+
+## [1.131.5] - 2023-11-20
+
+### API-Changes
+
+- deltachat-rpc-client: Add `Message.get_sender_contact()`.
+- Turn `ContactAddress` into an owned type.
+
+### Features / Changes
+
+- Lowercase addresses in Autocrypt and Autocrypt-Gossip headers.
+- Lowercase the address in member added/removed messages.
+- Lowercase `addr` when it is set.
+- Do not replace the message with an error in square brackets when the sender is not a member of the protected group.
+
+### Fixes
+
+- `Chat::sync_contacts()`: Fetch contact addresses in a single query.
+- `Chat::rename_ex()`: Sync improved chat name to other devices.
+- Recognize `Chat-Group-Member-Added` of self case-insensitively.
+- Compare verifier addr to peerstate addr case-insensitively.
+
+### Tests
+
+- Port [Secure-Join](https://securejoin.readthedocs.io/) tests to JSON-RPC.
+
+### CI
+
+- Test with Rust 1.74.
+
+
+## [1.131.4] - 2023-11-16
+
+### Documentation
+
+- Document DC_DOWNLOAD_UNDECIPHERABLE.
+
+### Fixes
+
+- Always add "Member added" as system message.
+
+## [1.131.3] - 2023-11-15
+
+### Fixes
+
+- Update async-imap to 0.9.4 which does not ignore EOF on FETCH.
+- Reset gossiped timestamp on securejoin.
+- sync: Ignore unknown sync items to provide forward compatibility and avoid creating empty message bubbles.
+- sync: Skip sync when chat name is set to the current one.
+- Return connectivity HTML with an error when IO is stopped.
+
+## [1.131.2] - 2023-11-14
+
+### API-Changes
+
+- deltachat-rpc-client: add `Account.get_chat_by_contact()`.
+
+### Features / Changes
+
+- Do not post "... verified" messages on QR scan success.
+- Never drop better message from `apply_group_changes()`.
+
+### Fixes
+
+- Assign MDNs to the trash chat early to prevent received MDNs from creating or unblocking 1:1 chats.
+- Allow to securejoin groups when 1:1 chat with the inviter is a contact request.
+- Add "setup changed" message for verified key before the message.
+- Ignore special chats when calculating similar chats.
+
+## [1.131.1] - 2023-11-13
+
+### Fixes
+
+- Do not skip actual message parts when group change messages are inserted.
+
+## [1.131.0] - 2023-11-13
+
+### Features / Changes
+
+- Sync chat contacts across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Sync creating broadcast lists across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Sync Chat::name across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Multi-device broadcast lists ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+
+### Fixes
+
+- Encode chat name in the `List-ID` header to avoid SMTPUTF8 errors.
+- Ignore errors from generating sync messages.
+- `Context::execute_sync_items`: Ignore all errors ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+- Allow to send unverified securejoin messages to protected chats ([#4982](https://github.com/deltachat/deltachat-core-rust/pull/4982)).
+
+## [1.130.0] - 2023-11-10
+
+### API-Changes
+
+- Emit JoinerProgress(1000) event when Bob verifies Alice.
+- JSON-RPC: add `ContactObject.is_profile_verified` property.
+- Hide `ChatId::get_for_contact()` from public API.
+
+### Features / Changes
+
+- Add secondary verified key.
+- Add info messages about implicitly added members.
+- Treat reset state as encryption not preferred.
+- Grow sleep durations on errors in Imap::fake_idle() ([#4424](https://github.com/deltachat/deltachat-core-rust/pull/4424)).
+
+### Fixes
+
+- Mark 1:1 chat as protected when joining a group.
+- Raise lower auto-download limit to 160k.
+- Remove `Reporting-UA` from read receipts.
+- Do not apply group changes to special chats. Avoid adding members to the trash chat.
+- imap: make `UidGrouper` robust against duplicate UIDs.
+- Do not return hidden chat from `dc_get_chat_id_by_contact_id`.
+- Smtp_loop(): Don't grow timeout if interrupted early ([#4833](https://github.com/deltachat/deltachat-core-rust/pull/4833)).
+
+### Refactor
+
+- imap: Do not FETCH right after `scan_folders()`.
+- deltachat-rpc-client: Use `itertools` instead of `Lock` for thread-safe request ID generation.
+
+### Tests
+
+- Remove unused `--liveconfig` option.
+- Test chatlist can load for corrupted chats ([#4979](https://github.com/deltachat/deltachat-core-rust/pull/4979)).
+
+### Miscellaneous Tasks
+
+- Update provider-db ([#4949](https://github.com/deltachat/deltachat-core-rust/pull/4949)).
+
+## [1.129.1] - 2023-11-06
+
+### Fixes
+
+- Update tokio-imap to fix Outlook STATUS parsing bug.
+- deltachat-rpc-client: Add the Lock around request ID.
+- `apply_group_changes`: Don't implicitly delete members locally, add absent ones instead ([#4934](https://github.com/deltachat/deltachat-core-rust/pull/4934)).
+- Partial messages do not change group state ([#4900](https://github.com/deltachat/deltachat-core-rust/pull/4900)).
+
+### Tests
+
+- Group chats device synchronisation.
+
+## [1.129.0] - 2023-11-06
+
+### API-Changes
+
+- Add JSON-RPC `get_chat_id_by_contact_id` API ([#4918](https://github.com/deltachat/deltachat-core-rust/pull/4918)).
+- [**breaking**] Remove deprecated `get_verifier_addr`.
+
+### Features / Changes
+
+- Sync chat `Blocked` state, chat visibility, chat mute duration and contact blocked status across devices ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+- Add 'group created instructions' as info message ([#4916](https://github.com/deltachat/deltachat-core-rust/pull/4916)).
+- Add hardcoded fallback DNS cache.
+
+### Fixes
+
+- Switch to `EncryptionPreference::Mutual` on a receipt of encrypted+signed message ([#4707](https://github.com/deltachat/deltachat-core-rust/pull/4707)).
+- imap: Check UIDNEXT with a STATUS command before going IDLE.
+- Allow to change verified key via "member added" message.
+- json-rpc: Return verifier even if the contact is not "verified" (Autocrypt key does not equal Secure-Join key).
+
+### Documentation
+
+- Refine `Contact::get_verifier_id` and `Contact::is_verified` documentation ([#4922](https://github.com/deltachat/deltachat-core-rust/pull/4922)).
+- Contact profile view should not use `dc_contact_is_verified()`.
+- Remove documentation for non-existing `dc_accounts_new` `os_name` param.
+
+### Refactor
+
+- Remove unused or useless code paths in Secure-Join ([#4897](https://github.com/deltachat/deltachat-core-rust/pull/4897)).
+- Improve error handling in Secure-Join code.
+- Add hostname to "no DNS resolution results" error message.
+- Accept `&str` instead of `Option<String>` in idle().
+
+## [1.128.0] - 2023-11-02
+
+### Build system
+- [**breaking**] Upgrade nodejs version to 18 ([#4903](https://github.com/deltachat/deltachat-core-rust/pull/4903)).
+
+### Features / Changes
+
+- deltachat-rpc-client: Add `Account.wait_for_incoming_msg_event()`.
+- Decrease ratelimit for .testrun.org subdomains.
+
+### Fixes
+
+- Do not fail securejoin due to unrelated pending bobstate  ([#4896](https://github.com/deltachat/deltachat-core-rust/pull/4896)).
+- Allow other verified group recipients to be unverified, only check the sender verification.
+- Remove not working attempt to recover from verified key changes.
+
 ## [1.127.2] - 2023-10-29
 
 ### API-Changes
@@ -36,6 +326,8 @@
 - [**breaking**] Remove unused `dc_set_chat_protection()`
 - Hide `DcSecretKey` trait from the API.
 - Verified 1:1 chats ([#4315](https://github.com/deltachat/deltachat-core-rust/pull/4315)). Disabled by default, enable with `verified_one_on_one_chats` config.
+- Add api `chat::Chat::is_protection_broken`
+- Add `dc_chat_is_protection_broken()` C API.
 
 ### CI
 
@@ -3049,3 +3341,19 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.127.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.1...v1.127.0
 [1.127.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.0...v1.127.1
 [1.127.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.1...v1.127.2
+[1.128.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.2...v1.128.0
+[1.129.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.128.0...v1.129.0
+[1.129.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.0...v1.129.1
+[1.130.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.1...v1.130.0
+[1.131.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.130.0...v1.131.0
+[1.131.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.0...v1.131.1
+[1.131.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.1...v1.131.2
+[1.131.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.2...v1.131.3
+[1.131.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.3...v1.131.4
+[1.131.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.4...v1.131.5
+[1.131.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.5...v1.131.6
+[1.131.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.6...v1.131.7
+[1.131.8]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.7...v1.131.8
+[1.131.9]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.8...v1.131.9
+[1.132.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.9...v1.132.0
+[1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1

CONTRIBUTING.md

@@ -86,6 +86,17 @@ For example:
 .with_context(|| format!("Unable to trash message {msg_id}"))
 ```
 
+All errors should be handled in one of these ways:
+- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
+- With `.log_err().ok()`.
+- Bubbled up with `?`.
+
+`backtrace` feature is enabled for `anyhow` crate
+and `debug = 1` option is set in the test profile.
+This allows to run `RUST_BACKTRACE=1 cargo test`
+and get a backtrace with line numbers in resultified tests
+which return `anyhow::Result`.
+
 ### Logging
 
 For logging, use `info!`, `warn!` and `error!` macros.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.127.2"
+version = "1.132.1"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.70"
@@ -11,6 +11,10 @@ panic = 'abort'
 opt-level = 1
 
 [profile.test]
+# Make anyhow `backtrace` feature useful.
+# With `debug = 0` there are no line numbers in the backtrace
+# produced with RUST_BACKTRACE=1.
+debug = 1
 opt-level = 0
 
 # Always optimize dependencies.
@@ -26,6 +30,9 @@ opt-level = "z"
 codegen-units = 1
 strip = true
 
+[patch.crates-io]
+imap-proto = { git = "https://github.com/djc/tokio-imap.git", rev = "01ff256a7e42a9f7d2732706f8b71a16ce93427e" }
+
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 format-flowed = { path = "./format-flowed" }
@@ -33,7 +40,7 @@ ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
 async-channel = "2.0.0"
-async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
+async-imap = { version = "0.9.5", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
@@ -44,15 +51,15 @@ chrono = { version = "0.4", default-features=false, features = ["clock", "std"]
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
-fast-socks5 = "0.8"
-fd-lock = "3.0.11"
+fast-socks5 = "0.9"
+fd-lock = "4"
 futures = "0.3"
 futures-lite = "2.0.0"
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
 image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh = { git = "https://github.com/deltachat/iroh", branch = "0.4-update-quic", default-features = false }
+iroh = { git = "https://github.com/n0-computer/iroh", branch = "maint-0.4", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -72,7 +79,7 @@ quick-xml = "0.31"
 rand = "0.8"
 regex = "1.9"
 reqwest = { version = "0.11.20", features = ["json"] }
-rusqlite = { version = "0.29", features = ["sqlcipher"] }
+rusqlite = { version = "0.30", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
 serde_json = "1.0"
@@ -90,19 +97,20 @@ tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 tokio-util = "0.7.9"
-toml = "0.7"
+toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
+anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
 futures-lite = "2.0.0"
 log = "0.4"
 pretty_env_logger = "0.5"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
-testdir = "0.8.0"
+testdir = "0.9.0"
 tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
 pretty_assertions = "1.3.0"
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.127.2"
+version = "1.132.1"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -25,7 +25,6 @@ typedef struct _dc_event     dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
 typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
 typedef struct _dc_backup_provider dc_backup_provider_t;
-typedef struct _dc_http_response dc_http_response_t;
 
 // Alias for backwards compatibility, use dc_event_emitter_t instead.
 typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
@@ -1112,6 +1111,7 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  * received overrides all previously received reactions. It is
  * possible to remove all reactions by sending an empty string.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `send_reaction` instead
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id ID of the message you react to.
@@ -1124,6 +1124,7 @@ uint32_t dc_send_reaction (dc_context_t* context, uint32_t msg_id, char *reactio
 /**
  * Get a structure with reactions to the message.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id The message ID to get reactions for.
@@ -2561,7 +2562,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     the Verified-Group-Invite protocol is offered in the QR code;
  *     works for protected groups as well as for normal groups.
  *     If set to 0, the Setup-Contact protocol is offered in the QR code.
- *     See https://countermitm.readthedocs.io/en/latest/new.html
+ *     See https://securejoin.readthedocs.io/en/latest/new.html
  *     for details about both protocols.
  * @return The text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
@@ -2597,7 +2598,7 @@ char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_
  *
  * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
- * See https://countermitm.readthedocs.io/en/latest/new.html
+ * See https://securejoin.readthedocs.io/en/latest/new.html
  * for details about both protocols.
  *
  * @memberof dc_context_t
@@ -2941,7 +2942,6 @@ int dc_receive_backup (dc_context_t* context, const char* qr);
  * use dc_accounts_remove_account().
  *
  * @memberof dc_accounts_t
- * @param os_name
  * @param dir The directory to create the context-databases in.
  *     If the directory does not exist,
  *     dc_accounts_new() will try to create it.
@@ -3732,8 +3732,22 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
 
 /**
  * Check if a chat is protected.
- * Protected chats contain only verified members and encryption is always enabled.
- * Protected chats are created using dc_create_group_chat() by setting the 'protect' parameter to 1.
+ *
+ * End-to-end encryption is guaranteed in protected chats
+ * and only verified contacts
+ * as determined by dc_contact_is_verified()
+ * can be added to protected chats.
+ *
+ * Protected chats are created using dc_create_group_chat()
+ * by setting the 'protect' parameter to 1.
+ * 1:1 chats become protected or unprotected automatically
+ * if `verified_one_on_one_chats` setting is enabled.
+ *
+ * UI should display a green checkmark
+ * in the chat title,
+ * in the chatlist item
+ * and in the chat profile
+ * if chat protection is enabled.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -4567,15 +4581,18 @@ int dc_msg_has_html (dc_msg_t* msg);
   * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
   *
   * The function returns one of:
-  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
-  *                                  and should be rendered as usual.
-  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
-  *                                  In addition to the usual message rendering,
-  *                                  the UI shall show a download button that calls dc_download_full_msg()
-  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
-  *                                  If the download fails or succeeds,
-  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
-  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
+  * - @ref DC_DOWNLOAD_DONE           - The message does not need any further download action
+  *                                     and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE      - There is additional content to download.
+  *                                     In addition to the usual message rendering,
+  *                                     the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS    - Download was started with dc_download_full_msg() and is still in progress.
+  *                                     If the download fails or succeeds,
+  *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  *
+  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
+  *                                     It was fully downloaded, but we failed to decrypt it.
+  * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
   *
   * @memberof dc_msg_t
   * @param msg The message object.
@@ -5033,10 +5050,16 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 
 
 /**
- * Check if a contact was verified. E.g. by a secure-join QR code scan
- * and if the key has not changed since this verification.
+ * Check if the contact
+ * can be added to verified chats,
+ * i.e. has a verified key
+ * and Autocrypt key matches the verified key.
  *
- * The UI may draw a checkbox or something like that beside verified contacts.
+ * If contact is verified
+ * UI should display green checkmark after the contact name
+ * in contact list items,
+ * in chat member list items
+ * and in profiles if no chat with the contact exist (otherwise, use dc_chat_is_protected()).
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5046,32 +5069,19 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
-
 /**
- * Return the address that verified a contact
+ * Return the contact ID that verified a contact.
  *
- * The UI may use this in addition to a checkmark showing the verification status.
- * In case of verification chains,
- * the last contact in the chain is shown.
- * This is because of privacy reasons, but also as it would not help the user
- * to see a unknown name here - where one can mostly always ask the shown name
- * as it is directly known.
+ * If the function returns non-zero result,
+ * display green checkmark in the profile and "Introduced by ..." line
+ * with the name and address of the contact
+ * formatted by dc_contact_get_name_n_addr.
  *
- * @memberof dc_contact_t
- * @param contact The contact object.
- * @return 
- *    A string containing the verifiers address. If it is the same address as the contact itself,
- *    we verified the contact ourself. If it is an empty string, we don't have verifier 
- *    information or the contact is not verified.
- * @deprecated 2023-09-28, use dc_contact_get_verifier_id instead
- */
-char*           dc_contact_get_verifier_addr       (dc_contact_t* contact);
-
-
-/**
- * Return the `ContactId` that verified a contact
- *
- * The UI may use this in addition to a checkmark showing the verification status
+ * If this function returns a verifier,
+ * this does not necessarily mean
+ * you can add the contact to verified chats.
+ * Use dc_contact_is_verified() to check
+ * if a contact can be added to a verified chat instead.
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5176,72 +5186,6 @@ int             dc_provider_get_status                (const dc_provider_t* prov
 void            dc_provider_unref                     (dc_provider_t* provider);
 
 
-/**
- * Return an HTTP(S) GET response.
- * This function can be used to download remote content for HTML emails.
- *
- * @memberof dc_context_t
- * @param context The context object to take proxy settings from.
- * @param url HTTP or HTTPS URL.
- * @return The response must be released using dc_http_response_unref() after usage.
- *     NULL is returned on errors.
- */
-dc_http_response_t*     dc_get_http_response      (const dc_context_t* context, const char* url);
-
-
-/**
- * @class dc_http_response_t
- *
- * An object containing an HTTP(S) GET response.
- * Created by dc_get_http_response().
- */
-
-
-/**
- * Returns HTTP response MIME type as a string, e.g. "text/plain" or "text/html".
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The string which must be released using dc_str_unref() after usage. May be NULL.
- */
-char*                   dc_http_response_get_mimetype (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response encoding, e.g. "utf-8".
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The string which must be released using dc_str_unref() after usage. May be NULL.
- */
-char*                   dc_http_response_get_encoding (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response contents.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The blob which must be released using dc_str_unref() after usage. NULL is never returned.
- */
-uint8_t*                dc_http_response_get_blob     (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response content size.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The blob size.
- */
-size_t                  dc_http_response_get_size     (const dc_http_response_t* response);
-
-/**
- * Free an HTTP response object.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- */
-void                    dc_http_response_unref        (const dc_http_response_t* response);
-
-
 /**
  * @class dc_lot_t
  *
@@ -5341,6 +5285,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * @class dc_reactions_t
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  *
  * An object representing all reactions for a single message.
  */
@@ -5348,6 +5293,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 /**
  * Returns array of contacts which reacted to the given message.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_reactions_t
  * @param reactions The object containing message reactions.
  * @return array of contact IDs. Use dc_array_get_cnt() to get array length and
@@ -5359,6 +5305,7 @@ dc_array_t*     dc_reactions_get_contacts(dc_reactions_t* reactions);
 /**
  * Returns a string containing space-separated reactions of a single contact.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_reactions_t
  * @param reactions The object containing message reactions.
  * @param contact_id ID of the contact.
@@ -5374,6 +5321,7 @@ char*           dc_reactions_get_by_contact_id(dc_reactions_t* reactions, uint32
  *
  * Reactions objects are created by dc_get_msg_reactions().
  *
+ * @deprecated 2023-11-27
  * @memberof dc_reactions_t
  * @param reactions The object to free.
  *     If NULL is given, nothing is done.
@@ -6234,6 +6182,7 @@ void dc_event_unref(dc_event_t* event);
  * @param data2 (int) The progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
+ *     1000=vg-member-added/vc-contact-confirm received
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
@@ -6426,22 +6375,27 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Download not needed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_DONE          0
+#define DC_DOWNLOAD_DONE           0
 
 /**
  * Download available, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_AVAILABLE    10
+#define DC_DOWNLOAD_AVAILABLE      10
 
 /**
  * Download failed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_FAILURE      20
+#define DC_DOWNLOAD_FAILURE        20
+
+/**
+ * Download not needed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_UNDECIPHERABLE 30
 
 /**
  * Download in progress, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_IN_PROGRESS  1000
+#define DC_DOWNLOAD_IN_PROGRESS    1000
 
 
 
@@ -6606,7 +6560,7 @@ void dc_event_unref(dc_event_t* event);
 /// - %1$s will be replaced by the name of the verified contact
 #define DC_STR_CONTACT_VERIFIED           35
 
-/// "Cannot verify %1$s."
+/// "Cannot establish guaranteed end-to-end encryption with %1$s."
 ///
 /// Used in status messages.
 /// - %1$s will be replaced by the name of the contact that cannot be verified
@@ -7260,6 +7214,11 @@ void dc_event_unref(dc_event_t* event);
 /// Used in info messages.
 #define DC_STR_CHAT_PROTECTION_DISABLED 171
 
+/// "Others will only see this group after you sent a first message."
+///
+/// Used as the first info messages in newly created groups.
+#define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -31,7 +31,6 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
 use deltachat::message::MsgId;
-use deltachat::net::read_url_blob;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
@@ -489,7 +488,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     if context.is_null() {
         return;
     }
-    let ctx = &*context;
+    let ctx = &mut *context;
 
     block_on(ctx.start_io())
 }
@@ -4113,27 +4112,17 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
 
-    block_on(ffi_contact.contact.is_verified(ctx))
+    if block_on(ffi_contact.contact.is_verified(ctx))
         .context("is_verified failed")
         .log_err(ctx)
-        .unwrap_or_default() as libc::c_int
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_verifier_addr(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_verifier_addr()");
-        return "".strdup();
-    }
-    let ffi_contact = &*contact;
-    let ctx = &*ffi_contact.context;
-    block_on(ffi_contact.contact.get_verifier_addr(ctx))
-        .context("failed to get verifier for contact")
-        .log_err(ctx)
         .unwrap_or_default()
-        .strdup()
+    {
+        // Return value is essentially a boolean,
+        // but we return 2 for true for backwards compatibility.
+        2
+    } else {
+        0
+    }
 }
 
 #[no_mangle]
@@ -4607,96 +4596,6 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     // this may change once we start localizing string.
 }
 
-// dc_http_response_t
-
-pub type dc_http_response_t = net::HttpResponse;
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_http_response(
-    context: *const dc_context_t,
-    url: *const libc::c_char,
-) -> *mut dc_http_response_t {
-    if context.is_null() || url.is_null() {
-        eprintln!("ignoring careless call to dc_get_http_response()");
-        return ptr::null_mut();
-    }
-
-    let context = &*context;
-    let url = to_string_lossy(url);
-    if let Ok(response) = block_on(read_url_blob(context, &url))
-        .context("read_url_blob")
-        .log_err(context)
-    {
-        Box::into_raw(Box::new(response))
-    } else {
-        ptr::null_mut()
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_mimetype(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_mimetype()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    response.mimetype.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_encoding(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_encoding()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    response.encoding.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_blob(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_blob()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    let blob_len = response.blob.len();
-    let ptr = libc::malloc(blob_len);
-    libc::memcpy(ptr, response.blob.as_ptr() as *mut libc::c_void, blob_len);
-    ptr as *mut libc::c_char
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_size(
-    response: *const dc_http_response_t,
-) -> libc::size_t {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_size()");
-        return 0;
-    }
-
-    let response = &*response;
-    response.blob.len()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_unref(response: *mut dc_http_response_t) {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_unref()");
-        return;
-    }
-    drop(Box::from_raw(response));
-}
-
 // -- Accounts
 
 /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using
@@ -4946,8 +4845,8 @@ pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
         return;
     }
 
-    let accounts = &*accounts;
-    block_on(async move { accounts.read().await.start_io().await });
+    let accounts = &mut *accounts;
+    block_on(async move { accounts.write().await.start_io().await });
 }
 
 #[no_mangle]

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.127.2"
+version = "1.132.1"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -30,7 +30,7 @@ walkdir = "2.3.3"
 base64 = "0.21"
 
 # optional dependencies
-axum = { version = "0.6.20", optional = true, features = ["ws"] }
+axum = { version = "0.7", optional = true, features = ["ws"] }
 env_logger = { version = "0.10.0", optional = true }
 
 [dev-dependencies]

deltachat-jsonrpc/src/api.rs

@@ -221,13 +221,13 @@ impl CommandApi {
 
     /// Starts background tasks for all accounts.
     async fn start_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.read().await.start_io().await;
+        self.accounts.write().await.start_io().await;
         Ok(())
     }
 
     /// Stops background tasks for all accounts.
     async fn stop_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.read().await.stop_io().await;
+        self.accounts.write().await.stop_io().await;
         Ok(())
     }
 
@@ -237,7 +237,7 @@ impl CommandApi {
 
     /// Starts background tasks for a single account.
     async fn start_io(&self, account_id: u32) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
+        let mut ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
@@ -383,7 +383,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
+        let mut ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -678,7 +678,7 @@ impl CommandApi {
     ///     the Verified-Group-Invite protocol is offered in the QR code;
     ///     works for protected groups as well as for normal groups.
     ///     If not set, the Setup-Contact protocol is offered in the QR code.
-    ///     See https://countermitm.readthedocs.io/en/latest/new.html
+    ///     See https://securejoin.readthedocs.io/en/latest/new.html
     ///     for details about both protocols.
     ///
     /// return format: `[code, svg]`
@@ -707,7 +707,7 @@ impl CommandApi {
     ///
     /// Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
     ///
-    /// See https://countermitm.readthedocs.io/en/latest/new.html
+    /// See https://securejoin.readthedocs.io/en/latest/new.html
     /// for details about both protocols.
     ///
     /// **qr**: The text of the scanned QR code. Typically, the same string as given
@@ -1391,6 +1391,19 @@ impl CommandApi {
     //                   chat
     // ---------------------------------------------
 
+    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id` if it exists.
+    ///
+    /// If it does not exist, `None` is returned.
+    async fn get_chat_id_by_contact_id(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<Option<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let chat_id = ChatId::lookup_by_contact(&ctx, ContactId::new(contact_id)).await?;
+        Ok(chat_id.map(|id| id.to_u32()))
+    }
+
     /// Returns all message IDs of the given types in a chat.
     /// Typically used to show a gallery.
     ///

deltachat-jsonrpc/src/api/types/chat.rs

@@ -18,6 +18,17 @@ use super::contact::ContactObject;
 pub struct FullChat {
     id: u32,
     name: String,
+
+    /// True if the chat is protected.
+    ///
+    /// UI should display a green checkmark
+    /// in the chat title,
+    /// in the chat profile title and
+    /// in the chatlist item
+    /// if chat protection is enabled.
+    /// UI should also display a green checkmark
+    /// in the contact profile
+    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -128,6 +139,17 @@ impl FullChat {
 pub struct BasicChat {
     id: u32,
     name: String,
+
+    /// True if the chat is protected.
+    ///
+    /// UI should display a green checkmark
+    /// in the chat title,
+    /// in the chat profile title and
+    /// in the chatlist item
+    /// if chat protection is enabled.
+    /// UI should also display a green checkmark
+    /// in the contact profile
+    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,

deltachat-jsonrpc/src/api/types/contact.rs

@@ -1,5 +1,4 @@
 use anyhow::Result;
-use deltachat::contact::VerifiedStatus;
 use deltachat::context::Context;
 use serde::Serialize;
 use typescript_type_def::TypeDef;
@@ -19,11 +18,30 @@ pub struct ContactObject {
     profile_image: Option<String>, // BLOBS
     name_and_addr: String,
     is_blocked: bool,
+
+    /// True if the contact can be added to verified groups.
+    ///
+    /// If this is true
+    /// UI should display green checkmark after the contact name
+    /// in contact list items,
+    /// in chat member list items
+    /// and in profiles if no chat with the contact exist.
     is_verified: bool,
-    /// the address that verified this contact
-    verifier_addr: Option<String>,
-    /// the id of the contact that verified this contact
+
+    /// True if the contact profile title should have a green checkmark.
+    ///
+    /// This indicates whether 1:1 chat has a green checkmark
+    /// or will have a green checkmark if created.
+    is_profile_verified: bool,
+
+    /// The ID of the contact that verified this contact.
+    ///
+    /// If this is present,
+    /// display a green checkmark and "Introduced by ..."
+    /// string followed by the verifier contact name and address
+    /// in the contact profile.
     verifier_id: Option<u32>,
+
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
@@ -38,19 +56,13 @@ impl ContactObject {
             Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
             None => None,
         };
-        let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+        let is_verified = contact.is_verified(context).await?;
+        let is_profile_verified = contact.is_profile_verified(context).await?;
 
-        let (verifier_addr, verifier_id) = if is_verified {
-            (
-                contact.get_verifier_addr(context).await?,
-                contact
-                    .get_verifier_id(context)
-                    .await?
-                    .map(|contact_id| contact_id.to_u32()),
-            )
-        } else {
-            (None, None)
-        };
+        let verifier_id = contact
+            .get_verifier_id(context)
+            .await?
+            .map(|contact_id| contact_id.to_u32());
 
         Ok(ContactObject {
             address: contact.get_addr().to_owned(),
@@ -64,7 +76,7 @@ impl ContactObject {
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
             is_verified,
-            verifier_addr,
+            is_profile_verified,
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),

deltachat-jsonrpc/src/webserver.rs

@@ -28,15 +28,13 @@ async fn main() -> Result<(), std::io::Error> {
         .layer(Extension(state.clone()));
 
     tokio::spawn(async move {
-        state.accounts.read().await.start_io().await;
+        state.accounts.write().await.start_io().await;
     });
 
     let addr = SocketAddr::from(([127, 0, 0, 1], port));
     log::info!("JSON-RPC WebSocket server listening on {}", addr);
-    axum::Server::bind(&addr)
-        .serve(app.into_make_service())
-        .await
-        .unwrap();
+    let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
+    axum::serve(listener, app).await.unwrap();
 
     Ok(())
 }

deltachat-jsonrpc/typescript/package.json

@@ -9,7 +9,6 @@
     "@types/chai": "^4.2.21",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^9.0.0",
-    "@types/node-fetch": "^2.5.7",
     "@types/ws": "^7.2.4",
     "c8": "^7.10.0",
     "chai": "^4.3.4",
@@ -17,7 +16,6 @@
     "esbuild": "^0.17.9",
     "http-server": "^14.1.1",
     "mocha": "^9.1.1",
-    "node-fetch": "^2.6.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.6.2",
     "typedoc": "^0.23.2",
@@ -55,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.127.2"
+  "version": "1.132.1"
 }

deltachat-jsonrpc/typescript/test/basic.ts

@@ -79,6 +79,9 @@ describe("basic tests", () => {
       accountId = await dc.rpc.addAccount();
     });
     it("should block and unblock contact", async function () {
+      // Cannot send sync messages to self as we do not have a self address.
+      await dc.rpc.setConfig(accountId, "sync_msgs", "0");
+
       const contactId = await dc.rpc.createContact(
         accountId,
         "example@delta.chat",

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -2,7 +2,6 @@ import { tmpdir } from "os";
 import { join, resolve } from "path";
 import { mkdtemp, rm } from "fs/promises";
 import { spawn, exec } from "child_process";
-import fetch from "node-fetch";
 import { Readable, Writable } from "node:stream";
 
 export type RpcServerHandle = {

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.127.2"
+version = "1.132.1"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -11,7 +11,7 @@ deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
 log = "0.4.20"
 pretty_env_logger = "0.5"
-rusqlite = "0.29"
+rusqlite = "0.30"
 rustyline = "12"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 

deltachat-repl/src/cmdline.rs

@@ -284,13 +284,8 @@ async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
         let addr = contact.get_addr();
-        let verified_state = contact.is_verified(context).await?;
-        let verified_str = if VerifiedStatus::Unverified != verified_state {
-            if verified_state == VerifiedStatus::BidirectVerified {
-                " √√"
-            } else {
-                " √"
-            }
+        let verified_str = if contact.is_verified(context).await? {
+            " √"
         } else {
             ""
         };

deltachat-repl/src/main.rs

@@ -401,7 +401,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    ctx: Context,
+    mut ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');

deltachat-rpc-client/README.md

@@ -25,7 +25,7 @@ $ pip install .
 ## Testing
 
 1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
-2. Run `PATH="../target/debug:$PATH" tox`.
+2. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
 
 Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
 

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -40,13 +40,13 @@ def main():
 
         while True:
             event = account.wait_for_event()
-            if event["type"] == EventType.INFO:
+            if event["kind"] == EventType.INFO:
                 logging.info("%s", event["msg"])
-            elif event["type"] == EventType.WARNING:
+            elif event["kind"] == EventType.WARNING:
                 logging.warning("%s", event["msg"])
-            elif event["type"] == EventType.ERROR:
+            elif event["kind"] == EventType.ERROR:
                 logging.error("%s", event["msg"])
-            elif event["type"] == EventType.INCOMING_MSG:
+            elif event["kind"] == EventType.INCOMING_MSG:
                 logging.info("Got an incoming message")
                 process_messages()
 

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -4,7 +4,7 @@ from warnings import warn
 
 from ._utils import AttrDict
 from .chat import Chat
-from .const import ChatlistFlag, ContactFlag, SpecialContactId
+from .const import ChatlistFlag, ContactFlag, EventType, SpecialContactId
 from .contact import Contact
 from .message import Message
 
@@ -111,6 +111,20 @@ class Account:
         contacts = self._rpc.get_blocked_contacts(self.id)
         return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
 
+    def get_chat_by_contact(self, contact: Union[int, Contact]) -> Optional[Chat]:
+        """Return 1:1 chat for a contact if it exists."""
+        if isinstance(contact, Contact):
+            assert contact.account == self
+            contact_id = contact.id
+        elif isinstance(contact, int):
+            contact_id = contact
+        else:
+            raise ValueError(f"{contact!r} is not a contact")
+        chat_id = self._rpc.get_chat_id_by_contact_id(self.id, contact_id)
+        if chat_id:
+            return Chat(self, chat_id)
+        return None
+
     def get_contacts(
         self,
         query: Optional[str] = None,
@@ -204,7 +218,7 @@ class Account:
         The function returns immediately and the handshake runs in background, sending
         and receiving several messages.
         Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
-        See https://countermitm.readthedocs.io/en/latest/new.html for protocol details.
+        See https://securejoin.readthedocs.io/en/latest/new.html for protocol details.
 
         :param qrdata: The text of the scanned QR code.
         """
@@ -250,6 +264,25 @@ class Account:
         next_msg_ids = self._rpc.wait_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
+    def wait_for_incoming_msg_event(self):
+        """Wait for incoming message event and return it."""
+        while True:
+            event = self.wait_for_event()
+            if event.kind == EventType.INCOMING_MSG:
+                return event
+
+    def wait_for_securejoin_inviter_success(self):
+        while True:
+            event = self.wait_for_event()
+            if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
+                break
+
+    def wait_for_securejoin_joiner_success(self):
+        while True:
+            event = self.wait_for_event()
+            if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+                break
+
     def get_fresh_messages_in_arrival_order(self) -> List[Message]:
         """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
         warn(
@@ -267,3 +300,13 @@ class Account:
     def import_backup(self, path, passphrase: str = "") -> None:
         """Import backup."""
         self._rpc.import_backup(self.id, str(path), passphrase)
+
+    def export_self_keys(self, path) -> None:
+        """Export keys."""
+        passphrase = ""  # Setting passphrase is currently not supported.
+        self._rpc.export_self_keys(self.id, str(path), passphrase)
+
+    def import_self_keys(self, path) -> None:
+        """Import keys."""
+        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
+        self._rpc.import_self_keys(self.id, str(path), passphrase)

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -195,7 +195,7 @@ class Client:
 
 
 class Bot(Client):
-    """Simple bot implementation that listent to events of a single account."""
+    """Simple bot implementation that listens to events of a single account."""
 
     def configure(self, email: str, password: str, **kwargs) -> None:
         kwargs.setdefault("bot", "1")

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -42,6 +42,10 @@ class Message:
             return AttrDict(reactions)
         return None
 
+    def get_sender_contact(self) -> Contact:
+        from_id = self.get_snapshot().from_id
+        return self.account.get_contact_by_id(from_id)
+
     def mark_seen(self) -> None:
         """Mark the message as seen."""
         self._rpc.markseen_msgs(self.account.id, [self.id])

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -21,7 +21,9 @@ class ACFactory:
         self.deltachat = deltachat
 
     def get_unconfigured_account(self) -> Account:
-        return self.deltachat.add_account()
+        account = self.deltachat.add_account()
+        account.set_config("verified_one_on_one_chats", "1")
+        return account
 
     def get_unconfigured_bot(self) -> Bot:
         return Bot(self.get_unconfigured_account())
@@ -59,6 +61,16 @@ class ACFactory:
     def get_online_accounts(self, num: int) -> List[Account]:
         return [self.get_online_account() for _ in range(num)]
 
+    def resetup_account(self, ac: Account) -> Account:
+        """Resetup account from scratch, losing the encryption key."""
+        ac.stop_io()
+        ac_clone = self.get_unconfigured_account()
+        for i in ["addr", "mail_pw"]:
+            ac_clone.set_config(i, ac.get_config(i))
+        ac.remove()
+        ac_clone.configure()
+        return ac_clone
+
     def send_message(
         self,
         to_account: Account,

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 import logging
 import os
@@ -5,7 +6,7 @@ import subprocess
 import sys
 from queue import Queue
 from threading import Event, Thread
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Iterator, Optional
 
 
 class JsonRpcError(Exception):
@@ -23,7 +24,7 @@ class Rpc:
 
         self._kwargs = kwargs
         self.process: subprocess.Popen
-        self.id: int
+        self.id_iterator: Iterator[int]
         self.event_queues: Dict[int, Queue]
         # Map from request ID to `threading.Event`.
         self.request_events: Dict[int, Event]
@@ -54,7 +55,7 @@ class Rpc:
                 preexec_fn=os.setpgrp,  # noqa: PLW1509
                 **self._kwargs,
             )
-        self.id = 0
+        self.id_iterator = itertools.count(start=1)
         self.event_queues = {}
         self.request_events = {}
         self.request_results = {}
@@ -131,7 +132,9 @@ class Rpc:
                 event = self.get_next_event()
                 account_id = event["contextId"]
                 queue = self.get_queue(account_id)
-                queue.put(event["event"])
+                event = event["event"]
+                logging.debug("account_id=%d got an event %s", account_id, event)
+                queue.put(event)
         except Exception:
             # Log an exception if the event loop dies.
             logging.exception("Exception in the event loop")
@@ -143,14 +146,12 @@ class Rpc:
 
     def __getattr__(self, attr: str):
         def method(*args) -> Any:
-            self.id += 1
-            request_id = self.id
-
+            request_id = next(self.id_iterator)
             request = {
                 "jsonrpc": "2.0",
                 "method": attr,
                 "params": args,
-                "id": self.id,
+                "id": request_id,
             }
             event = Event()
             self.request_events[request_id] = event

deltachat-rpc-client/tests/test_securejoin.py

@@ -0,0 +1,552 @@
+import logging
+
+from deltachat_rpc_client import Chat, SpecialContactId
+
+
+def test_qr_setup_contact(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+
+    alice.wait_for_securejoin_inviter_success()
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+
+    bob.wait_for_securejoin_joiner_success()
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+
+
+def test_qr_securejoin(acfactory):
+    alice, bob = acfactory.get_online_accounts(2)
+
+    logging.info("Alice creates a verified group")
+    alice_chat = alice.create_group("Verified group", protect=True)
+    assert alice_chat.get_basic_snapshot().is_protected
+
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+
+    # Check that at least some of the handshake messages are deleted.
+    for ac in [alice, bob]:
+        while True:
+            event = ac.wait_for_event()
+            if event["kind"] == "ImapMessageDeleted":
+                break
+
+    alice.wait_for_securejoin_inviter_success()
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+
+    bob.wait_for_securejoin_joiner_success()
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
+    assert snapshot.chat.get_basic_snapshot().is_protected
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+
+
+def test_qr_securejoin_contact_request(acfactory) -> None:
+    """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+    bob_chat_alice = snapshot.chat
+    assert bob_chat_alice.get_basic_snapshot().is_contact_request
+
+    alice_chat = alice.create_group("Verified group", protect=True)
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Chat stays being a contact request.
+    assert bob_chat_alice.get_basic_snapshot().is_contact_request
+
+
+def test_qr_readreceipt(acfactory) -> None:
+    alice, bob, charlie = acfactory.get_online_accounts(3)
+
+    logging.info("Bob and Charlie setup contact with Alice")
+    qr_code, _svg = alice.get_qr_code()
+
+    bob.secure_join(qr_code)
+    charlie.secure_join(qr_code)
+
+    for joiner in [bob, charlie]:
+        joiner.wait_for_securejoin_joiner_success()
+
+    logging.info("Alice creates a verified group")
+    group = alice.create_group("Group", protect=True)
+
+    bob_addr = bob.get_config("addr")
+    charlie_addr = charlie.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_contact_charlie = alice.create_contact(charlie_addr, "Charlie")
+
+    group.add_contact(alice_contact_bob)
+    group.add_contact(alice_contact_charlie)
+
+    # Promote a group.
+    group.send_message(text="Hello")
+
+    logging.info("Bob and Charlie receive a group")
+
+    bob_msg_id = bob.wait_for_incoming_msg_event().msg_id
+    bob_message = bob.get_message_by_id(bob_msg_id)
+    bob_snapshot = bob_message.get_snapshot()
+    assert bob_snapshot.text == "Hello"
+
+    # Charlie receives the same "Hello" message as Bob.
+    charlie.wait_for_incoming_msg_event()
+
+    logging.info("Bob sends a message to the group")
+
+    bob_out_message = bob_snapshot.chat.send_message(text="Hi from Bob!")
+
+    charlie_msg_id = charlie.wait_for_incoming_msg_event().msg_id
+    charlie_message = charlie.get_message_by_id(charlie_msg_id)
+    charlie_snapshot = charlie_message.get_snapshot()
+    assert charlie_snapshot.text == "Hi from Bob!"
+
+    bob_contact_charlie = bob.create_contact(charlie_addr, "Charlie")
+    assert not bob.get_chat_by_contact(bob_contact_charlie)
+
+    logging.info("Charlie reads Bob's message")
+    charlie_message.mark_seen()
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "MsgRead" and event["msg_id"] == bob_out_message.id:
+            break
+
+    # Receiving a read receipt from Charlie
+    # should not unblock hidden chat with Charlie for Bob.
+    assert not bob.get_chat_by_contact(bob_contact_charlie)
+
+
+def test_verified_group_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifying a member and sending a message in a group."""
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    ac3.wait_for_securejoin_inviter_success()
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hi!"
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Hi!"
+
+    # ac1 contact is verified for ac2 because ac3 gossiped ac1 key in the "Hi!" message.
+    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
+    assert ac1_contact.get_snapshot().is_verified
+
+    # ac2 can write messages to the group.
+    snapshot.chat.send_text("Works again!")
+
+    snapshot = ac3.get_message_by_id(ac3.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1_chat_messages = snapshot.chat.get_messages()
+    ac2_addr = ac2.get_config("addr")
+    assert ac1_chat_messages[-2].get_snapshot().text == f"Changed setup for {ac2_addr}"
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_verified_group_member_added_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifiying than removing and adding a member back."""
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    ac3.wait_for_securejoin_inviter_success()
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("Received message %s", snapshot.text)
+    assert snapshot.text == "Hi!"
+
+    ac1.wait_for_incoming_msg_event()  # Hi!
+
+    ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
+    ac3_chat.remove_contact(ac3_contact_ac2)
+    ac3_chat.add_contact(ac3_contact_ac2)
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert "removed" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "removed" in snapshot.text
+
+    event = ac2.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    chat_id = event.chat_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("ac2 got event message: %s", snapshot.text)
+    assert "added" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "added" in snapshot.text
+
+    chat = Chat(ac2, chat_id)
+    chat.send_text("Works again!")
+
+    msg_id = ac3.wait_for_incoming_msg_event().msg_id
+    message = ac3.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
+    assert ac1_contact_ac2_snapshot.is_verified
+    assert ac1_contact_ac2_snapshot.verifier_id == ac1.get_contact_by_addr(ac3.get_config("addr")).id
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
+    """Regression test for
+    issue <https://github.com/deltachat/deltachat-core-rust/issues/4894>.
+    """
+    ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
+
+    logging.info("ac3: verify with ac2")
+    qr_code, _svg = ac2.get_qr_code()
+    ac3.secure_join(qr_code)
+    ac2.wait_for_securejoin_inviter_success()
+
+    # in order for ac2 to have pending bobstate with a verified group
+    # we first create a fully joined verified group, and then start
+    # joining a second time but interrupt it, to create pending bob state
+
+    logging.info("ac1: create verified group that ac2 fully joins")
+    ch1 = ac1.create_group("Group", protect=True)
+    qr_code, _svg = ch1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    # ensure ac1 can write and ac2 receives messages in verified chat
+    ch1.send_text("ac1 says hello")
+    while 1:
+        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        if snapshot.text == "ac1 says hello":
+            assert snapshot.chat.get_basic_snapshot().is_protected
+            break
+
+    logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
+    qr_code, _svg = ch1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.remove()
+    logging.info("ac2 now has pending bobstate but ac1 is shutoff")
+
+    # we meanwhile expect ac3/ac2 verification started in the beginning to have completed
+    assert ac3.get_contact_by_addr(ac2.get_config("addr")).get_snapshot().is_verified
+    assert ac2.get_contact_by_addr(ac3.get_config("addr")).get_snapshot().is_verified
+
+    logging.info("ac3: create a verified group VG with ac2")
+    vg = ac3.create_group("ac3-created", protect=True)
+    vg.add_contact(ac3.get_contact_by_addr(ac2.get_config("addr")))
+
+    # ensure ac2 receives message in VG
+    vg.send_text("hello")
+    while 1:
+        msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        if msg.text == "hello":
+            assert msg.chat.get_basic_snapshot().is_protected
+            break
+
+    logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
+    qr_code, _svg = vg.get_qr_code()
+    ac4.secure_join(qr_code)
+    ac3.wait_for_securejoin_inviter_success()
+    while 1:
+        ev = ac2.wait_for_event()
+        if "added by unrelated SecureJoin" in str(ev):
+            return
+
+
+def test_qr_new_group_unblocked(acfactory):
+    """Regression test for a bug introduced in core v1.113.0.
+    ac2 scans a verified group QR code created by ac1.
+    This results in creation of a blocked 1:1 chat with ac1 on ac2,
+    but ac1 contact is not blocked on ac2.
+    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
+    ac2 should receive a message and create a contact request for the group.
+    Due to a bug previously ac2 created a blocked group.
+    """
+
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1_chat = ac1.create_group("Group for joining", protect=True)
+    qr_code, _svg = ac1_chat.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    ac1.wait_for_securejoin_inviter_success()
+
+    ac1_new_chat = ac1.create_group("Another group")
+    ac1_new_chat.add_contact(ac1.get_contact_by_addr(ac2.get_config("addr")))
+    # Receive "Member added" message.
+    ac2.wait_for_incoming_msg_event()
+
+    ac1_new_chat.send_text("Hello!")
+    ac2_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert ac2_msg.text == "Hello!"
+    assert ac2_msg.chat.get_basic_snapshot().is_contact_request
+
+
+def test_aeap_flow_verified(acfactory):
+    """Test that a new address is added to a contact when it changes its address."""
+    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
+
+    logging.info("ac1: create verified-group QR, ac2 scans and joins")
+    chat = ac1.create_group("hello", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+    qr_code, _svg = chat.get_qr_code()
+    logging.info("ac2: start QR-code based join-group protocol")
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    logging.info("sending first message")
+    msg_out = chat.send_text("old address").get_snapshot()
+
+    logging.info("receiving first message")
+    ac2.wait_for_incoming_msg_event()  # member added message
+    msg_in_1 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert msg_in_1.text == msg_out.text
+
+    logging.info("changing email account")
+    ac1.set_config("addr", ac1new.get_config("addr"))
+    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
+    ac1.stop_io()
+    ac1.configure()
+    ac1.start_io()
+
+    logging.info("sending second message")
+    msg_out = chat.send_text("changed address").get_snapshot()
+
+    logging.info("receiving second message")
+    msg_in_2 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    msg_in_2_snapshot = msg_in_2.get_snapshot()
+    assert msg_in_2_snapshot.text == msg_out.text
+    assert msg_in_2_snapshot.chat.id == msg_in_1.chat.id
+    assert msg_in_2.get_sender_contact().get_snapshot().address == ac1new.get_config("addr")
+    assert len(msg_in_2_snapshot.chat.get_contacts()) == 2
+    assert ac1new.get_config("addr") in [
+        contact.get_snapshot().address for contact in msg_in_2_snapshot.chat.get_contacts()
+    ]
+
+
+def test_gossip_verification(acfactory) -> None:
+    alice, bob, carol = acfactory.get_online_accounts(3)
+
+    # Bob verifies Alice.
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    # Bob verifies Carol.
+    qr_code, _svg = carol.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    bob_contact_alice = bob.create_contact(alice.get_config("addr"), "Alice")
+    bob_contact_carol = bob.create_contact(carol.get_config("addr"), "Carol")
+    carol_contact_alice = carol.create_contact(alice.get_config("addr"), "Alice")
+
+    logging.info("Bob creates an Autocrypt group")
+    bob_group_chat = bob.create_group("Autocrypt Group")
+    assert not bob_group_chat.get_basic_snapshot().is_protected
+    bob_group_chat.add_contact(bob_contact_alice)
+    bob_group_chat.add_contact(bob_contact_carol)
+    bob_group_chat.send_message(text="Hello Autocrypt group")
+
+    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Autocrypt group"
+    assert snapshot.show_padlock
+
+    # Autocrypt group does not propagate verification.
+    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
+    assert not carol_contact_alice_snapshot.is_verified
+
+    logging.info("Bob creates a Securejoin group")
+    bob_group_chat = bob.create_group("Securejoin Group", protect=True)
+    assert bob_group_chat.get_basic_snapshot().is_protected
+    bob_group_chat.add_contact(bob_contact_alice)
+    bob_group_chat.add_contact(bob_contact_carol)
+    bob_group_chat.send_message(text="Hello Securejoin group")
+
+    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Securejoin group"
+    assert snapshot.show_padlock
+
+    # Securejoin propagates verification.
+    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
+    assert carol_contact_alice_snapshot.is_verified
+
+
+def test_securejoin_after_contact_resetup(acfactory) -> None:
+    """
+    Regression test for a bug that prevented joining verified group with a QR code
+    if the group is already created and contains
+    a contact with inconsistent (Autocrypt and verified keys exist but don't match) key state.
+    """
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    # ac3 creates protected group with ac1.
+    ac3_chat = ac3.create_group("Verified group", protect=True)
+
+    # ac1 joins ac3 group.
+    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac1.secure_join(ac3_qr_code)
+    ac1.wait_for_securejoin_joiner_success()
+
+    # ac1 waits for member added message and creates a QR code.
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+
+    # ac2 verifies ac1
+    qr_code, _svg = ac1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    # ac1 is verified for ac2.
+    ac2_contact_ac1 = ac2.create_contact(ac1.get_config("addr"), "")
+    assert ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 resetups the account.
+    ac1 = acfactory.resetup_account(ac1)
+
+    # ac1 sends a message to ac2.
+    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
+    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
+    ac1_chat_ac2.send_text("Hello!")
+
+    # ac2 receives a message.
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+
+    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
+    assert not ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 goes offline.
+    ac1.remove()
+
+    # Scanning a QR code results in creating an unprotected group with an inviter.
+    # In this case inviter is ac1 which has an inconsistent key state.
+    # Normally inviter becomes verified as a result of Securejoin protocol
+    # and then the group chat becomes verified when "Member added" is received,
+    # but in this case ac1 is offline and this Securejoin process will never finish.
+    logging.info("ac2 scans ac1 QR code, this is not expected to finish")
+    ac2.secure_join(ac1_qr_code)
+
+    logging.info("ac2 scans ac3 QR code")
+    ac2.secure_join(ac3_qr_code)
+
+    logging.info("ac2 waits for joiner success")
+    ac2.wait_for_securejoin_joiner_success()
+
+    # Wait for member added.
+    logging.info("ac2 waits for member added message")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.is_info
+    ac2_chat = snapshot.chat
+    assert ac2_chat.get_basic_snapshot().is_protected
+    assert len(ac2_chat.get_contacts()) == 3
+
+    # ac1 is still "not verified" for ac2 due to inconsistent state.
+    assert not ac2_contact_ac1.get_snapshot().is_verified

deltachat-rpc-client/tests/test_something.py

@@ -331,7 +331,7 @@ def test_wait_next_messages(acfactory) -> None:
         next_messages_task = executor.submit(bot.wait_next_messages)
 
         bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
         alice_chat_bot = alice_contact_bot.create_chat()
         alice_chat_bot.send_text("Hello!")
 
@@ -341,7 +341,7 @@ def test_wait_next_messages(acfactory) -> None:
         assert snapshot.text == "Hello!"
 
 
-def test_import_export(acfactory, tmp_path) -> None:
+def test_import_export_backup(acfactory, tmp_path) -> None:
     alice = acfactory.new_configured_account()
     alice.export_backup(tmp_path)
 
@@ -352,6 +352,31 @@ def test_import_export(acfactory, tmp_path) -> None:
     assert alice2.manager.get_system_info()
 
 
+def test_import_export_keys(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello Bob!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Bob!"
+
+    # Alice resetups account, but keeps the key.
+    alice_keys_path = tmp_path / "alice_keys"
+    alice_keys_path.mkdir()
+    alice.export_self_keys(alice_keys_path)
+    alice = acfactory.resetup_account(alice)
+    alice.import_self_keys(alice_keys_path)
+
+    snapshot.chat.accept()
+    snapshot.chat.send_text("Hello Alice!")
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Alice!"
+    assert snapshot.show_padlock
+
+
 def test_openrpc_command_line() -> None:
     """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
     out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
@@ -377,15 +402,3 @@ def test_provider_info(rpc) -> None:
     rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
-
-
-def test_qr_setup_contact(acfactory) -> None:
-    alice, bob = acfactory.get_online_accounts(2)
-
-    qr_code, _svg = alice.get_qr_code()
-    bob.secure_join(qr_code)
-
-    while True:
-        event = alice.wait_for_event()
-        if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            return

deltachat-rpc-client/tests/test_webxdc.py

@@ -43,3 +43,15 @@ def test_webxdc(acfactory) -> None:
     assert status_updates == [
         {"payload": "Second update", "serial": 2, "max_serial": 2},
     ]
+
+
+def test_webxdc_insert_lots_of_updates(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    message = alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+
+    for i in range(2000):
+        message.send_webxdc_status_update({"payload": str(i)}, "description")

deltachat-rpc-client/tox.ini

@@ -30,4 +30,4 @@ commands =
 [pytest]
 timeout = 300
 log_cli = true
-log_level = info
+log_level = debug

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.127.2"
+version = "1.132.1"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deny.toml

@@ -3,6 +3,13 @@ unmaintained = "allow"
 ignore = [
     "RUSTSEC-2020-0071",
     "RUSTSEC-2022-0093",
+
+    # Timing attack on RSA.
+    # Delta Chat does not use RSA for new keys
+    # and this requires precise measurement of the decryption time by the attacker.
+    # There is no fix at the time of writing this (2023-11-28).
+    # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
+    "RUSTSEC-2023-0071",
 ]
 
 [bans]
@@ -26,19 +33,23 @@ skip = [
      { name = "ed25519-dalek", version = "1.0.1" },
      { name = "ed25519", version = "1.5.3" },
      { name = "event-listener", version = "2.5.3" },
+     { name = "fd-lock", version = "3.0.13" },
      { name = "getrandom", version = "<0.2" },
-     { name = "hashbrown", version = "<0.14.0" },
-     { name = "indexmap", version = "<2.0.0" },
+     { name = "h2", version = "0.3.22" },
+     { name = "http-body", version = "0.4.5" },
+     { name = "http", version = "0.2.11" },
+     { name = "hyper", version = "0.14.27" },
+     { name = "idna", version = "0.4.0" },
      { name = "pem-rfc7468", version = "0.6.0" },
      { name = "pkcs8", version = "0.9.0" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
-     { name = "redox_syscall", version = "0.2.16" },
+     { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
-     { name = "ring", version = "0.16.20" },
      { name = "regex-syntax", version = "0.6.29" },
+     { name = "ring", version = "0.16.20" },
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },
@@ -49,12 +60,16 @@ skip = [
      { name = "time", version = "<0.3" },
      { name = "untrusted", version = "0.7.1" },
      { name = "wasi", version = "<0.11" },
-     { name = "windows_aarch64_msvc", version = "<0.48" },
-     { name = "windows_i686_gnu", version = "<0.48" },
-     { name = "windows_i686_msvc", version = "<0.48" },
+     { name = "windows_aarch64_gnullvm", version = "<0.52" },
+     { name = "windows_aarch64_msvc", version = "<0.52" },
+     { name = "windows_i686_gnu", version = "<0.52" },
+     { name = "windows_i686_msvc", version = "<0.52" },
+     { name = "windows-sys", version = "<0.52" },
+     { name = "windows-targets", version = "<0.52" },
      { name = "windows", version = "0.32.0" },
-     { name = "windows_x86_64_gnu", version = "<0.48" },
-     { name = "windows_x86_64_msvc", version = "<0.48" },
+     { name = "windows_x86_64_gnullvm", version = "<0.52" },
+     { name = "windows_x86_64_gnu", version = "<0.52" },
+     { name = "windows_x86_64_msvc", version = "<0.52" },
 ]
 
 
@@ -86,4 +101,6 @@ license-files = [
 github = [
        "async-email",
        "deltachat",
+       "djc",
+       "n0-computer", # iroh
 ]

fuzz/Cargo.lock

@@ -47,7 +47,7 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
 dependencies = [
- "getrandom 0.2.8",
+ "getrandom 0.2.11",
  "once_cell",
  "version_check",
 ]
@@ -158,10 +158,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833"
 dependencies = [
  "concurrent-queue",
- "event-listener",
+ "event-listener 2.5.3",
  "futures-core",
 ]
 
+[[package]]
+name = "async-channel"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c"
+dependencies = [
+ "concurrent-queue",
+ "event-listener 4.0.0",
+ "event-listener-strategy",
+ "futures-core",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "async-compression"
 version = "0.3.15"
@@ -181,7 +194,7 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b538b767cbf9c162a6c5795d4b932bd2c20ba10b5a91a94d2b2b6886c1dce6a8"
 dependencies = [
- "async-channel",
+ "async-channel 1.8.0",
  "base64 0.21.0",
  "bytes",
  "chrono",
@@ -484,9 +497,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "3.3.4"
+version = "3.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68"
+checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -495,9 +508,9 @@ dependencies = [
 
 [[package]]
 name = "brotli-decompressor"
-version = "2.3.4"
+version = "2.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744"
+checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -568,9 +581,12 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.78"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "cfb-mode"
@@ -646,9 +662,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
 
 [[package]]
 name = "concurrent-queue"
-version = "2.0.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b"
+checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -906,10 +922,10 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.123.0"
+version = "1.131.9"
 dependencies = [
  "anyhow",
- "async-channel",
+ "async-channel 2.1.1",
  "async-imap",
  "async-native-tls",
  "async-smtp",
@@ -928,6 +944,7 @@ dependencies = [
  "futures",
  "futures-lite",
  "hex",
+ "hickory-resolver",
  "humansize",
  "image",
  "iroh",
@@ -943,6 +960,7 @@ dependencies = [
  "parking_lot",
  "percent-encoding",
  "pgp",
+ "pin-project",
  "qrcodegen",
  "quick-xml",
  "rand 0.8.5",
@@ -968,7 +986,6 @@ dependencies = [
  "tokio-tar",
  "tokio-util",
  "toml",
- "trust-dns-resolver",
  "url",
  "uuid 1.2.2",
 ]
@@ -1434,6 +1451,12 @@ dependencies = [
  "syn 1.0.107",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -1482,10 +1505,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
 
 [[package]]
-name = "fallible-iterator"
-version = "0.2.0"
+name = "event-listener"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae"
+dependencies = [
+ "concurrent-queue",
+ "parking",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "event-listener-strategy"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3"
+dependencies = [
+ "event-listener 4.0.0",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "fallible-iterator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
 
 [[package]]
 name = "fallible-streaming-iterator"
@@ -1495,11 +1539,12 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
 
 [[package]]
 name = "fast-socks5"
-version = "0.8.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2687b5a6108f18ba8621e0e618a3be1dcc2768632dad24b7cea1f87975375a9"
+checksum = "d449e348301d5fb9b0e5781510d8235ffe3bbac3286bd305462736a9e7043039"
 dependencies = [
  "anyhow",
+ "async-trait",
  "log",
  "thiserror",
  "tokio",
@@ -1525,14 +1570,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "fd-lock"
-version = "3.0.13"
+name = "fastrand"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
+checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+
+[[package]]
+name = "fd-lock"
+version = "4.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b93f7a0db71c99f68398f80653ed05afb0b00e062e1a20c7ff849c4edfabbbcc"
 dependencies = [
  "cfg-if",
  "rustix 0.38.14",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -1593,7 +1644,7 @@ dependencies = [
  "futures-sink",
  "nanorand",
  "pin-project",
- "spin 0.9.6",
+ "spin 0.9.8",
 ]
 
 [[package]]
@@ -1680,17 +1731,16 @@ checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
 
 [[package]]
 name = "futures-lite"
-version = "1.13.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce"
+checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb"
 dependencies = [
- "fastrand",
+ "fastrand 2.0.1",
  "futures-core",
  "futures-io",
  "memchr",
  "parking",
  "pin-project-lite",
- "waker-fn",
 ]
 
 [[package]]
@@ -1758,9 +1808,9 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.8"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -1819,7 +1869,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 1.9.2",
  "slab",
  "tokio",
  "tokio-util",
@@ -1835,13 +1885,19 @@ dependencies = [
  "ahash",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+
 [[package]]
 name = "hashlink"
 version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
 dependencies = [
- "hashbrown",
+ "hashbrown 0.12.3",
 ]
 
 [[package]]
@@ -1862,6 +1918,51 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "hickory-proto"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "091a6fbccf4860009355e3efc52ff4acf37a63489aad7435372d44ceeb6fbbcf"
+dependencies = [
+ "async-trait",
+ "cfg-if",
+ "data-encoding",
+ "enum-as-inner",
+ "futures-channel",
+ "futures-io",
+ "futures-util",
+ "idna",
+ "ipnet",
+ "once_cell",
+ "rand 0.8.5",
+ "thiserror",
+ "tinyvec",
+ "tokio",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "hickory-resolver"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35b8f021164e6a984c9030023544c57789c51760065cd510572fedcfb04164e8"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "hickory-proto",
+ "ipconfig",
+ "lru-cache",
+ "once_cell",
+ "parking_lot",
+ "rand 0.8.5",
+ "resolv-conf",
+ "smallvec",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "hkdf"
 version = "0.12.3"
@@ -2052,7 +2153,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.3",
 ]
 
 [[package]]
@@ -2103,9 +2214,8 @@ checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e"
 
 [[package]]
 name = "iroh"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4fb9858c8cd3dd924a5da5bc511363845a9bcfdfac066bb2ef8454eb6111546"
+version = "0.4.2"
+source = "git+https://github.com/n0-computer/iroh?branch=maint-0.4#9881b7886235035a1124e4371f7a4cd59379e51b"
 dependencies = [
  "abao",
  "anyhow",
@@ -2126,8 +2236,9 @@ dependencies = [
  "quinn",
  "rand 0.7.3",
  "rcgen",
- "ring",
+ "ring 0.16.20",
  "rustls",
+ "rustls-webpki",
  "serde",
  "serde-error",
  "ssh-key",
@@ -2140,7 +2251,6 @@ dependencies = [
  "tracing-futures",
  "tracing-subscriber",
  "walkdir",
- "webpki",
  "x509-parser",
  "zeroize",
 ]
@@ -2219,9 +2329,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.148"
+version = "0.2.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
+checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
 [[package]]
 name = "libm"
@@ -2237,9 +2347,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.26.0"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
+checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
 dependencies = [
  "cc",
  "openssl-sys",
@@ -2404,7 +2514,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
 dependencies = [
- "getrandom 0.2.8",
+ "getrandom 0.2.11",
 ]
 
 [[package]]
@@ -2484,15 +2594,6 @@ dependencies = [
  "minimal-lexical",
 ]
 
-[[package]]
-name = "nom8"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -2638,11 +2739,11 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.55"
+version = "0.10.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
+checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.4.0",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -2670,18 +2771,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "111.25.0+1.1.1t"
+version = "300.1.6+3.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6"
+checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.90"
+version = "0.9.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
+checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f"
 dependencies = [
  "cc",
  "libc",
@@ -2754,9 +2855,9 @@ dependencies = [
 
 [[package]]
 name = "parking"
-version = "2.0.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
+checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
 
 [[package]]
 name = "parking_lot"
@@ -3062,11 +3163,12 @@ checksum = "4339fc7a1021c9c1621d87f5e3505f2805c8c105420ba2f2a4df86814590c142"
 
 [[package]]
 name = "quic-rpc"
-version = "0.5.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d453504fc3e456160ae3b9ebe4d83c1f6477af167aa9b67e2d7bf11a096f179d"
+checksum = "6d60c2fc2390baad4b9d41ae9957ae88c3095496f88e252ef50722df8b5b78d7"
 dependencies = [
  "bincode",
+ "educe",
  "flume",
  "futures",
  "pin-project",
@@ -3086,18 +3188,18 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
 [[package]]
 name = "quick-xml"
-version = "0.30.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "quinn"
-version = "0.9.3"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "445cbfe2382fa023c4f2f3c7e1c95c03dcc1df2bf23cebcb2b13e1402c4394d1"
+checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75"
 dependencies = [
  "bytes",
  "pin-project-lite",
@@ -3108,18 +3210,17 @@ dependencies = [
  "thiserror",
  "tokio",
  "tracing",
- "webpki",
 ]
 
 [[package]]
 name = "quinn-proto"
-version = "0.9.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c956be1b23f4261676aed05a0046e204e8a6836e50203902683a718af0797989"
+checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a"
 dependencies = [
  "bytes",
  "rand 0.8.5",
- "ring",
+ "ring 0.16.20",
  "rustc-hash",
  "rustls",
  "rustls-native-certs",
@@ -3127,20 +3228,19 @@ dependencies = [
  "thiserror",
  "tinyvec",
  "tracing",
- "webpki",
 ]
 
 [[package]]
 name = "quinn-udp"
-version = "0.3.2"
+version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "641538578b21f5e5c8ea733b736895576d0fe329bb883b937db6f4d163dbaaf4"
+checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7"
 dependencies = [
+ "bytes",
  "libc",
- "quinn-proto",
- "socket2 0.4.7",
+ "socket2 0.5.4",
  "tracing",
- "windows-sys 0.42.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -3217,7 +3317,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.8",
+ "getrandom 0.2.11",
 ]
 
 [[package]]
@@ -3240,7 +3340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b"
 dependencies = [
  "pem",
- "ring",
+ "ring 0.16.20",
  "time 0.3.20",
  "yasna",
 ]
@@ -3260,7 +3360,7 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
 dependencies = [
- "getrandom 0.2.8",
+ "getrandom 0.2.11",
  "redox_syscall",
  "thiserror",
 ]
@@ -3387,11 +3487,25 @@ dependencies = [
  "libc",
  "once_cell",
  "spin 0.5.2",
- "untrusted",
+ "untrusted 0.7.1",
  "web-sys",
  "winapi",
 ]
 
+[[package]]
+name = "ring"
+version = "0.17.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866"
+dependencies = [
+ "cc",
+ "getrandom 0.2.11",
+ "libc",
+ "spin 0.9.8",
+ "untrusted 0.9.0",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "ripemd"
 version = "0.1.3"
@@ -3445,9 +3559,9 @@ dependencies = [
 
 [[package]]
 name = "rusqlite"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
+checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
 dependencies = [
  "bitflags 2.4.0",
  "fallible-iterator",
@@ -3522,13 +3636,13 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.20.8"
+version = "0.21.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f"
+checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
 dependencies = [
- "ring",
+ "ring 0.17.6",
+ "rustls-webpki",
  "sct",
- "webpki",
 ]
 
 [[package]]
@@ -3552,6 +3666,16 @@ dependencies = [
  "base64 0.21.0",
 ]
 
+[[package]]
+name = "rustls-webpki"
+version = "0.101.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
+dependencies = [
+ "ring 0.17.6",
+ "untrusted 0.9.0",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.11"
@@ -3617,8 +3741,8 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
 dependencies = [
- "ring",
- "untrusted",
+ "ring 0.16.20",
+ "untrusted 0.7.1",
 ]
 
 [[package]]
@@ -3674,9 +3798,9 @@ dependencies = [
 
 [[package]]
 name = "self_cell"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c309e515543e67811222dbc9e3dd7e1056279b782e1dacffe4242b718734fb6"
+checksum = "e388332cd64eb80cd595a00941baf513caffae8dce9cfd0467fc9c66397dade6"
 
 [[package]]
 name = "semver"
@@ -3735,9 +3859,9 @@ dependencies = [
 
 [[package]]
 name = "serde_spanned"
-version = "0.6.1"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4"
+checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80"
 dependencies = [
  "serde",
 ]
@@ -3897,9 +4021,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 
 [[package]]
 name = "spin"
-version = "0.9.6"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5d6e0250b93c8427a177b849d144a96d5acc57006149479403d7861ab721e34"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
 dependencies = [
  "lock_api",
 ]
@@ -3959,7 +4083,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af91f480ee899ab2d9f8435bfdfc14d08a5754bd9d3fef1f1a1c23336aad6c8b"
 dependencies = [
- "async-channel",
+ "async-channel 1.8.0",
  "cfg-if",
  "futures-core",
  "pin-project-lite",
@@ -4064,7 +4188,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95"
 dependencies = [
  "cfg-if",
- "fastrand",
+ "fastrand 1.8.0",
  "redox_syscall",
  "rustix 0.36.7",
  "windows-sys 0.42.0",
@@ -4280,9 +4404,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.7.2"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6"
+checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -4292,24 +4416,24 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.1"
+version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622"
+checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.19.3"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e6a7712b49e1775fb9a7b998de6635b299237f48b404dde71704f2e0e7f37e5"
+checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03"
 dependencies = [
- "indexmap",
- "nom8",
+ "indexmap 2.1.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
+ "winnow",
 ]
 
 [[package]]
@@ -4325,6 +4449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
 dependencies = [
  "cfg-if",
+ "log",
  "pin-project-lite",
  "tracing-attributes",
  "tracing-core",
@@ -4390,52 +4515,6 @@ dependencies = [
  "tracing-log",
 ]
 
-[[package]]
-name = "trust-dns-proto"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dc775440033cb114085f6f2437682b194fa7546466024b1037e82a48a052a69"
-dependencies = [
- "async-trait",
- "cfg-if",
- "data-encoding",
- "enum-as-inner",
- "futures-channel",
- "futures-io",
- "futures-util",
- "idna",
- "ipnet",
- "once_cell",
- "rand 0.8.5",
- "smallvec",
- "thiserror",
- "tinyvec",
- "tokio",
- "tracing",
- "url",
-]
-
-[[package]]
-name = "trust-dns-resolver"
-version = "0.23.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dff7aed33ef3e8bf2c9966fccdfed93f93d46f432282ea875cd66faabc6ef2f"
-dependencies = [
- "cfg-if",
- "futures-util",
- "ipconfig",
- "lru-cache",
- "once_cell",
- "parking_lot",
- "rand 0.8.5",
- "resolv-conf",
- "smallvec",
- "thiserror",
- "tokio",
- "tracing",
- "trust-dns-proto",
-]
-
 [[package]]
 name = "try-lock"
 version = "0.2.3"
@@ -4475,7 +4554,7 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137"
 dependencies = [
- "hashbrown",
+ "hashbrown 0.12.3",
  "regex",
 ]
 
@@ -4506,6 +4585,12 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
 version = "2.4.1"
@@ -4523,7 +4608,7 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
 dependencies = [
- "getrandom 0.2.8",
+ "getrandom 0.2.11",
 ]
 
 [[package]]
@@ -4532,7 +4617,7 @@ version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
 dependencies = [
- "getrandom 0.2.8",
+ "getrandom 0.2.11",
  "serde",
 ]
 
@@ -4554,12 +4639,6 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
-[[package]]
-name = "waker-fn"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
-
 [[package]]
 name = "walkdir"
 version = "2.3.3"
@@ -4674,16 +4753,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "webpki"
-version = "0.22.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f"
-dependencies = [
- "ring",
- "untrusted",
-]
-
 [[package]]
 name = "weezl"
 version = "0.1.7"
@@ -4774,7 +4843,16 @@ version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
- "windows-targets",
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.0",
 ]
 
 [[package]]
@@ -4792,6 +4870,21 @@ dependencies = [
  "windows_x86_64_msvc 0.48.5",
 ]
 
+[[package]]
+name = "windows-targets"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.0",
+ "windows_aarch64_msvc 0.52.0",
+ "windows_i686_gnu 0.52.0",
+ "windows_i686_msvc 0.52.0",
+ "windows_x86_64_gnu 0.52.0",
+ "windows_x86_64_gnullvm 0.52.0",
+ "windows_x86_64_msvc 0.52.0",
+]
+
 [[package]]
 name = "windows_aarch64_gnullvm"
 version = "0.42.0"
@@ -4804,6 +4897,12 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.32.0"
@@ -4828,6 +4927,12 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.32.0"
@@ -4852,6 +4957,12 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.32.0"
@@ -4876,6 +4987,12 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.32.0"
@@ -4900,6 +5017,12 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
+
 [[package]]
 name = "windows_x86_64_gnullvm"
 version = "0.42.0"
@@ -4912,6 +5035,12 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.32.0"
@@ -4936,6 +5065,21 @@ version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
+
+[[package]]
+name = "winnow"
+version = "0.5.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "winreg"
 version = "0.10.1"

node/README.md

@@ -40,7 +40,7 @@ npm install deltachat-node
 
 ## Dependencies
 
-- Nodejs >= `v16.0.0`
+- Nodejs >= `v18.0.0`
 - rustup (optional if you can't use the prebuilds)
 
 > On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
@@ -113,8 +113,8 @@ Then, in the `deltachat-desktop` repository, run:
 deltachat doesn't support universal (fat) binaries (that contain builds for both cpu architectures) yet, until it does you can use the following workaround to get x86_64 builds:
 
 ```
-$ fnm install 17 --arch x64
-$ fnm use 17
+$ fnm install 19 --arch x64
+$ fnm use 19
 $ node -p process.arch
 # result should be x64
 $ rustup target add x86_64-apple-darwin
@@ -127,8 +127,8 @@ $ npm run test
 If your node and electron are already build for arm64 you can also try building for arm:
 
 ```
-$ fnm install 16 --arch arm64
-$ fnm use 16
+$ fnm install 18 --arch arm64
+$ fnm use 18
 $ node -p process.arch
 # result should be arm64
 $ npm_config_arch=arm64 npm run build

node/constants.js

@@ -28,6 +28,7 @@ module.exports = {
   DC_DOWNLOAD_DONE: 0,
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
@@ -239,6 +240,7 @@ module.exports = {
   DC_STR_MSGGRPNAME: 15,
   DC_STR_MSGLOCATIONDISABLED: 65,
   DC_STR_MSGLOCATIONENABLED: 64,
+  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE: 172,
   DC_STR_NOMESSAGES: 1,
   DC_STR_NOT_CONNECTED: 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER: 113,

node/lib/constants.ts

@@ -28,6 +28,7 @@ export enum C {
   DC_DOWNLOAD_DONE = 0,
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
@@ -239,6 +240,7 @@ export enum C {
   DC_STR_MSGGRPNAME = 15,
   DC_STR_MSGLOCATIONDISABLED = 65,
   DC_STR_MSGLOCATIONENABLED = 64,
+  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE = 172,
   DC_STR_NOMESSAGES = 1,
   DC_STR_NOT_CONNECTED = 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER = 113,

node/test/test.js

@@ -8,7 +8,6 @@ import { EventId2EventName, C } from '../dist/constants'
 import { join } from 'path'
 import { statSync } from 'fs'
 import { Context } from '../dist/context'
-import fetch from 'node-fetch'
 chai.use(chaiAsPromised)
 chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 
@@ -668,9 +667,9 @@ describe('Offline Tests with unconfigured account', function () {
 
     const lot = chatList.getSummary(0)
     strictEqual(lot.getId(), 0, 'lot has no id')
-    strictEqual(lot.getState(), C.DC_STATE_UNDEFINED, 'correct state')
+    strictEqual(lot.getState(), C.DC_STATE_IN_NOTICED, 'correct state')
 
-    const text = 'No messages.'
+    const text = 'Others will only see this group after you sent a first message.'
     context.createGroupChat('groupchat1111')
     chatList = context.getChatList(0, 'groupchat1111', null)
     strictEqual(

node/tsconfig.json

@@ -13,10 +13,10 @@
     },
     "exclude": ["node_modules", "deltachat-core-rust", "dist", "scripts"],
     "typedocOptions": {
-        "mode": "file",
         "out": "docs",
-        "excludeNotExported": true,
+        "excludePrivate": true,
         "defaultCategory": "index",
-        "includeVersion": true
+        "includeVersion": true,
+        "entryPoints": ["lib/index.ts"]
     }
 }

node/windows.md

@@ -6,7 +6,7 @@ E.g via <https://git-scm.com/download/win>
 
 ## install node
 
-Download and install `v16` from <https://nodejs.org/en/>
+Download and install `v18` from <https://nodejs.org/en/>
 
 ## install rust
 

package.json

@@ -2,28 +2,25 @@
   "dependencies": {
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
-    "node-gyp-build": "^4.1.0"
+    "node-gyp-build": "^4.6.1"
   },
   "description": "node.js bindings for deltachat-core",
   "devDependencies": {
     "@types/debug": "^4.1.7",
-    "@types/node": "^16.11.26",
+    "@types/node": "^20.8.10",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "esm": "^3.2.25",
-    "hallmark": "^2.0.0",
     "mocha": "^8.2.1",
-    "node-fetch": "^2.6.7",
-    "node-gyp": "^9.0.0",
-    "opn-cli": "^5.0.0",
-    "prebuildify": "^3.0.0",
-    "prebuildify-ci": "^1.0.4",
-    "prettier": "^2.0.5",
-    "typedoc": "^0.17.0",
-    "typescript": "^3.9.10"
+    "node-gyp": "^10.0.0",
+    "prebuildify": "^5.0.1",
+    "prebuildify-ci": "^1.0.5",
+    "prettier": "^3.0.3",
+    "typedoc": "^0.25.3",
+    "typescript": "^5.2.2"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "files": [
     "node/scripts/*",
@@ -49,16 +46,15 @@
     "build:core:rust": "node node/scripts/rebuild-core.js",
     "clean": "rm -rf node/dist node/build node/prebuilds node/node_modules ./target",
     "download-prebuilds": "prebuildify-ci download",
-    "hallmark": "hallmark --fix",
     "install": "node node/scripts/install.js",
     "install:prebuilds": "cd node && node-gyp-build \"npm run build:core\" \"npm run build:bindings:c:postinstall\"",
     "lint": "prettier --check \"node/lib/**/*.{ts,tsx}\"",
     "lint-fix": "prettier --write \"node/lib/**/*.{ts,tsx}\" \"node/test/**/*.js\"",
-    "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
+    "prebuildify": "cd node && prebuildify -t 18.0.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.127.2"
+  "version": "1.132.1"
 }

python/README.rst

@@ -1,6 +1,6 @@
-=========================
-DeltaChat Python bindings
-=========================
+============================
+CFFI Python Bindings
+============================
 
 This package provides `Python bindings`_ to the `deltachat-core library`_
 which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
@@ -8,157 +8,3 @@ a low-level Chat/Contact/Message API to user interfaces and bots.
 
 .. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
 .. _`Python bindings`: https://py.delta.chat/
-
-Installing pre-built packages (Linux-only)
-==========================================
-
-If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself`__.
-
-__ sourceinstall_
-
-We recommend to first create a fresh Python virtual environment
-and activate it in your shell::
-
-    python -m venv env
-    source env/bin/activate
-
-Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``env`` directory and leaves
-your system installation alone.
-
-For Linux we build wheels for all releases and push them to a python package
-index. To install the latest release::
-
-    pip install deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
-
-Running tests
-=============
-
-Recommended way to run tests is using `scripts/run-python-test.sh`
-script provided in the core repository.
-
-This script compiles the library in debug mode and runs the tests using `tox`_.
-By default it will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _`tox`: https://tox.wiki
-.. _livetests:
-
-Running "live" tests with temporary accounts
---------------------------------------------
-
-If you want to run live functional tests
-you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
-that creates e-mail accounts like this::
-
-    export CHATMAIL_DOMAIN=nine.testrun.org
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the server.
-These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
-After setting the variable, either rerun `scripts/run-python-test.sh`
-or run offline and online tests with `tox` directly::
-
-    tox -e py
-
-Each test run creates new accounts.
-
-Developing the bindings
------------------------
-
-If you want to develop or debug the bindings,
-you can create a testing development environment using `tox`::
-
-    export DCC_RS_DEV="$PWD"
-    export DCC_RS_TARGET=debug
-    tox -c python --devenv env -e py
-    . env/bin/activate
-
-Inside this environment the bindings are installed
-in editable mode (as if installed with `python -m pip install -e`)
-together with the testing dependencies like `pytest` and its plugins.
-
-You can then edit the source code in the development tree
-and quickly run `pytest` manually without waiting  for `tox`
-to recreating the virtual environment each time.
-
-.. _sourceinstall:
-
-Installing bindings from source
-===============================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-First, build the core library::
-
-   cargo build --release -p deltachat_ffi --features jsonrpc
-
-`jsonrpc` feature is required even if not used by the bindings
-because `deltachat.h` includes JSON-RPC functions unconditionally.
-
-Create the virtual environment and activate it:
-
-   python -m venv env
-   source env/bin/activate
-
-Build and install the bindings:
-
-   export DCC_RS_DEV="$PWD"
-   export DCC_RS_TARGET=release
-   python -m pip install ./python
-
-`DCC_RS_DEV` environment variable specifies the location of
-the core development tree. If this variable is not set,
-`libdeltachat` library and `deltachat.h` header are expected
-to be installed system-wide.
-
-When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
-the build profile name to look up the artifacts
-in the target directory.
-In this case setting it can be skipped because
-`DCC_RS_TARGET=release` is the default.
-
-Building manylinux based wheels
-===============================
-
-Building portable manylinux wheels which come with libdeltachat.so
-can be done with Docker_ or Podman_.
-
-.. _Docker: https://www.docker.com/
-.. _Podman: https://podman.io/
-
-If you want to build your own wheels, build container image first::
-
-   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
-   $ docker build -t deltachat/coredeps scripts/coredeps
-
-This will use the ``scripts/coredeps/Dockerfile`` to build
-container image called ``deltachat/coredeps``.  You can afterwards
-find it with::
-
-   $ docker images
-
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e CHATMAIL_DOMAIN \
-       --rm -it -v $(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh

python/doc/Makefile

@@ -1,197 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-VERSION       = $(shell python -c "import conf ; print(conf.version)")
-DOCZIP        = devpi-$(VERSION).doc.zip
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = _build
-RSYNCOPTS     = -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-
-export HOME=/tmp/home
-export TESTHOME=$(HOME)
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# This variable is not auto generated as the order is important.
-USER_MAN_CHAPTERS = commands\
-                    user\
-                    indices\
-                    packages\
-#                 userman/index.rst\
-#                 userman/devpi_misc.rst\
-#                 userman/devpi_concepts.rst\
-
-
-#export DEVPI_CLIENTDIR=$(CURDIR)/.tmp_devpi_user_man/client
-#export DEVPI_SERVERDIR=$(CURDIR)/.tmp_devpi_user_man/server
-
-chapter = commands
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \
-        epub latex latexpdf text man changes linkcheck doctest gettext install \
-        quickstart-releaseprocess quickstart-pypimirror quickstart-server regen \
-        prepare-quickstart\
-        regen.server-fresh regen.server-restart regen.server-clean\
-        regen.uman-all regen.uman
-
-help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html       to make standalone HTML files"
-	@echo "  dirhtml    to make HTML files named index.html in directories"
-	@echo "  singlehtml to make a single large HTML file"
-	@echo "  pickle     to make pickle files"
-	@echo "  json       to make JSON files"
-	@echo "  htmlhelp   to make HTML files and a HTML help project"
-	@echo "  qthelp     to make HTML files and a qthelp project"
-	@echo "  devhelp    to make HTML files and a Devhelp project"
-	@echo "  epub       to make an epub"
-	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
-	@echo "  text       to make text files"
-	@echo "  man        to make manual pages"
-	@echo "  texinfo    to make Texinfo files"
-	@echo "  info       to make Texinfo files and run them through makeinfo"
-	@echo "  gettext    to make PO message catalogs"
-	@echo "  changes    to make an overview of all changed/added/deprecated items"
-	@echo "  linkcheck  to check all external links for integrity"
-	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
-	@echo
-	@echo "User Manual Regen Targets"
-	@echo "  regen.uman        regenerates page. of the user manual chapeter e.g. regen.uman chapter=..."
-	@echo "  regen.uman-all    regenerates the user manual"
-	@echo "  regen.uman-clean  stop temp server and clean up directory"
-	@echo "  Chapter List: $(USER_MAN_CHAPTERS)"
-
-clean:
-	-rm -rf $(BUILDDIR)/*
-
-version:
-	@echo "version $(VERSION)"
-
-doczip: html
-	python doczip.py $(DOCZIP) _build/html
-
-install: html
-	rsync -avz $(RSYNCOPTS) _build/html/ delta@py.delta.chat:build/master
-
-
-html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
-	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
-	@echo
-	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json:
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/devpi.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/devpi.qhc"
-
-devhelp:
-	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
-	@echo
-	@echo "Build finished."
-	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/devpi"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/devpi"
-	@echo "# devhelp"
-
-epub:
-	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
-	@echo
-	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make' in that directory to run these through (pdf)latex" \
-	      "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo "Running LaTeX files through pdflatex..."
-	$(MAKE) -C $(BUILDDIR)/latex all-pdf
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
-	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
-	@echo
-	@echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
-	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
-	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-texinfo:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo
-	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
-	@echo "Run \`make' in that directory to run these through makeinfo" \
-	      "(use \`make info' here to do that automatically)."
-
-info:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo "Running Texinfo files through makeinfo..."
-	make -C $(BUILDDIR)/texinfo info
-	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
-
-gettext:
-	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
-	@echo
-	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
-
-changes:
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
-	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
-	@echo
-	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
-
-

python/doc/_templates/globaltoc.html

@@ -1,17 +0,0 @@
-
-<div class="globaltoc">
-
-<ul>
-  <li><a href="{{ pathto('index') }}">index</a></li>
-  <li><a href="{{ pathto('install') }}">install</a></li>
-  <li><a href="{{ pathto('api') }}">high level API</a></li>
-  <li><a href="{{ pathto('lapi') }}">low level API</a></li>
-</ul>
-<b>external links:</b>
-<ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
-  <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
-  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
-</ul>
-
-</div>

python/doc/_templates/sidebarintro.html

@@ -1 +0,0 @@
-<h3>deltachat {{release}}</h3>

python/doc/cffi/api.rst

@@ -1,5 +1,4 @@
-
-high level API reference
+High Level API Reference
 ========================
 
 - :class:`deltachat.Account` (your main entry point, creates the
@@ -8,28 +7,14 @@ high level API reference
 - :class:`deltachat.Chat`
 - :class:`deltachat.Message`
 
-Account
--------
-
 .. autoclass:: deltachat.Account
-    :members:
-
-
-Contact
--------
+   :members:
 
 .. autoclass:: deltachat.Contact
-    :members:
-
-Chat
-----
+   :members:
 
 .. autoclass:: deltachat.Chat
-    :members:
-
-Message
--------
+   :members:
 
 .. autoclass:: deltachat.Message
-    :members:
-
+   :members:

python/doc/cffi/examples.rst

@@ -1,11 +1,10 @@
-
-examples
+Examples
 ========
 
 Once you have :doc:`installed deltachat bindings <install>`
 you need email/password credentials for an IMAP/SMTP account.
 Delta Chat developers and the CI system use a special URL to create
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
+temporary email accounts on `testrun.org <https://testrun.org/>`_ for testing.
 
 Receiving a Chat message from the command line
 ----------------------------------------------
@@ -16,11 +15,11 @@ Here is a simple bot that:
 
 - terminates the bot if the message `/quit` is sent
 
-.. include:: ../examples/echo_and_quit.py
+.. include:: ../../examples/echo_and_quit.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
+by specifying a database path, an email address and password of
 a SMTP-IMAP account::
 
     $ cd examples
@@ -40,11 +39,11 @@ Here is a simple bot that:
 
 - tracks member additions and removals for all chat groups
 
-.. include:: ../examples/group_tracking.py
+.. include:: ../../examples/group_tracking.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
+by specifying a database path, an email address and password of
 a SMTP-IMAP account::
 
     python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db

python/doc/cffi/install.rst

@@ -0,0 +1,80 @@
+Install
+=======
+
+Installing pre-built packages (Linux-only)
+------------------------------------------
+
+If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
+
+We recommend to first create a fresh Python virtual environment
+and activate it in your shell::
+
+    python -m venv env
+    source env/bin/activate
+
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``env`` directory and leaves
+your system installation alone.
+
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
+
+    pip install deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
+
+.. _sourceinstall:
+
+Installing bindings from source
+-------------------------------
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+First, build the core library::
+
+   cargo build --release -p deltachat_ffi --features jsonrpc
+
+`jsonrpc` feature is required even if not used by the bindings
+because `deltachat.h` includes JSON-RPC functions unconditionally.
+
+Create the virtual environment and activate it::
+
+   python -m venv env
+   source env/bin/activate
+
+Build and install the bindings::
+
+   export DCC_RS_DEV="$PWD"
+   export DCC_RS_TARGET=release
+   python -m pip install ./python
+
+`DCC_RS_DEV` environment variable specifies the location of
+the core development tree. If this variable is not set,
+`libdeltachat` library and `deltachat.h` header are expected
+to be installed system-wide.
+
+When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
+the build profile name to look up the artifacts
+in the target directory.
+In this case setting it can be skipped because
+`DCC_RS_TARGET=release` is the default.

python/doc/cffi/intro.rst

@@ -0,0 +1,11 @@
+Introduction
+============
+
+CFFI bindings are available via the `deltachat <https://pypi.org/project/deltachat/>`_ Python package.
+The package contains both the Python bindings and the Delta Chat core.
+It is provided only for Linux.
+
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
+low-level CFFI bindings to the C interface of the Delta Chat core
+and high-level Python bindings built on top of CFFI bindings.

python/doc/cffi/lapi.rst

@@ -1,8 +1,7 @@
+Low Level API Reference
+=======================
 
-low level API reference
-===================================
-
-for full doxygen-generated C-docs, defines and functions please checkout
+For full doxygen-generated C-docs, defines and functions please checkout
 
 https://c.delta.chat
 

python/doc/cffi/manylinux.rst

@@ -0,0 +1,25 @@
+Building Manylinux-Based Wheels
+===============================
+
+Building portable manylinux wheels which come with libdeltachat.so
+can be done with Docker_ or Podman_.
+
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
+
+If you want to build your own wheels, build container image first::
+
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
+
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
+find it with::
+
+   $ docker images
+
+This docker image can be used to run tests and build Python wheels for all interpreters::
+
+    $ docker run -e CHATMAIL_DOMAIN \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/doc/cffi/tests.rst

@@ -0,0 +1,49 @@
+Running Tests
+=============
+
+Recommended way to run tests is using `scripts/run-python-test.sh`
+script provided in the core repository.
+
+This script compiles the library in debug mode and runs the tests using `tox`_.
+By default it will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real email servers.
+
+.. _`tox`: https://tox.wiki
+.. _livetests:
+
+Running "Live" Tests With Temporary Accounts
+--------------------------------------------
+
+If you want to run live functional tests
+you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
+that creates email accounts like this::
+
+    export CHATMAIL_DOMAIN=nine.testrun.org
+
+With this account-creation setting, pytest runs create ephemeral email accounts on the server.
+These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
+After setting the variable, either rerun `scripts/run-python-test.sh`
+or run offline and online tests with `tox` directly::
+
+    tox -e py
+
+Each test run creates new accounts.
+
+Developing the Bindings
+-----------------------
+
+If you want to develop or debug the bindings,
+you can create a testing development environment using `tox`::
+
+    export DCC_RS_DEV="$PWD"
+    export DCC_RS_TARGET=debug
+    tox -c python --devenv env -e py
+    . env/bin/activate
+
+Inside this environment the bindings are installed
+in editable mode (as if installed with `python -m pip install -e`)
+together with the testing dependencies like `pytest` and its plugins.
+
+You can then edit the source code in the development tree
+and quickly run `pytest` manually without waiting  for `tox`
+to recreating the virtual environment each time.

python/doc/changelog.rst

@@ -1,4 +0,0 @@
-Changelog for deltachat-core's Python bindings
-==============================================
-
-.. include:: ../CHANGELOG

python/doc/conf.py

@@ -1,138 +1,94 @@
-# -*- coding: utf-8 -*-
-#
-# devpi documentation build configuration file, created by
-# sphinx-quickstart on Mon Jun  3 16:11:22 2013.
-#
-# This file is execfile()d with the current directory set to its containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import sys
-import os
+from pathlib import Path
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 from deltachat import __version__ as release
+
 version = ".".join(release.split(".")[:2])
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
-
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    #'sphinx.ext.intersphinx',
-    'sphinx.ext.todo',
-    'sphinx.ext.viewcode',
-    'breathe',
-    #'sphinx.ext.githubpages',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.viewcode",
+    "breathe",
+    "sphinx_rtd_theme",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+project = "Delta Chat"
+copyright = "2023, Delta Chat contributors"
+author = "Delta Chat contributors"
 
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['sketch', '_build', "attic"]
+exclude_patterns = ["sketch", "_build", "attic", "Thumbs.db", ".DS_Store"]
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 
 # -- breathe options ------
 
-breathe_projects = {
-    "deltachat": "../../docs/xml/"
-}
+breathe_projects = {"deltachat": Path("../../docs/xml/")}
 
 breathe_default_project = "deltachat"
 
 # -- Options for HTML output ---------------------------------------------------
 
-sys.path.append(os.path.abspath('_themes'))
-html_theme_path = ['_themes']
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-# html_theme = 'flask'
-html_theme = 'alabaster'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-html_theme_options = {
-    'logo': '_static/delta-chat.svg',
-    'font_size': "1.1em",
-    'caption_font_size': "0.9em",
-    'code_font_size': "1.1em",
-
-
-}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = ["_themes"]
+html_theme = "sphinx_rtd_theme"
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
@@ -141,51 +97,34 @@ html_logo = "_static/delta-chat.svg"
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = '_static/favicon.ico'
+html_favicon = "_static/favicon.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#
-html_sidebars = {
-    'index': [
-        'sidebarintro.html',
-        'globaltoc.html',
-        'searchbox.html'
-    ],
-    '**': [
-        'sidebarintro.html',
-        'globaltoc.html',
-        'relations.html',
-        'searchbox.html'
-    ]
-}
-
+# html_use_smartypants = True
 
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
@@ -194,71 +133,65 @@ html_show_sourcelink = False
 html_show_sphinx = False
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-html_use_opensearch = 'https://doc.devpi.net'
+html_use_opensearch = "https://doc.devpi.net"
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'deltachat-python'
+htmlhelp_basename = "deltachat-python"
 
 # -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-'pointsize': '12pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '12pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'devpi.tex', u'deltachat documentation',
-   u'holger krekel', 'manual'),
+    ("index", "devpi.tex", "deltachat documentation", "holger krekel", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'deltachat', u'deltachat documentation',
-     [u'holger krekel'], 1)
-]
+man_pages = [("index", "deltachat", "deltachat documentation", ["holger krekel"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output ------------------------------------------------
@@ -267,30 +200,38 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'devpi', u'devpi Documentation',
-   u'holger krekel', 'devpi', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "devpi",
+        "devpi Documentation",
+        "holger krekel",
+        "devpi",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {"http://docs.python.org/": None}
 
 # autodoc options
 autodoc_member_order = "bysource"
+
+
 # always document __init__ functions
 def skip(app, what, name, obj, skip, options):
     return skip
 
+
 def setup(app):
     app.connect("autodoc-skip-member", skip)
-

python/doc/index.rst

@@ -1,41 +1,44 @@
-deltachat python bindings
-=========================
+Delta Chat Python bindings, new and old
+=======
 
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
-
-- :doc:`api` is a high level interface to deltachat-core.
-
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
-
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
-  <https://github.com/deltachat/deltachat-core-rust>`_.
-
-
-
-getting started
----------------
+`Delta Chat <https://delta.chat/>`_ provides two kinds of Python bindings
+to the `Rust Core <https://github.com/deltachat/deltachat-core-rust>`_:
+JSON-RPC bindings and CFFI bindings.
+When starting a new project it is recommended to use JSON-RPC bindings,
+which are used in the Delta Chat Desktop app through generated Typescript-bindings. 
+The Python JSON-RPC bindings are maintained by Delta Chat core developers. 
+Most existing bot projects and many tests in Delta Chat's own core library
+still use the CFFI-bindings, and it is going to be maintained certainly also in 2024. 
+New APIs might however only appear in the JSON-RPC bindings, 
+as the CFFI bindings are increasingly in maintenance-only mode. 
 
 .. toctree::
    :maxdepth: 2
+   :caption: JSON-RPC Bindings
 
-   install
-   examples
+   jsonrpc/intro
+   jsonrpc/install
+   jsonrpc/examples
+   jsonrpc/reference
+   jsonrpc/develop
 
 .. toctree::
-   :hidden:
+   :maxdepth: 2
+   :caption: CFFI Bindings
 
-   links
-   changelog
-   api
-   lapi
-   plugins
-
-..
-    Indices and tables
-    ==================
-
-    * :ref:`genindex`
-    * :ref:`modindex`
-    * :ref:`search`
+   cffi/intro
+   cffi/install
+   cffi/examples
+   cffi/manylinux
+   cffi/tests
+   cffi/api
+   cffi/lapi
+   cffi/plugins
 
+.. _`deltachat`: https://delta.chat
+.. _`deltachat-core repo`: https://github.com/deltachat
+.. _pip: http://pypi.org/project/pip/
+.. _virtualenv: http://pypi.org/project/virtualenv/
+.. _merlinux: http://merlinux.eu
+.. _pypi: http://pypi.org/
+.. _`issue-tracker`: https://github.com/deltachat/deltachat-core-rust

python/doc/install.rst

@@ -1,2 +0,0 @@
-
-.. include:: ../README.rst

python/doc/jsonrpc/develop.rst

@@ -0,0 +1,68 @@
+===========
+Development
+===========
+
+To develop JSON-RPC bindings,
+clone the `deltachat-core-rust <https://github.com/deltachat/deltachat-core-rust/>`_ repository::
+
+   git clone https://github.com/deltachat/deltachat-core-rust.git
+
+Testing
+=======
+
+To run online tests, set ``CHATMAIL_DOMAIN``
+to a domain of the email server
+that can be used to create testing accounts::
+
+    export CHATMAIL_DOMAIN=nine.testrun.org
+
+Then run ``scripts/run-rpc-test.sh``
+to build debug version of ``deltachat-rpc-server``
+and run ``deltachat-rpc-client`` tests
+in a separate virtual environment managed by `tox <https://tox.wiki/>`_.
+
+Development Environment
+=======================
+
+Creating a new virtual environment
+to run the tests each time
+as ``scripts/run-rpc-test.sh`` does is slow
+if you are changing the tests or the code
+and want to rerun the tests each time.
+
+If you are developing the tests,
+it is better to create a persistent virtual environment.
+You can do this by running ``scripts/make-rpc-testenv.sh``.
+This creates a virtual environment ``venv`` which you can then enter with::
+
+   . venv/bin/activate
+
+Then you can run the tests with
+
+::
+
+    pytest deltachat-rpc-client/tests/
+
+Refer to `pytest documentation <https://docs.pytest.org/>` for details.
+
+If make the changes to Delta Chat core
+or Python bindings, you can rebuild the environment by rerunning
+``scripts/make-rpc-testenv.sh``.
+It is ok to rebuild the activated environment this way,
+you do not need to deactivate or reactivate the environment each time.
+
+Using REPL
+==========
+
+Once you have a development environment,
+you can quickly test things in REPL::
+
+   $ python
+   >>> from deltachat_rpc_client import *
+   >>> rpc = Rpc()
+   >>> rpc.start()
+   >>> dc = DeltaChat(rpc)
+   >>> system_info = dc.get_system_info()
+   >>> system_info["level"]
+   'awesome'
+   >>> rpc.close()

python/doc/jsonrpc/examples.rst

@@ -0,0 +1,19 @@
+Examples
+========
+
+Echo bot
+--------
+.. include:: ../../../deltachat-rpc-client/examples/echobot_no_hooks.py
+    :literal:
+
+Echo bot with hooks
+-------------------
+.. include:: ../../../deltachat-rpc-client/examples/echobot.py
+    :literal:
+
+Advanced echo bot
+-----------------
+
+.. include:: ../../../deltachat-rpc-client/examples/echobot_advanced.py
+    :literal:
+

python/doc/jsonrpc/install.rst

@@ -0,0 +1,36 @@
+Install
+=======
+
+To use JSON-RPC bindings for Delta Chat core you will need
+a ``deltachat-rpc-server`` binary which provides Delta Chat core API over JSON-RPC
+and a ``deltachat-rpc-client`` Python package which is a JSON-RPC client that starts ``deltachat-rpc-server`` process and uses JSON-RPC API.
+
+`Create a virtual environment <https://docs.python.org/3/library/venv.html>`__ if you
+don’t have one already and activate it::
+
+   $ python -m venv venv
+   $ . venv/bin/activate
+
+Install ``deltachat-rpc-server``
+--------------------------------
+
+To get ``deltachat-rpc-server`` binary you have three options:
+
+1. Install ``deltachat-rpc-server`` from PyPI using ``pip install deltachat-rpc-server``.
+2. Build and install ``deltachat-rpc-server`` from source with ``cargo install --git https://github.com/deltachat/deltachat-core-rust/ deltachat-rpc-server``.
+3. Download prebuilt release from https://github.com/deltachat/deltachat-core-rust/releases and install it into ``PATH``.
+
+Check that ``deltachat-rpc-server`` is installed and can run::
+
+   $ deltachat-rpc-server --version
+   1.131.4
+
+Then install ``deltachat-rpc-client`` with ``pip install deltachat-rpc-client``.
+
+Install ``deltachat-rpc-client``
+--------------------------------
+
+To get ``deltachat-rpc-client`` Python library you can:
+
+1. Install ``deltachat-rpc-client`` from PyPI using ``pip install deltachat-rpc-client``.
+2. Install ``deltachat-rpc-client`` from source with ``pip install git+https://github.com/deltachat/deltachat-core-rust.git@main#subdirectory=deltachat-rpc-client``.

python/doc/jsonrpc/intro.rst

@@ -0,0 +1,8 @@
+Introduction
+============
+
+JSON-RPC bindings are available via the `deltachat-rpc-client <https://pypi.org/project/deltachat-rpc-client/>`_ Python package.
+This package provides only the Python bindings and requires ``deltachat-rpc-server`` binary to be installed.
+`deltachat-rpc-server <https://pypi.org/project/deltachat-rpc-server/>`_ package provides ``deltachat-rpc-server`` binary for Linux, Windows, macOS and Android.
+
+RPC client connects to standalone Delta Chat RPC server ``deltachat-rpc-server`` and provides Python interface to it.

python/doc/jsonrpc/reference.rst

@@ -0,0 +1,5 @@
+API Reference
+=============
+
+.. automodule:: deltachat_rpc_client
+   :members:

python/doc/links.rst

@@ -1,11 +0,0 @@
-
-links
-================================
-
-.. _`deltachat`: https://delta.chat
-.. _`deltachat-core repo`: https://github.com/deltachat
-.. _pip: http://pypi.org/project/pip/
-.. _virtualenv: http://pypi.org/project/virtualenv/
-.. _merlinux: http://merlinux.eu
-.. _pypi: http://pypi.org/
-.. _`issue-tracker`: https://github.com/deltachat/deltachat-core

python/doc/make.bat

@@ -1,190 +0,0 @@
-@ECHO OFF
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set BUILDDIR=_build
-set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
-set I18NSPHINXOPTS=%SPHINXOPTS% .
-if NOT "%PAPER%" == "" (
-	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
-	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
-)
-
-if "%1" == "" goto help
-
-if "%1" == "help" (
-	:help
-	echo.Please use `make ^<target^>` where ^<target^> is one of
-	echo.  html       to make standalone HTML files
-	echo.  dirhtml    to make HTML files named index.html in directories
-	echo.  singlehtml to make a single large HTML file
-	echo.  pickle     to make pickle files
-	echo.  json       to make JSON files
-	echo.  htmlhelp   to make HTML files and a HTML help project
-	echo.  qthelp     to make HTML files and a qthelp project
-	echo.  devhelp    to make HTML files and a Devhelp project
-	echo.  epub       to make an epub
-	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
-	echo.  text       to make text files
-	echo.  man        to make manual pages
-	echo.  texinfo    to make Texinfo files
-	echo.  gettext    to make PO message catalogs
-	echo.  changes    to make an overview over all changed/added/deprecated items
-	echo.  linkcheck  to check all external links for integrity
-	echo.  doctest    to run all doctests embedded in the documentation if enabled
-	goto end
-)
-
-if "%1" == "clean" (
-	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
-	del /q /s %BUILDDIR%\*
-	goto end
-)
-
-if "%1" == "html" (
-	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
-	goto end
-)
-
-if "%1" == "dirhtml" (
-	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
-	goto end
-)
-
-if "%1" == "singlehtml" (
-	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
-	goto end
-)
-
-if "%1" == "pickle" (
-	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can process the pickle files.
-	goto end
-)
-
-if "%1" == "json" (
-	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can process the JSON files.
-	goto end
-)
-
-if "%1" == "htmlhelp" (
-	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can run HTML Help Workshop with the ^
-.hhp project file in %BUILDDIR%/htmlhelp.
-	goto end
-)
-
-if "%1" == "qthelp" (
-	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can run "qcollectiongenerator" with the ^
-.qhcp project file in %BUILDDIR%/qthelp, like this:
-	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\devpi.qhcp
-	echo.To view the help file:
-	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\devpi.ghc
-	goto end
-)
-
-if "%1" == "devhelp" (
-	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished.
-	goto end
-)
-
-if "%1" == "epub" (
-	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The epub file is in %BUILDDIR%/epub.
-	goto end
-)
-
-if "%1" == "latex" (
-	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
-	goto end
-)
-
-if "%1" == "text" (
-	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The text files are in %BUILDDIR%/text.
-	goto end
-)
-
-if "%1" == "man" (
-	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The manual pages are in %BUILDDIR%/man.
-	goto end
-)
-
-if "%1" == "texinfo" (
-	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
-	goto end
-)
-
-if "%1" == "gettext" (
-	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
-	goto end
-)
-
-if "%1" == "changes" (
-	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.The overview file is in %BUILDDIR%/changes.
-	goto end
-)
-
-if "%1" == "linkcheck" (
-	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Link check complete; look for any errors in the above output ^
-or in %BUILDDIR%/linkcheck/output.txt.
-	goto end
-)
-
-if "%1" == "doctest" (
-	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Testing of doctests in the sources finished, look at the ^
-results in %BUILDDIR%/doctest/output.txt.
-	goto end
-)
-
-:end

python/src/deltachat/account.py

@@ -571,11 +571,11 @@ class Account:
         return ScannedQRCode(lot)
 
     def qr_setup_contact(self, qr):
-        """setup contact and return a Chat after contact is established.
+        """setup contact and return a `Chat` instance after contact is established.
+
+        This function triggers a network protocol in the background between
+        the emitter of the QR code and this account.
 
-        Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
-        is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
         assert self.check_qr(qr).is_ask_verifycontact()
@@ -585,11 +585,11 @@ class Account:
         return Chat(self, chat_id)
 
     def qr_join_chat(self, qr):
-        """join a chat group through a QR code.
+        """return a `Chat` instance for which the securejoin network
+        protocol has been started.
 
-        Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
-        is returned which is the chat that we just joined.
+        This function triggers a network protocol in the background between
+        the emitter of the QR code and this account.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
         """

python/src/deltachat/testplugin.py

@@ -23,12 +23,6 @@ from .events import FFIEventLogger, FFIEventTracker
 
 def pytest_addoption(parser):
     group = parser.getgroup("deltachat testplugin options")
-    group.addoption(
-        "--liveconfig",
-        action="store",
-        default=None,
-        help="a file with >=2 lines where each line contains NAME=VALUE config settings for one account",
-    )
     group.addoption(
         "--chatmail",
         action="store",
@@ -58,8 +52,6 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    cfg = config.getoption("--liveconfig")
-
     cfg = config.getoption("--chatmail")
     if not cfg:
         cfg = os.getenv("CHATMAIL_DOMAIN")
@@ -133,13 +125,10 @@ def pytest_report_header(config, startdir):
         ),
     ]
 
-    cfg = config.option.liveconfig
-    if cfg:
-        if "?" in cfg:
-            url, token = cfg.split("?", 1)
-            summary.append(f"Liveconfig provider: {url}?<token omitted>")
-        else:
-            summary.append(f"Liveconfig file: {cfg}")
+    chatmail_opt = config.getoption("--chatmail")
+    if chatmail_opt:
+        summary.append(f"Chatmail account provider: {chatmail_opt}")
+
     return summary
 
 
@@ -164,11 +153,7 @@ class TestProcess:
     def get_liveconfig_producer(self):
         """provide live account configs, cached on a per-test-process scope
         so that test functions can re-use already known live configs.
-        Depending on the --liveconfig option this comes from
-        a file with a line specifying each accounts config
-        or a --chatmail domain.
         """
-        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
         chatmail_opt = self.pytestconfig.getoption("--chatmail")
         if chatmail_opt:
             # Use a chatmail instance.
@@ -178,7 +163,8 @@ class TestProcess:
                 try:
                     yield self._configlist[index]
                 except IndexError:
-                    username = "ci-" + "".join(random.choice("2345789acdefghjkmnpqrstuvwxyz") for i in range(6))
+                    part = "".join(random.choices("2345789acdefghjkmnpqrstuvwxyz", k=6))
+                    username = f"ci-{part}"
                     password = f"{username}${username}"
                     addr = f"{username}@{domain}"
                     config = {"addr": addr, "mail_pw": password}
@@ -186,20 +172,9 @@ class TestProcess:
                     self._configlist.append(config)
                     yield config
             pytest.fail(f"more than {MAX_LIVE_CREATED_ACCOUNTS} live accounts requested.")
-        elif liveconfig_opt:
-            # Read a list of accounts from file.
-            for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith("#"):
-                    d = {}
-                    for part in line.split():
-                        name, value = part.split("=")
-                        d[name] = value
-                    self._configlist.append(d)
-
-            yield from iter(self._configlist)
         else:
             pytest.skip(
-                "specify --liveconfig, CHATMAIL_DOMAIN or --chatmail to provide live accounts",
+                "specify CHATMAIL_DOMAIN or --chatmail to provide live accounts",
             )
 
     def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
@@ -546,6 +521,7 @@ class ACFactory:
         configdict.setdefault("bcc_self", False)
         configdict.setdefault("mvbox_move", False)
         configdict.setdefault("sentbox_watch", False)
+        configdict.setdefault("sync_msgs", False)
         ac.update_config(configdict)
         self._preconfigure_key(ac, configdict["addr"])
         return ac

python/tests/test_0_complex_or_slow.py

@@ -140,6 +140,10 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
     assert msg.is_system_message()
     assert "added" in msg.text.lower()
 
+    assert any(
+        m.is_system_message() and m.text == "Messages are guaranteed to be end-to-end encrypted from now on."
+        for m in msg.chat.get_messages()
+    )
     lp.sec("ac1: send message")
     msg_out = chat1.send_text("hello")
     assert msg_out.is_encrypted()
@@ -580,6 +584,7 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.get_sender_contact().addr == ac1.get_config("addr")
     chat2 = msg_in.chat
     assert chat2.is_protected()
+    assert chat2.get_messages()[0].text == "Messages are guaranteed to be end-to-end encrypted from now on."
 
     lp.sec("ac2_offl: sending message")
     msg_out = chat2.send_text("hello")
@@ -647,6 +652,15 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     chat2.send_text("hi2")
 
     lp.sec("ac2_offl: receiving message")
+    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+    msg_in = ac2_offl.get_message_by_id(ev.data2)
+    assert msg_in.is_system_message()
+    assert msg_in.text == "Messages are guaranteed to be end-to-end encrypted from now on."
+
+    # We need to consume one event that has data2=0
+    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+    assert ev.data2 == 0
+
     ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
     msg_in = ac2_offl.get_message_by_id(ev.data2)
     assert not msg_in.is_system_message()

python/tests/test_1_online.py

@@ -9,7 +9,7 @@ import pytest
 from imap_tools import AND, U
 
 import deltachat as dc
-from deltachat import account_hookimpl, Message, Chat
+from deltachat import account_hookimpl, Message
 from deltachat.tracker import ImexTracker
 
 
@@ -367,7 +367,7 @@ def test_webxdc_download_on_demand(acfactory, data, lp):
 
     lp.sec("ac2 sets download limit")
     ac2.set_config("download_limit", "100")
-    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(50000))}, "some test data")
+    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data")
     ac2_update = ac2._evtracker.wait_next_incoming_message()
     assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE
     assert not msg2.get_status_updates()
@@ -489,7 +489,7 @@ def test_forward_messages(acfactory, lp):
     lp.sec("ac2: check new chat has a forwarded message")
     assert chat3.is_promoted()
     messages = chat3.get_messages()
-    assert len(messages) == 1
+    assert len(messages) == 2
     msg = messages[-1]
     assert msg.is_forwarded()
     ac2.delete_messages(messages)
@@ -1445,7 +1445,7 @@ def test_reaction_to_partially_fetched_msg(acfactory, lp, tmp_path):
       message, then processes a partially downloaded message.
     - As a result, Bob does not see a reaction
     """
-    download_limit = 32768
+    download_limit = 300000
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_addr = ac1.get_config("addr")
     chat = ac1.create_chat(ac2)
@@ -1658,75 +1658,6 @@ def test_ac_setup_message_twice(acfactory, lp):
     assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
 
 
-def test_qr_setup_contact(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-    qr = ac1.get_setup_contact_qr()
-
-    lp.sec("ac2: start QR-code based setup contact protocol")
-    ch = ac2.qr_setup_contact(qr)
-    assert ch.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-
-@pytest.mark.parametrize("verified_one_on_one_chats", [0, 1])
-def test_qr_join_chat(acfactory, lp, verified_one_on_one_chats):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("verified_one_on_one_chats", verified_one_on_one_chats)
-    ac2.set_config("verified_one_on_one_chats", verified_one_on_one_chats)
-
-    lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-    chat = ac1.create_group_chat("hello")
-    qr = chat.get_join_qr()
-    lp.sec("ac2: start QR-code based join-group protocol")
-    ch = ac2.qr_join_chat(qr)
-    lp.sec("ac2: qr_join_chat() returned")
-    assert ch.id >= 10
-    # check that at least some of the handshake messages are deleted
-    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
-    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "Member Me ({}) added by {}.".format(ac2.get_config("addr"), ac1.get_config("addr"))
-
-    # ac1 reloads the chat.
-    chat = Chat(chat.account, chat.id)
-    assert not chat.is_protected()
-
-    # ac2 reloads the chat.
-    ch = Chat(ch.account, ch.id)
-    assert not ch.is_protected()
-
-
-def test_qr_new_group_unblocked(acfactory, lp):
-    """Regression test for a bug intoduced in core v1.113.0.
-    ac2 scans a verified group QR code created by ac1.
-    This results in creation of a blocked 1:1 chat with ac1 on ac2,
-    but ac1 contact is not blocked on ac2.
-    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
-    ac2 should receive a message and create a contact request for the group.
-    Due to a bug previously ac2 created a blocked group.
-    """
-
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1_chat = ac1.create_group_chat("Group for joining", verified=True)
-    qr = ac1_chat.get_join_qr()
-    ac2.qr_join_chat(qr)
-
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    ac1_new_chat = ac1.create_group_chat("Another group")
-    ac1_new_chat.add_contact(ac2)
-    # Receive "Member added" message.
-    ac2._evtracker.wait_next_incoming_message()
-
-    ac1_new_chat.send_text("Hello!")
-    ac2_msg = ac2._evtracker.wait_next_incoming_message()
-    assert ac2_msg.text == "Hello!"
-    assert ac2_msg.chat.is_contact_request()
-
-
 def test_qr_email_capitalization(acfactory, lp):
     """Regression test for a bug
     that resulted in failure to propagate verification via gossip in a verified group
@@ -1904,7 +1835,7 @@ def test_system_group_msg_from_blocked_user(acfactory, lp):
     chat_on_ac2.send_text("This will arrive")
     msg = ac1._evtracker.wait_next_incoming_message()
     assert msg.text == "This will arrive"
-    message_texts = [m.text for m in chat_on_ac1.get_messages()]
+    message_texts = [m.text for m in chat_on_ac1.get_messages() if not m.is_system_message()]
     assert len(message_texts) == 2
     assert "First group message" in message_texts
     assert "This will arrive" in message_texts
@@ -2048,6 +1979,32 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
 
 
+def test_all_work_done(acfactory, lp):
+    """
+    Tests that calling start_io() immediately followed by maybe_network()
+    and then waiting for all_work_done() reliably fetches the messages
+    delivered while account was offline.
+    In other words, connectivity should not change to a state
+    where all_work_done() returns true until IMAP connection goes idle.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    ac1.stop_io()
+    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+    ac1.direct_imap.select_config_folder("inbox")
+    with ac1.direct_imap.idle() as idle1:
+        ac2.create_chat(ac1).send_text("Hi")
+        idle1.wait_for_new_message()
+
+    ac1.start_io()
+    ac1.maybe_network()
+    ac1._evtracker.wait_for_all_work_done()
+    msgs = ac1.create_chat(ac2).get_messages()
+    assert len(msgs) == 1
+    assert msgs[0].text == "Hi"
+
+
 def test_fetch_deleted_msg(acfactory, lp):
     """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
     hundreds of times, because uid_next was not updated.
@@ -2474,47 +2431,6 @@ def test_delete_deltachat_folder(acfactory):
     assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
-def test_aeap_flow_verified(acfactory, lp):
-    """Test that a new address is added to a contact when it changes its address."""
-    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
-
-    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
-    chat = ac1.create_group_chat("hello", verified=True)
-    assert chat.is_protected()
-    qr = chat.get_join_qr()
-    lp.sec("ac2: start QR-code based join-group protocol")
-    chat2 = ac2.qr_join_chat(qr)
-    assert chat2.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    lp.sec("sending first message")
-    msg_out = chat.send_text("old address")
-
-    lp.sec("receiving first message")
-    ac2._evtracker.wait_next_incoming_message()  # member added message
-    msg_in_1 = ac2._evtracker.wait_next_incoming_message()
-    assert msg_in_1.text == msg_out.text
-
-    lp.sec("changing email account")
-    ac1.set_config("addr", ac1new.get_config("addr"))
-    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
-    ac1.stop_io()
-    configtracker = ac1.configure()
-    configtracker.wait_finish()
-    ac1.start_io()
-
-    lp.sec("sending second message")
-    msg_out = chat.send_text("changed address")
-
-    lp.sec("receiving second message")
-    msg_in_2 = ac2._evtracker.wait_next_incoming_message()
-    assert msg_in_2.text == msg_out.text
-    assert msg_in_2.chat.id == msg_in_1.chat.id
-    assert msg_in_2.get_sender_contact().addr == ac1new.get_config("addr")
-    assert len(msg_in_2.chat.get_contacts()) == 2
-    assert ac1new.get_config("addr") in [contact.addr for contact in msg_in_2.chat.get_contacts()]
-
-
 def test_archived_muted_chat(acfactory, lp):
     """If an archived and muted chat receives a new message, DC_EVENT_MSGS_CHANGED for
     DC_CHAT_ID_ARCHIVED_LINK must be generated if the chat had only seen messages previously.

python/tests/test_2_increation.py

@@ -26,7 +26,7 @@ def wait_msgs_changed(account, msgs_list):
                     break
         else:
             account.log(f"waiting mismatch data1={data1} data2={data2}")
-    return ev.data1, ev.data2
+    return ev.data2
 
 
 class TestOnlineInCreation:
@@ -68,14 +68,17 @@ class TestOnlineInCreation:
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
 
-        lp.sec("forward the message while still in creation")
+        lp.sec("create a new group")
         chat2 = ac1.create_group_chat("newgroup")
+        wait_msgs_changed(ac1, [(0, 0)])
+
+        lp.sec("add a contact to new group")
         chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
+        wait_msgs_changed(ac1, [(chat2.id, None)])
+
+        lp.sec("forward the message while still in creation")
         ac1.forward_messages([prepared_original], chat2)
-        # XXX there might be two EVENT_MSGS_CHANGED and only one of them
-        # is the one caused by forwarding
-        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
+        forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 

python/tests/test_3_offline.py

@@ -5,8 +5,6 @@ from datetime import datetime, timedelta, timezone
 import pytest
 
 import deltachat as dc
-from deltachat.capi import ffi, lib
-from deltachat.cutil import iter_array
 from deltachat.tracker import ImexFailed
 from deltachat import Account, account_hookimpl, Message
 
@@ -802,6 +800,7 @@ class TestOfflineChat:
     def test_audit_log_view_without_daymarker(self, ac1, lp):
         lp.sec("ac1: test audit log (show only system messages)")
         chat = ac1.create_group_chat(name="audit log sample data")
+
         # promote chat
         chat.send_text("hello")
         assert chat.is_promoted()
@@ -811,12 +810,6 @@ class TestOfflineChat:
         chat.set_name("audit log test group")
         chat.send_text("a message in between")
 
-        lp.sec("check message count of all messages")
-        assert len(chat.get_messages()) == 4
-
         lp.sec("check message count of only system messages (without daymarkers)")
-        dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, dc.const.DC_GCM_INFO_ONLY, 0),
-            lib.dc_array_unref,
-        )
-        assert len(list(iter_array(dc_array, lambda x: x))) == 2
+        sysmessages = [x for x in chat.get_messages() if x.is_system_message()]
+        assert len(sysmessages) == 3

python/tests/test_4_lowlevel.py

@@ -156,6 +156,8 @@ def test_markseen_invalid_message_ids(acfactory):
     chat = contact1.create_chat()
     chat.send_text("one message")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    # Skip configuration-related warnings, but not errors.
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
     ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")

python/tox.ini

@@ -62,9 +62,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-# Pinned due to incompatibility of breathe with sphinx 7.2: <https://github.com/breathe-doc/breathe/issues/943>
-    sphinx<=7.1.2
+    sphinx
     breathe
+    sphinx_rtd_theme
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

release-date.in

@@ -1 +1 @@
-2023-10-29
+2023-12-12

scripts/README.md

@@ -39,6 +39,8 @@ and an own build machine.
 
 - `android-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Android NDK.
 
+- `build-python-docs.sh` builds Python documentation into `dist/html/`.
+
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 

scripts/build-python-docs.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export DCC_RS_TARGET=debug
+export DCC_RS_DEV="$PWD"
+cargo build -p deltachat_ffi --features jsonrpc
+
+python3 -m venv venv
+venv/bin/pip install ./python
+venv/bin/pip install ./deltachat-rpc-client
+venv/bin/pip install sphinx breathe sphinx_rtd_theme
+venv/bin/pip install ./deltachat-rpc-client
+venv/bin/sphinx-build -b html -a python/doc/ dist/html

scripts/concourse/docs_wheels.yml

@@ -102,8 +102,6 @@ jobs:
             - name: deltachat-core-rust-release
               path: .
           outputs:
-            - name: py-docs
-              path: ./python/doc/_build/
             # Binary wheels
             - name: py-wheels
               path: ./python/.docker-tox/wheelhouse/
@@ -115,28 +113,6 @@ jobs:
               - |
                 scripts/run_all.sh
 
-      # Upload python docs to py.delta.chat
-      - task: upload-py-docs
-        config:
-          inputs:
-            - name: py-docs
-          image_resource:
-            type: registry-image
-            source:
-              repository: alpine
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache rsync openssh-client
-                mkdir -p ~/.ssh
-                chmod 700 ~/.ssh
-                echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-                chmod 600 ~/.ssh/id_ed25519
-                rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
-
       # Upload x86_64 wheels and source packages
       - task: upload-wheels
         config:

scripts/run_all.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Build the Delta Chat Core Rust library, Python wheels and docs 
+# Build the Delta Chat Core Rust library and Python wheels
 
 set -e -x
 
@@ -34,9 +34,3 @@ unset CHATMAIL_DOMAIN
 tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"
-
-
-echo -----------------------
-echo generating python docs
-echo -----------------------
-tox --workdir "$TOXWORKDIR" -e doc

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=3c8f7e846c915a183dc44536fb5480d1f25d7c42
+REV=18f714cf73d0bdfb8b013fa344494ab80c92b477
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

src/accounts.rs

@@ -8,11 +8,14 @@ use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
-use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
-use tokio::time::{sleep, Duration};
 use uuid::Uuid;
 
+#[cfg(not(target_os = "ios"))]
+use tokio::sync::oneshot;
+#[cfg(not(target_os = "ios"))]
+use tokio::time::{sleep, Duration};
+
 use crate::context::Context;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::stock_str::StockStrings;
@@ -258,8 +261,8 @@ impl Accounts {
     }
 
     /// Starts background tasks such as IMAP and SMTP loops for all accounts.
-    pub async fn start_io(&self) {
-        for account in self.accounts.values() {
+    pub async fn start_io(&mut self) {
+        for account in self.accounts.values_mut() {
             account.start_io().await;
         }
     }
@@ -303,6 +306,7 @@ impl Accounts {
 const CONFIG_NAME: &str = "accounts.toml";
 
 /// Lockfile name.
+#[cfg(not(target_os = "ios"))]
 const LOCKFILE_NAME: &str = "accounts.lock";
 
 /// Database file name.
@@ -338,22 +342,16 @@ impl Drop for Config {
 }
 
 impl Config {
-    /// Creates a new Config for `file`, but doesn't open/sync it.
-    async fn new_nosync(file: PathBuf, lock: bool) -> Result<Self> {
-        let dir = file.parent().context("Cannot get config file directory")?;
-        let inner = InnerConfig {
-            accounts: Vec::new(),
-            selected_account: 0,
-            next_id: 1,
-        };
-        if !lock {
-            let cfg = Self {
-                file,
-                inner,
-                lock_task: None,
-            };
-            return Ok(cfg);
-        }
+    #[cfg(target_os = "ios")]
+    async fn create_lock_task(_dir: PathBuf) -> Result<Option<JoinHandle<anyhow::Result<()>>>> {
+        // Do not lock accounts.toml on iOS.
+        // This results in 0xdead10cc crashes on suspend.
+        // iOS itself ensures that multiple instances of Delta Chat are not running.
+        Ok(None)
+    }
+
+    #[cfg(not(target_os = "ios"))]
+    async fn create_lock_task(dir: PathBuf) -> Result<Option<JoinHandle<anyhow::Result<()>>>> {
         let lockfile = dir.join(LOCKFILE_NAME);
         let mut lock = fd_lock::RwLock::new(fs::File::create(lockfile).await?);
         let (locked_tx, locked_rx) = oneshot::channel();
@@ -384,12 +382,32 @@ impl Config {
             rx.await?;
             Ok(())
         });
+        locked_rx.await?;
+        Ok(Some(lock_task))
+    }
+
+    /// Creates a new Config for `file`, but doesn't open/sync it.
+    async fn new_nosync(file: PathBuf, lock: bool) -> Result<Self> {
+        let dir = file.parent().context("Cannot get config file directory")?;
+        let inner = InnerConfig {
+            accounts: Vec::new(),
+            selected_account: 0,
+            next_id: 1,
+        };
+        if !lock {
+            let cfg = Self {
+                file,
+                inner,
+                lock_task: None,
+            };
+            return Ok(cfg);
+        }
+        let lock_task = Self::create_lock_task(dir.to_path_buf()).await?;
         let cfg = Self {
             file,
             inner,
-            lock_task: Some(lock_task),
+            lock_task,
         };
-        locked_rx.await?;
         Ok(cfg)
     }
 
@@ -406,11 +424,13 @@ impl Config {
     /// Takes a mutable reference because the saved file is a part of the `Config` state. This
     /// protects from parallel calls resulting to a wrong file contents.
     async fn sync(&mut self) -> Result<()> {
+        #[cfg(not(target_os = "ios"))]
         ensure!(!self
             .lock_task
             .as_ref()
             .context("Config is read-only")?
             .is_finished());
+
         let tmp_path = self.file.with_extension("toml.tmp");
         let mut file = fs::File::create(&tmp_path)
             .await

src/aheader.rs

@@ -67,7 +67,7 @@ impl Aheader {
 
 impl fmt::Display for Aheader {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        write!(fmt, "addr={};", self.addr)?;
+        write!(fmt, "addr={};", self.addr.to_lowercase())?;
         if self.prefer_encrypt == EncryptPreference::Mutual {
             write!(fmt, " prefer-encrypt=mutual;")?;
         }
@@ -262,5 +262,16 @@ mod tests {
             )
         )
         .contains("prefer-encrypt"));
+
+        // Always lowercase the address in the header.
+        assert!(format!(
+            "{}",
+            Aheader::new(
+                "TeSt@eXaMpLe.cOm".to_string(),
+                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                EncryptPreference::Mutual
+            )
+        )
+        .contains("test@example.com"));
     }
 }

src/chatlist.rs

@@ -761,4 +761,40 @@ mod tests {
         let summary = chats.get_summary(&t, 0, None).await.unwrap();
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_load_broken() {
+        let t = TestContext::new_bob().await;
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+            .await
+            .unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
+            .await
+            .unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
+            .await
+            .unwrap();
+
+        // check that the chatlist starts with the most recent message
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 3);
+
+        // obfuscated one chat
+        t.sql
+            .execute("UPDATE chats SET type=10 WHERE id=?", (chat_id1,))
+            .await
+            .unwrap();
+
+        // obfuscated chat can't be loaded
+        assert!(Chat::load_from_db(&t, chat_id1).await.is_err());
+
+        // chatlist loads fine
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+
+        // only corrupted chat fails to create summary
+        assert!(chats.get_summary(&t, 0, None).await.is_ok());
+        assert!(chats.get_summary(&t, 1, None).await.is_ok());
+        assert!(chats.get_summary(&t, 2, None).await.is_err());
+        assert_eq!(chats.get_index_for_id(chat_id1).unwrap(), 2);
+    }
 }

src/config.rs

@@ -5,6 +5,7 @@ use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
+use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -13,8 +14,10 @@ use crate::constants::DC_VERSION_STR;
 use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::events::EventType;
+use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
+use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
 
 /// The available configuration keys.
@@ -31,6 +34,8 @@ use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
     EnumProperty,
     PartialOrd,
     Ord,
+    Serialize,
+    Deserialize,
 )]
 #[strum(serialize_all = "snake_case")]
 pub enum Config {
@@ -266,6 +271,10 @@ pub enum Config {
     /// True if it is a bot account.
     Bot,
 
+    /// True when to skip initial start messages in groups.
+    #[strum(props(default = "0"))]
+    SkipStartMessages,
+
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
     #[strum(props(default = "0"))]
@@ -298,7 +307,7 @@ pub enum Config {
     DownloadLimit,
 
     /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set.
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "1"))]
     SyncMsgs,
 
     /// Space-separated list of all the authserv-ids which we believe
@@ -336,6 +345,24 @@ pub enum Config {
     VerifiedOneOnOneChats,
 }
 
+impl Config {
+    /// Whether the config option is synced across devices.
+    ///
+    /// This must be checked on both sides so that if there are different client versions, the
+    /// synchronisation of a particular option is either done or not done in both directions.
+    /// Moreover, receivers of a config value need to check if a key can be synced because some
+    /// settings (e.g. Avatar path) could otherwise lead to exfiltration of files from a receiver's
+    /// device if we assume an attacker to have control of a device in a multi-device setting or if
+    /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
+    /// mustn't be controlled by other devices.
+    pub(crate) fn is_synced(&self) -> bool {
+        matches!(
+            self,
+            Self::Displayname | Self::MdnsEnabled | Self::ShowEmails
+        )
+    }
+}
+
 impl Context {
     /// Returns true if configuration value is set for the given key.
     pub async fn config_exists(&self, key: Config) -> Result<bool> {
@@ -456,6 +483,16 @@ impl Context {
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
+        self.set_config_ex(key.is_synced().into(), key, value).await
+    }
+
+    pub(crate) async fn set_config_ex(
+        &self,
+        sync: sync::Sync,
+        key: Config,
+        mut value: Option<&str>,
+    ) -> Result<()> {
+        let better_value;
         match key {
             Config::Selfavatar => {
                 self.sql
@@ -482,10 +519,11 @@ impl Context {
                 ret?
             }
             Config::Displayname => {
-                let value = value.map(improve_single_line_input);
-                self.sql
-                    .set_raw_config(key.as_ref(), value.as_deref())
-                    .await?;
+                if let Some(v) = value {
+                    better_value = improve_single_line_input(v);
+                    value = Some(&better_value);
+                }
+                self.sql.set_raw_config(key.as_ref(), value).await?;
             }
             Config::Socks5Enabled
             | Config::BccSelf
@@ -509,10 +547,32 @@ impl Context {
                 );
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
+            Config::Addr => {
+                self.sql
+                    .set_raw_config(key.as_ref(), value.map(|s| s.to_lowercase()).as_deref())
+                    .await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
         }
+
+        if sync != Sync {
+            return Ok(());
+        }
+        let Some(val) = value else {
+            return Ok(());
+        };
+        let val = val.to_string();
+        if self
+            .add_sync_item(SyncData::Config { key, val })
+            .await
+            .log_err(self)
+            .is_err()
+        {
+            return Ok(());
+        }
+        self.send_sync_msg().await.log_err(self).ok();
         Ok(())
     }
 
@@ -644,7 +704,7 @@ mod tests {
 
     use super::*;
     use crate::constants;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{sync, TestContext};
 
     #[test]
     fn test_to_string() {
@@ -658,6 +718,21 @@ mod tests {
         );
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_set_config_addr() {
+        let t = TestContext::new().await;
+
+        // Test that uppercase address get lowercased.
+        assert!(t
+            .set_config(Config::Addr, Some("Foobar@eXample.oRg"))
+            .await
+            .is_ok());
+        assert_eq!(
+            t.get_config(Config::Addr).await.unwrap().unwrap(),
+            "foobar@example.org"
+        );
+    }
+
     /// Tests that "bot" config can only be set to "0" or "1".
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_config_bot() {
@@ -773,4 +848,65 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync() -> Result<()> {
+        let alice0 = TestContext::new_alice().await;
+        let alice1 = TestContext::new_alice().await;
+        for a in [&alice0, &alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let mdns_enabled = alice0.get_config_bool(Config::MdnsEnabled).await?;
+        // Alice1 has a different config value.
+        alice1
+            .set_config_bool(Config::MdnsEnabled, !mdns_enabled)
+            .await?;
+        // This changes nothing, but still sends a sync message.
+        alice0
+            .set_config_bool(Config::MdnsEnabled, mdns_enabled)
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config_bool(Config::MdnsEnabled).await?,
+            mdns_enabled
+        );
+
+        // Reset to default. Test that it's not synced because defaults may differ across client
+        // versions.
+        alice0.set_config(Config::MdnsEnabled, None).await?;
+        assert!(alice0.get_config_bool(Config::MdnsEnabled).await?);
+        alice0.set_config_bool(Config::MdnsEnabled, false).await?;
+        sync(&alice0, &alice1).await;
+        assert!(!alice1.get_config_bool(Config::MdnsEnabled).await?);
+
+        let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
+        alice0
+            .set_config_bool(Config::ShowEmails, !show_emails)
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config_bool(Config::ShowEmails).await?,
+            !show_emails
+        );
+
+        // `Config::SyncMsgs` mustn't be synced.
+        alice0.set_config_bool(Config::SyncMsgs, false).await?;
+        alice0.set_config_bool(Config::SyncMsgs, true).await?;
+        alice0.set_config_bool(Config::MdnsEnabled, true).await?;
+        sync(&alice0, &alice1).await;
+        assert!(alice1.get_config_bool(Config::MdnsEnabled).await?);
+
+        // Usual sync scenario.
+        alice0
+            .set_config(Config::Displayname, Some("Alice Sync"))
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config(Config::Displayname).await?,
+            Some("Alice Sync".to_string())
+        );
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -31,10 +31,10 @@ use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
-use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
 use crate::socks::Socks5Config;
 use crate::stock_str;
+use crate::sync::Sync::*;
 use crate::tools::{time, EmailAddress};
 use crate::{chat, e2ee, provider};
 
@@ -133,7 +133,9 @@ async fn on_configure_completed(
             for def in config_defaults {
                 if !context.config_exists(def.key).await? {
                     info!(context, "apply config_defaults {}={}", def.key, def.value);
-                    context.set_config(def.key, Some(def.value)).await?;
+                    context
+                        .set_config_ex(Nosync, def.key, Some(def.value))
+                        .await?;
                 } else {
                     info!(
                         context,
@@ -481,9 +483,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     ctx.set_config_bool(Config::FetchedExistingMsgs, false)
         .await?;
-    ctx.scheduler
-        .interrupt_inbox(InterruptInfo::new(false))
-        .await;
+    ctx.scheduler.interrupt_inbox().await;
 
     progress!(ctx, 940);
     update_device_chats_handle.await??;

src/contact.rs

@@ -19,7 +19,7 @@ use tokio::task;
 use tokio::time::{timeout, Duration};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::ChatId;
+use crate::chat::{ChatId, ProtectionStatus};
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
@@ -30,8 +30,9 @@ use crate::login_param::LoginParam;
 use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
-use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::peerstate::Peerstate;
 use crate::sql::{self, params_iter};
+use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
     EmailAddress,
@@ -42,43 +43,43 @@ use crate::{chat, stock_str};
 const SEEN_RECENTLY_SECONDS: i64 = 600;
 
 /// Valid contact address.
-#[derive(Debug, Clone, Copy)]
-pub(crate) struct ContactAddress<'a>(&'a str);
+#[derive(Debug, Clone)]
+pub(crate) struct ContactAddress(String);
 
-impl Deref for ContactAddress<'_> {
+impl Deref for ContactAddress {
     type Target = str;
 
     fn deref(&self) -> &Self::Target {
-        self.0
+        &self.0
     }
 }
 
-impl AsRef<str> for ContactAddress<'_> {
+impl AsRef<str> for ContactAddress {
     fn as_ref(&self) -> &str {
-        self.0
+        &self.0
     }
 }
 
-impl fmt::Display for ContactAddress<'_> {
+impl fmt::Display for ContactAddress {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", self.0)
     }
 }
 
-impl<'a> ContactAddress<'a> {
+impl ContactAddress {
     /// Constructs a new contact address from string,
     /// normalizing and validating it.
-    pub fn new(s: &'a str) -> Result<Self> {
+    pub fn new(s: &str) -> Result<Self> {
         let addr = addr_normalize(s);
-        if !may_be_valid_addr(addr) {
+        if !may_be_valid_addr(&addr) {
             bail!("invalid address {:?}", s);
         }
-        Ok(Self(addr))
+        Ok(Self(addr.to_string()))
     }
 }
 
 /// Allow converting [`ContactAddress`] to an SQLite type.
-impl rusqlite::types::ToSql for ContactAddress<'_> {
+impl rusqlite::types::ToSql for ContactAddress {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         let val = rusqlite::types::Value::Text(self.0.to_string());
         let out = rusqlite::types::ToSqlOutput::Owned(val);
@@ -148,6 +149,22 @@ impl ContactId {
             .await?;
         Ok(())
     }
+
+    /// Reset gossip timestamp in all chats with this contact.
+    pub(crate) async fn regossip_keys(&self, context: &Context) -> Result<()> {
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+                 SET gossiped_timestamp=0
+                 WHERE EXISTS (SELECT 1 FROM chats_contacts
+                               WHERE chats_contacts.chat_id=chats.id
+                               AND chats_contacts.contact_id=?)",
+                (self,),
+            )
+            .await?;
+        Ok(())
+    }
 }
 
 impl fmt::Display for ContactId {
@@ -331,24 +348,6 @@ pub(crate) enum Modifier {
     Created,
 }
 
-/// Verification status of the contact.
-#[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)]
-#[repr(u8)]
-pub enum VerifiedStatus {
-    /// Contact is not verified.
-    Unverified = 0,
-    /// SELF has verified the fingerprint of a contact. Currently unused.
-    Verified = 1,
-    /// SELF and contact have verified their fingerprints in both directions; in the UI typically checkmarks are shown.
-    BidirectVerified = 2,
-}
-
-impl Default for VerifiedStatus {
-    fn default() -> Self {
-        Self::Unverified
-    }
-}
-
 impl Contact {
     /// Loads a single contact object from the database.
     ///
@@ -459,12 +458,12 @@ impl Contact {
 
     /// Block the given contact.
     pub async fn block(context: &Context, id: ContactId) -> Result<()> {
-        set_block_contact(context, id, true).await
+        set_blocked(context, Sync, id, true).await
     }
 
     /// Unblock the given contact.
     pub async fn unblock(context: &Context, id: ContactId) -> Result<()> {
-        set_block_contact(context, id, false).await
+        set_blocked(context, Sync, id, false).await
     }
 
     /// Add a single contact as a result of an _explicit_ user action.
@@ -483,7 +482,7 @@ impl Contact {
         let addr = ContactAddress::new(&addr)?;
 
         let (contact_id, sth_modified) =
-            Contact::add_or_lookup(context, &name, addr, Origin::ManuallyCreated)
+            Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated)
                 .await
                 .context("add_or_lookup")?;
         let blocked = Contact::is_blocked_load(context, contact_id).await?;
@@ -494,7 +493,7 @@ impl Contact {
             }
         }
         if blocked {
-            Contact::unblock(context, contact_id).await?;
+            set_blocked(context, Nosync, contact_id, false).await?;
         }
 
         Ok(contact_id)
@@ -523,10 +522,24 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `may_be_valid_addr()`.
+    ///
+    /// Returns the contact ID of the contact belonging to the e-mail address or 0 if there is no
+    /// contact that is or was introduced by an accepted contact.
     pub async fn lookup_id_by_addr(
         context: &Context,
         addr: &str,
         min_origin: Origin,
+    ) -> Result<Option<ContactId>> {
+        Self::lookup_id_by_addr_ex(context, addr, min_origin, Some(Blocked::Not)).await
+    }
+
+    /// The same as `lookup_id_by_addr()`, but internal function. Currently also allows looking up
+    /// not unblocked contacts.
+    pub(crate) async fn lookup_id_by_addr_ex(
+        context: &Context,
+        addr: &str,
+        min_origin: Origin,
+        blocked: Option<Blocked>,
     ) -> Result<Option<ContactId>> {
         if addr.is_empty() {
             bail!("lookup_id_by_addr: empty address");
@@ -534,7 +547,7 @@ impl Contact {
 
         let addr_normalized = addr_normalize(addr);
 
-        if context.is_self_addr(addr_normalized).await? {
+        if context.is_self_addr(&addr_normalized).await? {
             return Ok(Some(ContactId::SELF));
         }
 
@@ -543,8 +556,14 @@ impl Contact {
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
-            AND id>?2 AND origin>=?3 AND blocked=0;",
-                (&addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32),
+            AND id>?2 AND origin>=?3 AND (? OR blocked=?)",
+                (
+                    &addr_normalized,
+                    ContactId::LAST_SPECIAL,
+                    min_origin as u32,
+                    blocked.is_none(),
+                    blocked.unwrap_or_default(),
+                ),
             )
             .await?;
         Ok(id)
@@ -578,7 +597,7 @@ impl Contact {
     pub(crate) async fn add_or_lookup(
         context: &Context,
         name: &str,
-        addr: ContactAddress<'_>,
+        addr: &ContactAddress,
         mut origin: Origin,
     ) -> Result<(ContactId, Modifier)> {
         let mut sth_modified = Modifier::None;
@@ -586,7 +605,7 @@ impl Contact {
         ensure!(!addr.is_empty(), "Can not add_or_lookup empty address");
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
-        if context.is_self_addr(&addr).await? {
+        if context.is_self_addr(addr).await? {
             return Ok((ContactId::SELF, sth_modified));
         }
 
@@ -741,7 +760,7 @@ impl Contact {
                             } else {
                                 "".to_string()
                             },
-                            addr,
+                            &addr,
                             origin,
                             if update_authname {
                                 name.to_string()
@@ -753,7 +772,7 @@ impl Contact {
 
                 sth_modified = Modifier::Created;
                 row_id = u32::try_from(transaction.last_insert_rowid())?;
-                info!(context, "added contact id={} addr={}", row_id, &addr);
+                info!(context, "Added contact id={row_id} addr={addr}.");
             }
             Ok(row_id)
         }).await?;
@@ -786,7 +805,7 @@ impl Contact {
             let name = normalize_name(&name);
             match ContactAddress::new(&addr) {
                 Ok(addr) => {
-                    match Contact::add_or_lookup(context, &name, addr, Origin::AddressBook).await {
+                    match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await {
                         Ok((_, modified)) => {
                             if modified != Modifier::None {
                                 modify_cnt += 1
@@ -1018,11 +1037,9 @@ impl Contact {
             let loginparam = LoginParam::load_configured_params(context).await?;
             let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
 
-            if let Some(peerstate) = peerstate.filter(|peerstate| {
-                peerstate
-                    .peek_key(PeerstateVerifiedStatus::Unverified)
-                    .is_some()
-            }) {
+            if let Some(peerstate) =
+                peerstate.filter(|peerstate| peerstate.peek_key(false).is_some())
+            {
                 let stock_message = match peerstate.prefer_encrypt {
                     EncryptPreference::Mutual => stock_str::e2e_preferred(context).await,
                     EncryptPreference::NoPreference => stock_str::e2e_available(context).await,
@@ -1037,11 +1054,11 @@ impl Contact {
                     .fingerprint()
                     .to_string();
                 let fingerprint_other_verified = peerstate
-                    .peek_key(PeerstateVerifiedStatus::BidirectVerified)
+                    .peek_key(true)
                     .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
-                    .peek_key(PeerstateVerifiedStatus::Unverified)
+                    .peek_key(false)
                     .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 if loginparam.addr < peerstate.addr {
@@ -1230,40 +1247,57 @@ impl Contact {
         self.status.as_str()
     }
 
-    /// Check if a contact was verified. E.g. by a secure-join QR code scan
-    /// and if the key has not changed since this verification.
+    /// Returns true if the contact
+    /// can be added to verified chats,
+    /// i.e. has a verified key
+    /// and Autocrypt key matches the verified key.
     ///
-    /// The UI may draw a checkbox or something like that beside verified contacts.
-    pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
+    /// If contact is verified
+    /// UI should display green checkmark after the contact name
+    /// in contact list items and
+    /// in chat member list items.
+    ///
+    /// In contact profile view, us this function only if there is no chat with the contact,
+    /// otherwise use is_chat_protected().
+    /// Use [Self::get_verifier_id] to display the verifier contact
+    /// in the info section of the contact profile.
+    pub async fn is_verified(&self, context: &Context) -> Result<bool> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
         if self.id == ContactId::SELF {
-            return Ok(VerifiedStatus::BidirectVerified);
+            return Ok(true);
         }
 
         if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
             if peerstate.is_using_verified_key() {
-                return Ok(VerifiedStatus::BidirectVerified);
+                return Ok(true);
             }
         }
 
-        Ok(VerifiedStatus::Unverified)
+        Ok(false)
     }
 
-    /// Returns the address that verified the contact.
-    pub async fn get_verifier_addr(&self, context: &Context) -> Result<Option<String>> {
-        Ok(Peerstate::from_addr(context, self.get_addr())
-            .await?
-            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned())))
-    }
-
-    /// Returns the ContactId that verified the contact.
+    /// Returns the `ContactId` that verified the contact.
+    ///
+    /// If the function returns non-zero result,
+    /// display green checkmark in the profile and "Introduced by ..." line
+    /// with the name and address of the contact
+    /// formatted by [Self::get_name_n_addr].
+    ///
+    /// If this function returns a verifier,
+    /// this does not necessarily mean
+    /// you can add the contact to verified chats.
+    /// Use [Self::is_verified] to check
+    /// if a contact can be added to a verified chat instead.
     pub async fn get_verifier_id(&self, context: &Context) -> Result<Option<ContactId>> {
-        let Some(verifier_addr) = self.get_verifier_addr(context).await? else {
+        let Some(verifier_addr) = Peerstate::from_addr(context, self.get_addr())
+            .await?
+            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned()))
+        else {
             return Ok(None);
         };
 
-        if verifier_addr == self.addr {
+        if addr_cmp(&verifier_addr, &self.addr) {
             // Contact is directly verified via QR code.
             return Ok(Some(ContactId::SELF));
         }
@@ -1278,6 +1312,27 @@ impl Contact {
         }
     }
 
+    /// Returns if the contact profile title should display a green checkmark.
+    ///
+    /// This generally should be consistent with the 1:1 chat with the contact
+    /// so 1:1 chat with the contact and the contact profile
+    /// either both display the green checkmark or both don't display a green checkmark.
+    ///
+    /// UI often knows beforehand if a chat exists and can also call
+    /// `chat.is_protected()` (if there is a chat)
+    /// or `contact.is_verified()` (if there is no chat) directly.
+    /// This is often easier and also skips some database calls.
+    pub async fn is_profile_verified(&self, context: &Context) -> Result<bool> {
+        let contact_id = self.id;
+
+        if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
+            Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
+        } else {
+            // 1:1 chat does not exist.
+            Ok(self.is_verified(context).await?)
+        }
+    }
+
     /// Returns the number of real (i.e. non-special) contacts in the database.
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
@@ -1330,12 +1385,13 @@ pub fn may_be_valid_addr(addr: &str) -> bool {
     res.is_ok()
 }
 
-/// Returns address with whitespace trimmed and `mailto:` prefix removed.
-pub fn addr_normalize(addr: &str) -> &str {
-    let norm = addr.trim();
+/// Returns address lowercased,
+/// with whitespace trimmed and `mailto:` prefix removed.
+pub fn addr_normalize(addr: &str) -> String {
+    let norm = addr.trim().to_lowercase();
 
     if norm.starts_with("mailto:") {
-        norm.get(7..).unwrap_or(norm)
+        norm.get(7..).unwrap_or(&norm).to_string()
     } else {
         norm
     }
@@ -1363,8 +1419,9 @@ fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     }
 }
 
-async fn set_block_contact(
+pub(crate) async fn set_blocked(
     context: &Context,
+    sync: sync::Sync,
     contact_id: ContactId,
     new_blocking: bool,
 ) -> Result<()> {
@@ -1373,7 +1430,6 @@ async fn set_block_contact(
         "Can't block special contact {}",
         contact_id
     );
-
     let contact = Contact::get_by_id(context, contact_id).await?;
 
     if contact.blocked != new_blocking {
@@ -1415,9 +1471,23 @@ WHERE type=? AND id IN (
             if let Some((chat_id, _, _)) =
                 chat::get_chat_id_by_grpid(context, &contact.addr).await?
             {
-                chat_id.unblock(context).await?;
+                chat_id.unblock_ex(context, Nosync).await?;
             }
         }
+
+        if sync.into() {
+            let action = match new_blocking {
+                true => chat::SyncAction::Block,
+                false => chat::SyncAction::Unblock,
+            };
+            context
+                .add_sync_item(SyncData::AlterChat {
+                    id: chat::SyncId::ContactAddr(contact.addr.clone()),
+                    action,
+                })
+                .await?;
+            context.send_sync_msg().await?;
+        }
     }
 
     Ok(())
@@ -1576,8 +1646,8 @@ fn cat_fingerprint(
 
 /// Compares two email addresses, normalizing them beforehand.
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
-    let norm1 = addr_normalize(addr1).to_lowercase();
-    let norm2 = addr_normalize(addr2).to_lowercase();
+    let norm1 = addr_normalize(addr1);
+    let norm2 = addr_normalize(addr2);
 
     norm1 == norm2
 }
@@ -1775,10 +1845,7 @@ mod tests {
     fn test_normalize_addr() {
         assert_eq!(addr_normalize("mailto:john@doe.com"), "john@doe.com");
         assert_eq!(addr_normalize("  hello@world.com   "), "hello@world.com");
-
-        // normalisation preserves case to allow user-defined spelling.
-        // however, case is ignored on addr_cmp()
-        assert_ne!(addr_normalize("John@Doe.com"), "john@doe.com");
+        assert_eq!(addr_normalize("John@Doe.com"), "john@doe.com");
     }
 
     #[test]
@@ -1804,7 +1871,7 @@ mod tests {
         let (id, _modified) = Contact::add_or_lookup(
             &context.ctx,
             "bob",
-            ContactAddress::new("user@example.org")?,
+            &ContactAddress::new("user@example.org")?,
             Origin::IncomingReplyTo,
         )
         .await?;
@@ -1832,7 +1899,7 @@ mod tests {
         let (contact_bob_id, modified) = Contact::add_or_lookup(
             &context.ctx,
             "someone",
-            ContactAddress::new("user@example.org")?,
+            &ContactAddress::new("user@example.org")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -1897,7 +1964,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bla foo",
-            ContactAddress::new("one@eins.org").unwrap(),
+            &ContactAddress::new("one@eins.org").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -1916,7 +1983,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "Real one",
-            ContactAddress::new(" one@eins.org  ").unwrap(),
+            &ContactAddress::new(" one@eins.org  ").unwrap(),
             Origin::ManuallyCreated,
         )
         .await
@@ -1932,7 +1999,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "",
-            ContactAddress::new("three@drei.sam").unwrap(),
+            &ContactAddress::new("three@drei.sam").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -1949,7 +2016,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "m. serious",
-            ContactAddress::new("three@drei.sam").unwrap(),
+            &ContactAddress::new("three@drei.sam").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -1964,7 +2031,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "schnucki",
-            ContactAddress::new("three@drei.sam").unwrap(),
+            &ContactAddress::new("three@drei.sam").unwrap(),
             Origin::ManuallyCreated,
         )
         .await
@@ -1980,7 +2047,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "",
-            ContactAddress::new("alice@w.de").unwrap(),
+            &ContactAddress::new("alice@w.de").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -2122,7 +2189,7 @@ mod tests {
         let (contact_id, _) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            ContactAddress::new("bob@example.net")?,
+            &ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2201,7 +2268,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob1",
-            ContactAddress::new("bob@example.org").unwrap(),
+            &ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2217,7 +2284,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob2",
-            ContactAddress::new("bob@example.org").unwrap(),
+            &ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2243,7 +2310,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob4",
-            ContactAddress::new("bob@example.org").unwrap(),
+            &ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2272,7 +2339,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "claire1",
-            ContactAddress::new("claire@example.org").unwrap(),
+            &ContactAddress::new("claire@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2288,7 +2355,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "claire2",
-            ContactAddress::new("claire@example.org").unwrap(),
+            &ContactAddress::new("claire@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2313,7 +2380,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "Bob",
-            ContactAddress::new("bob@example.org")?,
+            &ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownFrom,
         )
         .await?;
@@ -2325,7 +2392,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "Not Bob",
-            ContactAddress::new("bob@example.org")?,
+            &ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownTo,
         )
         .await?;
@@ -2338,7 +2405,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "Bob",
-            ContactAddress::new("bob@example.org")?,
+            &ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownFrom,
         )
         .await?;
@@ -2367,7 +2434,7 @@ mod tests {
         Contact::add_or_lookup(
             &t,
             "dave2",
-            ContactAddress::new("dave@example.org").unwrap(),
+            &ContactAddress::new("dave@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2488,7 +2555,7 @@ mod tests {
         let (contact_bob_id, _modified) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            ContactAddress::new("bob@example.net")?,
+            &ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2651,7 +2718,7 @@ CCCB 5AA9 F6E1 141C 9431
         let (contact_id, _) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            ContactAddress::new("bob@example.net")?,
+            &ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2710,7 +2777,6 @@ Hi."#;
 
         let contact_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
         let contact = Contact::get_by_id(&alice, contact_id).await?;
-        assert!(contact.get_verifier_addr(&alice).await?.is_none());
         assert!(contact.get_verifier_id(&alice).await?.is_none());
 
         // Receive a message from Bob to create a peerstate.
@@ -2719,7 +2785,6 @@ Hi."#;
         alice.recv_msg(&sent_msg).await;
 
         let contact = Contact::get_by_id(&alice, contact_id).await?;
-        assert!(contact.get_verifier_addr(&alice).await?.is_none());
         assert!(contact.get_verifier_id(&alice).await?.is_none());
 
         Ok(())

src/context.rs

@@ -23,7 +23,7 @@ use crate::key::{load_self_public_key, DcKey as _};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
-use crate::scheduler::{InterruptInfo, SchedulerState};
+use crate::scheduler::SchedulerState;
 use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
@@ -398,11 +398,25 @@ impl Context {
     }
 
     /// Starts the IO scheduler.
-    pub async fn start_io(&self) {
-        if let Ok(false) = self.is_configured().await {
+    pub async fn start_io(&mut self) {
+        if !self.is_configured().await.unwrap_or_default() {
             warn!(self, "can not start io on a context that is not configured");
             return;
         }
+
+        {
+            if self
+                .get_config(Config::ConfiguredAddr)
+                .await
+                .unwrap_or_default()
+                .filter(|s| s.ends_with(".testrun.org"))
+                .is_some()
+            {
+                let mut lock = self.ratelimit.write().await;
+                // Allow at least 1 message every second + a burst of 3.
+                *lock = Ratelimit::new(Duration::new(3, 0), 3.0);
+            }
+        }
         self.scheduler.start(self.clone()).await;
     }
 
@@ -424,11 +438,7 @@ impl Context {
 
     pub(crate) async fn schedule_resync(&self) -> Result<()> {
         self.resync_request.store(true, Ordering::Relaxed);
-        self.scheduler
-            .interrupt_inbox(InterruptInfo {
-                probe_network: false,
-            })
-            .await;
+        self.scheduler.interrupt_inbox().await;
         Ok(())
     }
 
@@ -1302,6 +1312,7 @@ mod tests {
             "send_port",
             "send_security",
             "server_flags",
+            "skip_start_messages",
             "smtp_certificate_checks",
             "socks5_host",
             "socks5_port",

src/debug_logging.rs

@@ -54,7 +54,7 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
         match context
             .write_status_update_inner(
                 &msg_id,
-                StatusUpdateItem {
+                &StatusUpdateItem {
                     payload: json!({
                         "event": event,
                         "time": time,
@@ -62,6 +62,7 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                     info: None,
                     summary: None,
                     document: None,
+                    uid: None,
                 },
             )
             .await
@@ -70,10 +71,17 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                 eprintln!("Can't log event to webxdc status update: {err:#}");
             }
             Ok(serial) => {
-                context.emit_event(EventType::WebxdcStatusUpdate {
-                    msg_id,
-                    status_update_serial: serial,
-                });
+                if let Some(serial) = serial {
+                    if !matches!(event, EventType::WebxdcStatusUpdate { .. }) {
+                        context.emit_event(EventType::WebxdcStatusUpdate {
+                            msg_id,
+                            status_update_serial: serial,
+                        });
+                    }
+                } else {
+                    // This should not happen as the update has no `uid`.
+                    error!(context, "Debug logging update is not created.");
+                };
             }
         }
     }

src/decrypt.rs

@@ -23,32 +23,14 @@ use crate::pgp;
 ///
 /// If the message is wrongly signed, HashSet will be empty.
 pub fn try_decrypt(
-    context: &Context,
     mail: &ParsedMail<'_>,
     private_keyring: &[SignedSecretKey],
     public_keyring_for_validate: &[SignedPublicKey],
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let encrypted_data_part = match {
-        let mime = get_autocrypt_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected Autocrypt-mime message.");
-        }
-        mime
-    }
-    .or_else(|| {
-        let mime = get_mixed_up_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected mixed-up mime message.");
-        }
-        mime
-    })
-    .or_else(|| {
-        let mime = get_attachment_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected attached Autocrypt-mime message.");
-        }
-        mime
-    }) {
+    let encrypted_data_part = match get_autocrypt_mime(mail)
+        .or_else(|| get_mixed_up_mime(mail))
+        .or_else(|| get_attachment_mime(mail))
+    {
         None => return Ok(None),
         Some(res) => res,
     };

src/download.rs

@@ -12,18 +12,16 @@ use crate::context::Context;
 use crate::imap::{Imap, ImapActionResult};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
-use crate::scheduler::InterruptInfo;
 use crate::tools::time;
 use crate::{stock_str, EventType};
 
 /// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
 ///
-/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
-/// need to be downloaded completely to handle them correctly,
-/// eg. to assign them to the correct chat.
-/// As these messages are typically small,
-/// they're caught by `MIN_DOWNLOAD_LIMIT`.
-pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN)
+/// should always be downloaded completely to handle them correctly,
+/// also in larger groups and if group and contact avatar are attached.
+/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`.
+pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840;
 
 /// If a message is downloaded only partially
 /// and `delete_server_after` is set to small timeouts (eg. "at once"),
@@ -94,10 +92,7 @@ impl MsgId {
                     .sql
                     .execute("INSERT INTO download (msg_id) VALUES (?)", (self,))
                     .await?;
-                context
-                    .scheduler
-                    .interrupt_inbox(InterruptInfo::new(false))
-                    .await;
+                context.scheduler.interrupt_inbox().await;
             }
         }
         Ok(())

src/e2ee.rs

@@ -7,7 +7,7 @@ use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
 use crate::context::Context;
 use crate::key::{load_self_public_key, load_self_secret_key, SignedPublicKey};
-use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::peerstate::Peerstate;
 use crate::pgp;
 
 #[derive(Debug)]
@@ -62,26 +62,19 @@ impl EncryptHelper {
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    info!(
-                        context,
-                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
-                    );
+                    let prefer_encrypt = peerstate.prefer_encrypt;
+                    info!(context, "Peerstate for {addr:?} is {prefer_encrypt}.");
                     match peerstate.prefer_encrypt {
-                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::NoPreference | EncryptPreference::Reset => {}
                         EncryptPreference::Mutual => prefer_encrypt_count += 1,
-                        EncryptPreference::Reset => {
-                            if !e2ee_guaranteed {
-                                return Ok(false);
-                            }
-                        }
                     };
                 }
                 None => {
-                    let msg = format!("peerstate for {addr:?} missing, cannot encrypt");
+                    let msg = format!("Peerstate for {addr:?} missing, cannot encrypt");
                     if e2ee_guaranteed {
-                        return Err(format_err!("{}", msg));
+                        return Err(format_err!("{msg}"));
                     } else {
-                        info!(context, "{}", msg);
+                        info!(context, "{msg}.");
                         return Ok(false);
                     }
                 }
@@ -99,22 +92,45 @@ impl EncryptHelper {
     pub async fn encrypt(
         self,
         context: &Context,
-        min_verified: PeerstateVerifiedStatus,
+        verified: bool,
         mail_to_encrypt: lettre_email::PartBuilder,
         peerstates: Vec<(Option<Peerstate>, &str)>,
     ) -> Result<String> {
         let mut keyring: Vec<SignedPublicKey> = Vec::new();
 
+        let mut verifier_addresses: Vec<&str> = Vec::new();
+
         for (peerstate, addr) in peerstates
-            .into_iter()
-            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
+            .iter()
+            .filter_map(|(state, addr)| state.clone().map(|s| (s, addr)))
         {
             let key = peerstate
-                .take_key(min_verified)
+                .take_key(verified)
                 .with_context(|| format!("proper enc-key for {addr} missing, cannot encrypt"))?;
             keyring.push(key);
+            verifier_addresses.push(addr);
         }
+
+        // Encrypt to self.
         keyring.push(self.public_key.clone());
+
+        // Encrypt to secondary verified keys
+        // if we also encrypt to the introducer ("verifier") of the key.
+        if verified {
+            for (peerstate, _addr) in peerstates {
+                if let Some(peerstate) = peerstate {
+                    if let (Some(key), Some(verifier)) = (
+                        peerstate.secondary_verified_key.as_ref(),
+                        peerstate.secondary_verifier.as_deref(),
+                    ) {
+                        if verifier_addresses.contains(&verifier) {
+                            keyring.push(key.clone());
+                        }
+                    }
+                }
+            }
+        }
+
         let sign_key = load_self_secret_key(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
@@ -153,11 +169,10 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat;
     use crate::key::DcKey;
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
-    use crate::test_utils::{bob_keypair, TestContext};
+    use crate::test_utils::{bob_keypair, TestContext, TestContextManager};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -201,37 +216,35 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
 
         let chat_alice = alice.create_chat(&bob).await.id;
         let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
         // Bob receives unencrypted message from Alice
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
 
-        // Parsing a message is enough to update peerstate
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
-        // Bob sends encrypted message to Alice
+        // Bob sends empty encrypted message to Alice
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        let sent = bob.pop_sent_msg().await;
+        let sent = bob.send_msg(chat_bob, &mut msg).await;
 
-        // Alice receives encrypted message from Bob
-        let msg = alice.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        // Alice receives an empty encrypted message from Bob.
+        // This is also a regression test for previously existing bug
+        // that resulted in no padlock on encrypted empty messages.
+        let msg = alice.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
 
         let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
             .await?
@@ -243,12 +256,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends encrypted message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -257,12 +268,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -272,12 +281,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -301,8 +308,11 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             gossip_key_fingerprint: Some(pub_key.fingerprint()),
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
         vec![(Some(peerstate), addr)]
     }

src/events/payload.rs

@@ -250,6 +250,7 @@ pub enum EventType {
         /// Progress as:
         /// 400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
         /// (Bob has verified alice and waits until Alice does the same for him)
+        /// 1000=vg-member-added/vc-contact-confirm received
         progress: usize,
     },
 

src/imap.rs

@@ -9,6 +9,7 @@ use std::{
     collections::{BTreeMap, BTreeSet, HashMap},
     iter::Peekable,
     mem::take,
+    time::Duration,
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
@@ -16,6 +17,8 @@ use async_channel::Receiver;
 use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
 use futures::{StreamExt, TryStreamExt};
 use num_traits::FromPrimitive;
+use ratelimit::Ratelimit;
+use tokio::sync::RwLock;
 
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
@@ -35,11 +38,10 @@ use crate::receive_imf::{
     from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner, ReceivedMsg,
 };
 use crate::scheduler::connectivity::ConnectivityStore;
-use crate::scheduler::InterruptInfo;
 use crate::socks::Socks5Config;
 use crate::sql;
 use crate::stock_str;
-use crate::tools::create_id;
+use crate::tools::{create_id, duration_to_str};
 
 pub(crate) mod capabilities;
 mod client;
@@ -86,12 +88,13 @@ const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
 
 #[derive(Debug)]
 pub struct Imap {
-    pub(crate) idle_interrupt_receiver: Receiver<InterruptInfo>,
+    pub(crate) idle_interrupt_receiver: Receiver<()>,
     config: ImapConfig,
     pub(crate) session: Option<Session>,
     login_failed_once: bool,
 
     pub(crate) connectivity: ConnectivityStore,
+    ratelimit: RwLock<Ratelimit>,
 }
 
 #[derive(Debug)]
@@ -195,7 +198,7 @@ impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
 
                 while let Some((next_rowid, next_uid, _)) =
                     self.inner.next_if(|(_, next_uid, next_folder)| {
-                        next_folder == &folder && *next_uid == end_uid + 1
+                        next_folder == &folder && (*next_uid == end_uid + 1 || *next_uid == end_uid)
                     })
                 {
                     end_uid = next_uid;
@@ -228,7 +231,7 @@ impl Imap {
         socks5_config: Option<Socks5Config>,
         addr: &str,
         provider_strict_tls: bool,
-        idle_interrupt_receiver: Receiver<InterruptInfo>,
+        idle_interrupt_receiver: Receiver<()>,
     ) -> Result<Self> {
         if lp.server.is_empty() || lp.user.is_empty() || lp.password.is_empty() {
             bail!("Incomplete IMAP connection parameters");
@@ -253,6 +256,8 @@ impl Imap {
             session: None,
             login_failed_once: false,
             connectivity: Default::default(),
+            // 1 connection per minute + a burst of 2.
+            ratelimit: RwLock::new(Ratelimit::new(Duration::new(120, 0), 2.0)),
         };
 
         Ok(imap)
@@ -261,7 +266,7 @@ impl Imap {
     /// Creates new disconnected IMAP client using configured parameters.
     pub async fn new_configured(
         context: &Context,
-        idle_interrupt_receiver: Receiver<InterruptInfo>,
+        idle_interrupt_receiver: Receiver<()>,
     ) -> Result<Self> {
         if !context.is_configured().await? {
             bail!("IMAP Connect without configured params");
@@ -301,10 +306,20 @@ impl Imap {
         }
 
         self.connectivity.set_connecting(context).await;
+        let ratelimit_duration = self.ratelimit.read().await.until_can_send();
+        if !ratelimit_duration.is_zero() {
+            warn!(
+                context,
+                "IMAP got rate limited, waiting for {} until can connect",
+                duration_to_str(ratelimit_duration),
+            );
+            tokio::time::sleep(ratelimit_duration).await;
+        }
 
         let oauth2 = self.config.lp.oauth2;
 
         info!(context, "Connecting to IMAP server");
+        self.ratelimit.write().await.send();
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
         {
@@ -568,9 +583,11 @@ impl Imap {
         Ok(())
     }
 
-    /// Select a folder and take care of uidvalidity changes.
-    /// Also, when selecting a folder for the first time, sets the uid_next to the current
+    /// Selects a folder and takes care of UIDVALIDITY changes.
+    ///
+    /// When selecting a folder for the first time, sets the uid_next to the current
     /// mailbox.uid_next so that no old emails are fetched.
+    ///
     /// Returns Result<new_emails> (i.e. whether new emails arrived),
     /// if in doubt, returns new_emails=true so emails are fetched.
     pub(crate) async fn select_with_uidvalidity(
@@ -588,10 +605,6 @@ impl Imap {
             .as_mut()
             .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
 
-        let new_uid_validity = mailbox
-            .uid_validity
-            .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
-
         let old_uid_validity = get_uidvalidity(context, folder)
             .await
             .with_context(|| format!("failed to get old UID validity for folder {folder}"))?;
@@ -599,6 +612,45 @@ impl Imap {
             .await
             .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
 
+        let new_uid_validity = mailbox
+            .uid_validity
+            .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
+        let new_uid_next = if let Some(uid_next) = mailbox.uid_next {
+            Some(uid_next)
+        } else {
+            warn!(
+                context,
+                "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
+            );
+
+            // RFC 3501 says STATUS command SHOULD NOT be used
+            // on the currently selected mailbox because the same
+            // information can be obtained by other means,
+            // such as reading SELECT response.
+            //
+            // However, it also says that UIDNEXT is REQUIRED
+            // in the SELECT response and if we are here,
+            // it is actually not returned.
+            //
+            // In particular, Winmail Pro Mail Server 5.1.0616
+            // never returns UIDNEXT in SELECT response,
+            // but responds to "STATUS INBOX (UIDNEXT)" command.
+            let status = session
+                .inner
+                .status(folder, "(UIDNEXT)")
+                .await
+                .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
+
+            if status.uid_next.is_none() {
+                // This happens with mail.163.com as of 2023-11-26.
+                // It does not return UIDNEXT on SELECT and returns invalid
+                // `* STATUS "INBOX" ()` response on explicit request for UIDNEXT.
+                warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT.");
+            }
+            status.uid_next
+        };
+        mailbox.uid_next = new_uid_next;
+
         if new_uid_validity == old_uid_validity {
             let new_emails = if newly_selected == NewlySelected::No {
                 // The folder was not newly selected i.e. no SELECT command was run. This means that mailbox.uid_next
@@ -606,19 +658,21 @@ impl Imap {
                 // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
                 // new messages is only one command, just as a SELECT command)
                 true
-            } else if let Some(uid_next) = mailbox.uid_next {
-                if uid_next < old_uid_next {
+            } else if let Some(new_uid_next) = new_uid_next {
+                if new_uid_next < old_uid_next {
                     warn!(
                         context,
-                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
+                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
                     );
-                    set_uid_next(context, folder, uid_next).await?;
+                    set_uid_next(context, folder, new_uid_next).await?;
                     context.schedule_resync().await?;
                 }
-                uid_next != old_uid_next // If uid_next changed, there are new emails
+                new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
             } else {
-                true // We have no uid_next and if in doubt, return true
+                // We have no UIDNEXT and if in doubt, return true.
+                true
             };
+
             return Ok(new_emails);
         }
 
@@ -627,43 +681,7 @@ impl Imap {
 
         // ==============  uid_validity has changed or is being set the first time.  ==============
 
-        let new_uid_next = match mailbox.uid_next {
-            Some(uid_next) => uid_next,
-            None => {
-                warn!(
-                    context,
-                    "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
-                );
-
-                // RFC 3501 says STATUS command SHOULD NOT be used
-                // on the currently selected mailbox because the same
-                // information can be obtained by other means,
-                // such as reading SELECT response.
-                //
-                // However, it also says that UIDNEXT is REQUIRED
-                // in the SELECT response and if we are here,
-                // it is actually not returned.
-                //
-                // In particular, Winmail Pro Mail Server 5.1.0616
-                // never returns UIDNEXT in SELECT response,
-                // but responds to "SELECT INBOX (UIDNEXT)" command.
-                let status = session
-                    .inner
-                    .status(folder, "(UIDNEXT)")
-                    .await
-                    .context("STATUS (UIDNEXT) error for {folder:?}")?;
-
-                if let Some(uid_next) = status.uid_next {
-                    uid_next
-                } else {
-                    warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT");
-
-                    // Set UIDNEXT to 1 as a last resort fallback.
-                    1
-                }
-            }
-        };
-
+        let new_uid_next = new_uid_next.unwrap_or_default();
         set_uid_next(context, folder, new_uid_next).await?;
         set_uidvalidity(context, folder, new_uid_validity).await?;
 
@@ -867,14 +885,24 @@ impl Imap {
             uids_fetch_in_batch.push(uid);
         }
 
-        // determine which uid_next to use to update to
-        // receive_imf() returns an `Err` value only on recoverable errors, otherwise it just logs an error.
-        // `largest_uid_processed` is the largest uid where receive_imf() did NOT return an error.
-
-        // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was
-        // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times.
-        let largest_uid_without_errors = max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0));
-        let new_uid_next = largest_uid_without_errors + 1;
+        // Advance uid_next to the maximum of the largest known UID plus 1
+        // and mailbox UIDNEXT.
+        // Largest known UID is normally less than UIDNEXT,
+        // but a message may have arrived between determining UIDNEXT
+        // and executing the FETCH command.
+        let mailbox_uid_next = self
+            .session
+            .as_ref()
+            .context("No IMAP session")?
+            .selected_mailbox
+            .as_ref()
+            .with_context(|| format!("Expected {folder:?} to be selected"))?
+            .uid_next
+            .unwrap_or_default();
+        let new_uid_next = max(
+            max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0)) + 1,
+            mailbox_uid_next,
+        );
 
         if new_uid_next > old_uid_next {
             set_uid_next(context, folder, new_uid_next).await?;
@@ -1441,13 +1469,8 @@ impl Imap {
                             break;
                         };
 
-                    let next_fetch_response = match next_fetch_response {
-                        Ok(next_fetch_response) => next_fetch_response,
-                        Err(err) => {
-                            warn!(context, "Failed to process IMAP FETCH result: {}.", err);
-                            continue;
-                        }
-                    };
+                    let next_fetch_response =
+                        next_fetch_response.context("Failed to process IMAP FETCH result")?;
 
                     if let Some(next_uid) = next_fetch_response.uid {
                         if next_uid == request_uid {
@@ -2279,10 +2302,7 @@ pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str)
             (message_id,),
         )
         .await?;
-    context
-        .scheduler
-        .interrupt_inbox(InterruptInfo::new(false))
-        .await;
+    context.scheduler.interrupt_inbox().await;
 
     Ok(())
 }
@@ -2512,7 +2532,7 @@ async fn add_all_recipients_as_contacts(
         let (_, modified) = Contact::add_or_lookup(
             context,
             &display_name_normalized,
-            recipient_addr,
+            &recipient_addr,
             Origin::OutgoingTo,
         )
         .await?;
@@ -2830,4 +2850,31 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_uid_grouper() {
+        // Input: sequence of (rowid: i64, uid: u32, target: String)
+        // Output: sequence of (target: String, rowid_set: Vec<i64>, uid_set: String)
+        let grouper = UidGrouper::from([(1, 2, "INBOX".to_string())]);
+        let res: Vec<(String, Vec<i64>, String)> = grouper.into_iter().collect();
+        assert_eq!(res, vec![("INBOX".to_string(), vec![1], "2".to_string())]);
+
+        let grouper = UidGrouper::from([(1, 2, "INBOX".to_string()), (2, 3, "INBOX".to_string())]);
+        let res: Vec<(String, Vec<i64>, String)> = grouper.into_iter().collect();
+        assert_eq!(
+            res,
+            vec![("INBOX".to_string(), vec![1, 2], "2:3".to_string())]
+        );
+
+        let grouper = UidGrouper::from([
+            (1, 2, "INBOX".to_string()),
+            (2, 2, "INBOX".to_string()),
+            (3, 3, "INBOX".to_string()),
+        ]);
+        let res: Vec<(String, Vec<i64>, String)> = grouper.into_iter().collect();
+        assert_eq!(
+            res,
+            vec![("INBOX".to_string(), vec![1, 2, 3], "2:3".to_string())]
+        );
+    }
 }

src/imap/client.rs

@@ -17,8 +17,8 @@ use crate::net::tls::wrap_tls;
 use crate::socks::Socks5Config;
 use fast_socks5::client::Socks5Stream;
 
-/// IMAP write and read timeout.
-pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(30);
+/// IMAP connection, write and read timeout.
+pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(60);
 
 #[derive(Debug)]
 pub(crate) struct Client {

src/imap/idle.rs

@@ -8,40 +8,36 @@ use futures_lite::FutureExt;
 use super::session::Session;
 use super::Imap;
 use crate::config::Config;
+use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
 use crate::log::LogExt;
-use crate::{context::Context, scheduler::InterruptInfo};
 
-const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
+/// Timeout after which IDLE is finished
+/// if there are no responses from the server.
+///
+/// If `* OK Still here` keepalives are sent more frequently
+/// than this duration, timeout should never be triggered.
+/// For example, Dovecot sends keepalives every 2 minutes by default.
+const IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
 
 impl Session {
     pub async fn idle(
         mut self,
         context: &Context,
-        idle_interrupt_receiver: Receiver<InterruptInfo>,
-        watch_folder: Option<String>,
-    ) -> Result<(Self, InterruptInfo)> {
+        idle_interrupt_receiver: Receiver<()>,
+        folder: &str,
+    ) -> Result<Self> {
         use futures::future::FutureExt;
 
-        if context.get_config_bool(Config::DisableIdle).await? {
-            bail!("IMAP IDLE is disabled");
-        }
-
-        if !self.can_idle() {
-            bail!("IMAP server does not have IDLE capability");
-        }
-
-        let mut info = Default::default();
-
-        self.select_folder(context, watch_folder.as_deref()).await?;
+        self.select_folder(context, Some(folder)).await?;
 
         if self.server_sent_unsolicited_exists(context)? {
-            return Ok((self, info));
+            return Ok(self);
         }
 
-        if let Ok(info) = idle_interrupt_receiver.try_recv() {
-            info!(context, "skip idle, got interrupt {:?}", info);
-            return Ok((self, info));
+        if let Ok(()) = idle_interrupt_receiver.try_recv() {
+            info!(context, "skip idle, got interrupt");
+            return Ok(self);
         }
 
         let mut handle = self.inner.idle();
@@ -58,59 +54,45 @@ impl Session {
 
         enum Event {
             IdleResponse(IdleResponse),
-            Interrupt(InterruptInfo),
+            Interrupt,
         }
 
-        let folder_name = watch_folder.as_deref().unwrap_or("None");
-        info!(
-            context,
-            "{}: Idle entering wait-on-remote state", folder_name
-        );
+        info!(context, "{folder}: Idle entering wait-on-remote state");
         let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
-            let info = idle_interrupt_receiver.recv().await;
+            idle_interrupt_receiver.recv().await.ok();
 
             // cancel imap idle connection properly
             drop(interrupt);
 
-            Ok(Event::Interrupt(info.unwrap_or_default()))
+            Ok(Event::Interrupt)
         });
 
         match fut.await {
             Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
-                info!(context, "{}: Idle has NewData {:?}", folder_name, x);
+                info!(context, "{folder}: Idle has NewData {:?}", x);
             }
             Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                info!(
-                    context,
-                    "{}: Idle-wait timeout or interruption", folder_name
-                );
+                info!(context, "{folder}: Idle-wait timeout or interruption");
             }
             Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                info!(
-                    context,
-                    "{}: Idle wait was interrupted manually", folder_name
-                );
+                info!(context, "{folder}: Idle wait was interrupted manually");
             }
-            Ok(Event::Interrupt(i)) => {
-                info!(
-                    context,
-                    "{}: Idle wait was interrupted: {:?}", folder_name, &i
-                );
-                info = i;
+            Ok(Event::Interrupt) => {
+                info!(context, "{folder}: Idle wait was interrupted");
             }
             Err(err) => {
-                warn!(context, "{}: Idle wait errored: {:?}", folder_name, err);
+                warn!(context, "{folder}: Idle wait errored: {err:?}");
             }
         }
 
         let mut session = tokio::time::timeout(Duration::from_secs(15), handle.done())
             .await
-            .with_context(|| format!("{folder_name}: IMAP IDLE protocol timed out"))?
-            .with_context(|| format!("{folder_name}: IMAP IDLE failed"))?;
+            .with_context(|| format!("{folder}: IMAP IDLE protocol timed out"))?
+            .with_context(|| format!("{folder}: IMAP IDLE failed"))?;
         session.as_mut().set_read_timeout(Some(IMAP_TIMEOUT));
         self.inner = session;
 
-        Ok((self, info))
+        Ok(self)
     }
 }
 
@@ -120,7 +102,7 @@ impl Imap {
         context: &Context,
         watch_folder: Option<String>,
         folder_meaning: FolderMeaning,
-    ) -> InterruptInfo {
+    ) {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
@@ -131,32 +113,33 @@ impl Imap {
             watch_folder
         } else {
             info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
-            return self
-                .idle_interrupt_receiver
-                .recv()
-                .await
-                .unwrap_or_default();
+            self.idle_interrupt_receiver.recv().await.ok();
+            return;
         };
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-        // check every minute if there are new messages
-        // TODO: grow sleep durations / make them more flexible
-        let mut interval = tokio::time::interval(Duration::from_secs(60));
-
+        const TIMEOUT_INIT_MS: u64 = 60_000;
+        let mut timeout_ms: u64 = TIMEOUT_INIT_MS;
         enum Event {
             Tick,
-            Interrupt(InterruptInfo),
+            Interrupt,
         }
         // loop until we are interrupted or if we fetched something
-        let info = loop {
+        loop {
             use futures::future::FutureExt;
+            use rand::Rng;
+
+            let mut interval = tokio::time::interval(Duration::from_millis(timeout_ms));
+            timeout_ms = timeout_ms
+                .saturating_add(rand::thread_rng().gen_range((timeout_ms / 2)..=timeout_ms));
+            interval.tick().await; // The first tick completes immediately.
             match interval
                 .tick()
                 .map(|_| Event::Tick)
                 .race(
                     self.idle_interrupt_receiver
                         .recv()
-                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                        .map(|_| Event::Interrupt),
                 )
                 .await
             {
@@ -178,7 +161,7 @@ impl Imap {
                                 .unwrap_or_default()
                         {
                             // we only fake-idled because network was gone during IDLE, probably
-                            break InterruptInfo::new(false);
+                            break;
                         }
                     }
                     info!(context, "fake_idle is connected");
@@ -192,8 +175,9 @@ impl Imap {
                     {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
+                            timeout_ms = TIMEOUT_INIT_MS;
                             if res {
-                                break InterruptInfo::new(false);
+                                break;
                             }
                         }
                         Err(err) => {
@@ -202,13 +186,12 @@ impl Imap {
                         }
                     }
                 }
-                Event::Interrupt(info) => {
-                    // Interrupt
+                Event::Interrupt => {
                     info!(context, "Fake IDLE interrupted");
-                    break info;
+                    break;
                 }
             }
-        };
+        }
 
         info!(
             context,
@@ -219,7 +202,5 @@ impl Imap {
                 .as_millis() as f64
                 / 1000.,
         );
-
-        info
     }
 }