chore(release): prepare for 1.131.4

This was introduced in https://github.com/deltachat/deltachat-core-rust/pull/4685
for internal use when looking up a chat based on In-Reply-To,
but actually affects the UIs as they should not display Download button
when the message is downloaded but cannot be decrypted.

.github/workflows/ci.yml

@@ -130,7 +130,7 @@ jobs:
     name: Build deltachat-rpc-server
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -145,7 +145,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
-          path: target/debug/deltachat-rpc-server
+          path: ${{ matrix.os == 'windows-latest' && 'target/debug/deltachat-rpc-server.exe' || 'target/debug/deltachat-rpc-server' }}
           retention-days: 1
 
   python_lint:
@@ -211,7 +211,7 @@ jobs:
 
       - name: Run python tests
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
           DCC_RS_TARGET: debug
           DCC_RS_DEV: ${{ github.workspace }}
         working-directory: python
@@ -228,6 +228,8 @@ jobs:
             python: 3.12
           - os: macos-latest
             python: 3.12
+          - os: windows-latest
+            python: 3.12
 
           # PyPy tests
           - os: ubuntu-latest
@@ -258,13 +260,20 @@ jobs:
           path: target/debug
 
       - name: Make deltachat-rpc-server executable
+        if: ${{ matrix.os != 'windows-latest' }}
         run: chmod +x target/debug/deltachat-rpc-server
 
       - name: Add deltachat-rpc-server to path
+        if: ${{ matrix.os != 'windows-latest' }}
         run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
 
+      - name: Add deltachat-rpc-server to path
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          "${{ github.workspace }}/target/debug" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
       - name: Run deltachat-rpc-client tests
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
         working-directory: deltachat-rpc-client
         run: tox -e py

.github/workflows/jsonrpc-client-npm-package.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/jsonrpc.yml

@@ -15,10 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
       - name: Add Rust cache
         uses: Swatinem/rust-cache@v2
       - name: npm install
@@ -31,7 +31,7 @@ jobs:
         working-directory: deltachat-jsonrpc/typescript
         run: npm run test
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
       - name: make sure websocket server version still builds
         working-directory: deltachat-jsonrpc
         run: cargo build --bin deltachat-jsonrpc-server --features webserver

.github/workflows/node-docs.yml

@@ -16,10 +16,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
 
       - name: npm install and generate documentation
         working-directory: node

.github/workflows/node-package.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: System info
         run: |
           rustc -vV
@@ -66,10 +66,8 @@ jobs:
     runs-on: ubuntu-latest
 
     # Build Linux prebuilds inside a container with old glibc for backwards compatibility.
-    # Debian 10 contained glibc 2.28 at the time of the writing (2023-06-04): https://packages.debian.org/buster/libc6
-    # Ubuntu 18.04 is at the End of Standard Support since June 2023, but it contains glibc 2.27,
-    # so we are using it to support Ubuntu 18.04 setups that are still not upgraded.
-    container: ubuntu:18.04
+    # Debian 10 contained glibc 2.28: https://packages.debian.org/buster/libc6
+    container: debian:10
     steps:
       # Working directory is owned by 1001:1001 by default.
       # Change it to our user.
@@ -80,7 +78,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - run: apt-get update
 
       # Python is needed for node-gyp
@@ -143,7 +141,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v2
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/node-tests.yml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: System info
         run: |
           rustc -vV
@@ -63,5 +63,5 @@ jobs:
         working-directory: node
         run: npm run test
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
           NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

CHANGELOG.md

@@ -1,5 +1,246 @@
 # Changelog
 
+## [1.131.4] - 2023-11-16
+
+### Documentation
+
+- Document DC_DOWNLOAD_UNDECIPHERABLE.
+
+### Fixes
+
+- Always add "Member added" as system message.
+
+## [1.131.3] - 2023-11-15
+
+### Fixes
+
+- Update async-imap to 0.9.4 which does not ignore EOF on FETCH.
+- Reset gossiped timestamp on securejoin.
+- sync: Ignore unknown sync items to provide forward compatibility and avoid creating empty message bubbles.
+- sync: Skip sync when chat name is set to the current one.
+- Return connectivity HTML with an error when IO is stopped.
+
+## [1.131.2] - 2023-11-14
+
+### API-Changes
+
+- deltachat-rpc-client: add `Account.get_chat_by_contact()`.
+
+### Features / Changes
+
+- Do not post "... verified" messages on QR scan success.
+- Never drop better message from `apply_group_changes()`.
+
+### Fixes
+
+- Assign MDNs to the trash chat early to prevent received MDNs from creating or unblocking 1:1 chats.
+- Allow to securejoin groups when 1:1 chat with the inviter is a contact request.
+- Add "setup changed" message for verified key before the message.
+- Ignore special chats when calculating similar chats.
+
+## [1.131.1] - 2023-11-13
+
+### Fixes
+
+- Do not skip actual message parts when group change messages are inserted.
+
+## [1.131.0] - 2023-11-13
+
+### Features / Changes
+
+- Sync chat contacts across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Sync creating broadcast lists across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Sync Chat::name across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Multi-device broadcast lists ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+
+### Fixes
+
+- Encode chat name in the `List-ID` header to avoid SMTPUTF8 errors.
+- Ignore errors from generating sync messages.
+- `Context::execute_sync_items`: Ignore all errors ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+- Allow to send unverified securejoin messages to protected chats ([#4982](https://github.com/deltachat/deltachat-core-rust/pull/4982)).
+
+## [1.130.0] - 2023-11-10
+
+### API-Changes
+
+- Emit JoinerProgress(1000) event when Bob verifies Alice.
+- JSON-RPC: add `ContactObject.is_profile_verified` property.
+- Hide `ChatId::get_for_contact()` from public API.
+
+### Features / Changes
+
+- Add secondary verified key.
+- Add info messages about implicitly added members.
+- Treat reset state as encryption not preferred.
+- Grow sleep durations on errors in Imap::fake_idle() ([#4424](https://github.com/deltachat/deltachat-core-rust/pull/4424)).
+
+### Fixes
+
+- Mark 1:1 chat as protected when joining a group.
+- Raise lower auto-download limit to 160k.
+- Remove `Reporting-UA` from read receipts.
+- Do not apply group changes to special chats. Avoid adding members to the trash chat.
+- imap: make `UidGrouper` robust against duplicate UIDs.
+- Do not return hidden chat from `dc_get_chat_id_by_contact_id`.
+- Smtp_loop(): Don't grow timeout if interrupted early ([#4833](https://github.com/deltachat/deltachat-core-rust/pull/4833)).
+
+### Refactor
+
+- imap: Do not FETCH right after `scan_folders()`.
+- deltachat-rpc-client: Use `itertools` instead of `Lock` for thread-safe request ID generation.
+
+### Tests
+
+- Remove unused `--liveconfig` option.
+- Test chatlist can load for corrupted chats ([#4979](https://github.com/deltachat/deltachat-core-rust/pull/4979)).
+
+### Miscellaneous Tasks
+
+- Update provider-db ([#4949](https://github.com/deltachat/deltachat-core-rust/pull/4949)).
+
+## [1.129.1] - 2023-11-06
+
+### Fixes
+
+- Update tokio-imap to fix Outlook STATUS parsing bug.
+- deltachat-rpc-client: Add the Lock around request ID.
+- `apply_group_changes`: Don't implicitly delete members locally, add absent ones instead ([#4934](https://github.com/deltachat/deltachat-core-rust/pull/4934)).
+- Partial messages do not change group state ([#4900](https://github.com/deltachat/deltachat-core-rust/pull/4900)).
+
+### Tests
+
+- Group chats device synchronisation.
+
+## [1.129.0] - 2023-11-06
+
+### API-Changes
+
+- Add JSON-RPC `get_chat_id_by_contact_id` API ([#4918](https://github.com/deltachat/deltachat-core-rust/pull/4918)).
+- [**breaking**] Remove deprecated `get_verifier_addr`.
+
+### Features / Changes
+
+- Sync chat `Blocked` state, chat visibility, chat mute duration and contact blocked status across devices ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+- Add 'group created instructions' as info message ([#4916](https://github.com/deltachat/deltachat-core-rust/pull/4916)).
+- Add hardcoded fallback DNS cache.
+
+### Fixes
+
+- Switch to `EncryptionPreference::Mutual` on a receipt of encrypted+signed message ([#4707](https://github.com/deltachat/deltachat-core-rust/pull/4707)).
+- imap: Check UIDNEXT with a STATUS command before going IDLE.
+- Allow to change verified key via "member added" message.
+- json-rpc: Return verifier even if the contact is not "verified" (Autocrypt key does not equal Secure-Join key).
+
+### Documentation
+
+- Refine `Contact::get_verifier_id` and `Contact::is_verified` documentation ([#4922](https://github.com/deltachat/deltachat-core-rust/pull/4922)).
+- Contact profile view should not use `dc_contact_is_verified()`.
+- Remove documentation for non-existing `dc_accounts_new` `os_name` param.
+
+### Refactor
+
+- Remove unused or useless code paths in Secure-Join ([#4897](https://github.com/deltachat/deltachat-core-rust/pull/4897)).
+- Improve error handling in Secure-Join code.
+- Add hostname to "no DNS resolution results" error message.
+- Accept `&str` instead of `Option<String>` in idle().
+
+## [1.128.0] - 2023-11-02
+
+### Build system
+- [**breaking**] Upgrade nodejs version to 18 ([#4903](https://github.com/deltachat/deltachat-core-rust/pull/4903)).
+
+### Features / Changes
+
+- deltachat-rpc-client: Add `Account.wait_for_incoming_msg_event()`.
+- Decrease ratelimit for .testrun.org subdomains.
+
+### Fixes
+
+- Do not fail securejoin due to unrelated pending bobstate  ([#4896](https://github.com/deltachat/deltachat-core-rust/pull/4896)).
+- Allow other verified group recipients to be unverified, only check the sender verification.
+- Remove not working attempt to recover from verified key changes.
+
+## [1.127.2] - 2023-10-29
+
+### API-Changes
+
+- [**breaking**] Jsonrpc `misc_set_draft` now requires setting the viewtype.
+- jsonrpc: Add `get_message_info_object`.
+
+### Tests
+
+- deltachat-rpc-client: Move pytest option from pyproject.toml to tox.ini and set log level.
+- deltachat-rpc-client: Test securejoin.
+- Increase pytest timeout to 10 minutes.
+- Compile deltachat-rpc-server in debug mode for tests.
+
+## [1.127.1] - 2023-10-27
+
+### API-Changes
+
+- jsonrpc: add `.is_protection_broken` to `FullChat` and `BasicChat`.
+- jsonrpc: Add `id` to `ProviderInfo`.
+
+## [1.127.0] - 2023-10-26
+
+### API-Changes
+
+- [**breaking**] `dc_accounts_new` API is changed. Unused `os_name` argument is removed and `writable` argument is added.
+- jsonrpc: Add `resend_messages`.
+- [**breaking**] Remove unused function `is_verified_ex()` ([#4551](https://github.com/deltachat/deltachat-core-rust/pull/4551))
+- [**breaking**] Make `MsgId.delete_from_db()` private.
+- [**breaking**] deltachat-jsonrpc: use `kind` as a tag for all union types
+- json-rpc: Force stickers to be sent as stickers ([#4819](https://github.com/deltachat/deltachat-core-rust/pull/4819)).
+- Add mailto parse api ([#4829](https://github.com/deltachat/deltachat-core-rust/pull/4829)).
+- [**breaking**] Remove unused `DC_STR_PROTECTION_(EN)ABLED` strings
+- [**breaking**] Remove unused `dc_set_chat_protection()`
+- Hide `DcSecretKey` trait from the API.
+- Verified 1:1 chats ([#4315](https://github.com/deltachat/deltachat-core-rust/pull/4315)). Disabled by default, enable with `verified_one_on_one_chats` config.
+- Add api `chat::Chat::is_protection_broken`
+- Add `dc_chat_is_protection_broken()` C API.
+
+### CI
+
+- Run Rust tests with `RUST_BACKTRACE` set.
+- Replace `master` branch with `main`.  Run CI only on `main` branch pushes.
+- Test `deltachat-rpc-client` on Windows.
+
+### Documentation
+
+- Document how logs and error messages should be formatted in `CONTRIBUTING.md`.
+- Clarify transitive behaviour of `dc_contact_is_verfified()`.
+- Document `configured_addr`.
+
+### Features / Changes
+
+- Add lockfile to account manager ([#4314](https://github.com/deltachat/deltachat-core-rust/pull/4314)). 
+- Don't show a contact as verified if their key changed since the verification ([#4574](https://github.com/deltachat/deltachat-core-rust/pull/4574)).
+- deltachat-rpc-server: Add `--openrpc` option to print OpenRPC specification for JSON-RPC API. This specification can be used to generate JSON-RPC API clients.
+- Track whether contact is a bot or not ([#4821](https://github.com/deltachat/deltachat-core-rust/pull/4821)).
+- Replace `Config::SendSyncMsgs` with `SyncMsgs` ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+
+### Fixes
+
+- Don't create 1:1 chat as protected for contact who doesn't prefer to encrypt ([#4538](https://github.com/deltachat/deltachat-core-rust/pull/4538)).
+- Allow to save a draft if the verification is broken ([#4542](https://github.com/deltachat/deltachat-core-rust/pull/4542)).
+- Fix info-message orderings of verified 1:1 chats ([#4545](https://github.com/deltachat/deltachat-core-rust/pull/4545)).
+- Fix example; this was changed some time ago, see https://docs.webxdc.org/spec.html#sendupdate
+- `receive_imf`: Update peerstate from db after handling Securejoin handshake ([#4600](https://github.com/deltachat/deltachat-core-rust/pull/4600)).
+- Sort old incoming messages below all outgoing ones ([#4621](https://github.com/deltachat/deltachat-core-rust/pull/4621)).
+- Do not mark non-verified group chats as verified when using securejoin.
+- `receive_imf`: Set protection only for Chattype::Single ([#4597](https://github.com/deltachat/deltachat-core-rust/pull/4597)).
+- Return from `dc_get_chatlist(DC_GCL_FOR_FORWARDING)` only chats where we can send ([#4616](https://github.com/deltachat/deltachat-core-rust/pull/4616)).
+- Clear VerifiedOneOnOneChats config on backup ([#4615](https://github.com/deltachat/deltachat-core-rust/pull/4615)).
+- Try removal of accounts multiple times with timeouts in case the database file is blocked (restore `try_many_times` workaround).
+
+### Build system
+
+- Remove examples/simple.rs.
+- Increase MSRV to 1.70.0.
+- Update dependencies.
+- Switch to iroh 0.4.x fork with updated dependencies.
+
 ## [1.126.1] - 2023-10-24
 
 ### Fixes
@@ -1014,7 +1255,7 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 ### Changes
 - Look at Authentication-Results. Don't accept Autocrypt key changes
-  if they come with negative authentiation results while this contact
+  if they come with negative authentication results while this contact
   sent emails with positive authentication results in the past. #3583
 - jsonrpc in cffi also sends events now #3662
 - jsonrpc: new format for events and better typescript autocompletion
@@ -2600,7 +2841,7 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 - delete all consumed secure-join handshake messagess #1209 #1212
 
-- rust-level cleanups #1218 #1217 #1210 #1205
+- Rust-level cleanups #1218 #1217 #1210 #1205
 
 - python-level cleanups #1204 #1202 #1201
 
@@ -2967,3 +3208,15 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.124.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.0...v1.124.1
 [1.125.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.1...v1.125.0
 [1.126.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.125.0...v1.126.0
+[1.126.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.0...v1.126.1
+[1.127.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.1...v1.127.0
+[1.127.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.0...v1.127.1
+[1.127.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.1...v1.127.2
+[1.128.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.2...v1.128.0
+[1.129.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.128.0...v1.129.0
+[1.129.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.0...v1.129.1
+[1.130.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.1...v1.130.0
+[1.131.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.130.0...v1.131.0
+[1.131.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.0...v1.131.1
+[1.131.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.1...v1.131.2
+[1.131.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.2...v1.131.3

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.126.1"
+version = "1.131.4"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.70"
@@ -26,17 +26,13 @@ opt-level = "z"
 codegen-units = 1
 strip = true
 
-[patch.crates-io]
-quinn-udp = { git = "https://github.com/quinn-rs/quinn", branch="main" }
-quinn-proto = { git = "https://github.com/quinn-rs/quinn", branch="main" }
-
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
-async-channel = "1.8.0"
+async-channel = "2.0.0"
 async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
@@ -56,7 +52,7 @@ hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
 image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh = { version = "0.4.1", default-features = false }
+iroh = { git = "https://github.com/deltachat/iroh", branch = "0.4-update-quic", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -69,13 +65,14 @@ once_cell = "1.18.0"
 percent-encoding = "2.3"
 parking_lot = "0.12"
 pgp = { version = "0.10", default-features = false }
+pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
 quick-xml = "0.31"
 rand = "0.8"
 regex = "1.9"
 reqwest = { version = "0.11.20", features = ["json"] }
-rusqlite = { version = "0.29", features = ["sqlcipher"] }
+rusqlite = { version = "0.30", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
 serde_json = "1.0"
@@ -93,7 +90,7 @@ tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 tokio-util = "0.7.9"
-toml = "0.7"
+toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.126.1"
+version = "1.131.4"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -2941,7 +2941,6 @@ int dc_receive_backup (dc_context_t* context, const char* qr);
  * use dc_accounts_remove_account().
  *
  * @memberof dc_accounts_t
- * @param os_name
  * @param dir The directory to create the context-databases in.
  *     If the directory does not exist,
  *     dc_accounts_new() will try to create it.
@@ -3732,8 +3731,22 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
 
 /**
  * Check if a chat is protected.
- * Protected chats contain only verified members and encryption is always enabled.
- * Protected chats are created using dc_create_group_chat() by setting the 'protect' parameter to 1.
+ *
+ * End-to-end encryption is guaranteed in protected chats
+ * and only verified contacts
+ * as determined by dc_contact_is_verified()
+ * can be added to protected chats.
+ *
+ * Protected chats are created using dc_create_group_chat()
+ * by setting the 'protect' parameter to 1.
+ * 1:1 chats become protected or unprotected automatically
+ * if `verified_one_on_one_chats` setting is enabled.
+ *
+ * UI should display a green checkmark
+ * in the chat title,
+ * in the chatlist item
+ * and in the chat profile
+ * if chat protection is enabled.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -4567,15 +4580,18 @@ int dc_msg_has_html (dc_msg_t* msg);
   * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
   *
   * The function returns one of:
-  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
-  *                                  and should be rendered as usual.
-  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
-  *                                  In addition to the usual message rendering,
-  *                                  the UI shall show a download button that calls dc_download_full_msg()
-  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
-  *                                  If the download fails or succeeds,
-  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
-  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
+  * - @ref DC_DOWNLOAD_DONE           - The message does not need any further download action
+  *                                     and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE      - There is additional content to download.
+  *                                     In addition to the usual message rendering,
+  *                                     the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS    - Download was started with dc_download_full_msg() and is still in progress.
+  *                                     If the download fails or succeeds,
+  *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  *
+  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
+  *                                     It was fully downloaded, but we failed to decrypt it.
+  * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
   *
   * @memberof dc_msg_t
   * @param msg The message object.
@@ -5033,10 +5049,16 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 
 
 /**
- * Check if a contact was verified. E.g. by a secure-join QR code scan
- * and if the key has not changed since this verification.
+ * Check if the contact
+ * can be added to verified chats,
+ * i.e. has a verified key
+ * and Autocrypt key matches the verified key.
  *
- * The UI may draw a checkbox or something like that beside verified contacts.
+ * If contact is verified
+ * UI should display green checkmark after the contact name
+ * in contact list items,
+ * in chat member list items
+ * and in profiles if no chat with the contact exist (otherwise, use dc_chat_is_protected()).
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5046,32 +5068,19 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
-
 /**
- * Return the address that verified a contact
+ * Return the contact ID that verified a contact.
  *
- * The UI may use this in addition to a checkmark showing the verification status.
- * In case of verification chains,
- * the last contact in the chain is shown.
- * This is because of privacy reasons, but also as it would not help the user
- * to see a unknown name here - where one can mostly always ask the shown name
- * as it is directly known.
+ * If the function returns non-zero result,
+ * display green checkmark in the profile and "Introduced by ..." line
+ * with the name and address of the contact
+ * formatted by dc_contact_get_name_n_addr.
  *
- * @memberof dc_contact_t
- * @param contact The contact object.
- * @return 
- *    A string containing the verifiers address. If it is the same address as the contact itself,
- *    we verified the contact ourself. If it is an empty string, we don't have verifier 
- *    information or the contact is not verified.
- * @deprecated 2023-09-28, use dc_contact_get_verifier_id instead
- */
-char*           dc_contact_get_verifier_addr       (dc_contact_t* contact);
-
-
-/**
- * Return the `ContactId` that verified a contact
- *
- * The UI may use this in addition to a checkmark showing the verification status
+ * If this function returns a verifier,
+ * this does not necessarily mean
+ * you can add the contact to verified chats.
+ * Use dc_contact_is_verified() to check
+ * if a contact can be added to a verified chat instead.
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -6234,6 +6243,7 @@ void dc_event_unref(dc_event_t* event);
  * @param data2 (int) The progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
+ *     1000=vg-member-added/vc-contact-confirm received
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
@@ -6426,22 +6436,27 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Download not needed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_DONE          0
+#define DC_DOWNLOAD_DONE           0
 
 /**
  * Download available, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_AVAILABLE    10
+#define DC_DOWNLOAD_AVAILABLE      10
 
 /**
  * Download failed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_FAILURE      20
+#define DC_DOWNLOAD_FAILURE        20
+
+/**
+ * Download not needed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_UNDECIPHERABLE 30
 
 /**
  * Download in progress, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_IN_PROGRESS  1000
+#define DC_DOWNLOAD_IN_PROGRESS    1000
 
 
 
@@ -7260,6 +7275,11 @@ void dc_event_unref(dc_event_t* event);
 /// Used in info messages.
 #define DC_STR_CHAT_PROTECTION_DISABLED 171
 
+/// "Others will only see this group after you sent a first message."
+///
+/// Used as the first info messages in newly created groups.
+#define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -489,7 +489,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     if context.is_null() {
         return;
     }
-    let ctx = &*context;
+    let ctx = &mut *context;
 
     block_on(ctx.start_io())
 }
@@ -4119,23 +4119,6 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
         .unwrap_or_default() as libc::c_int
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_verifier_addr(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_verifier_addr()");
-        return "".strdup();
-    }
-    let ffi_contact = &*contact;
-    let ctx = &*ffi_contact.context;
-    block_on(ffi_contact.contact.get_verifier_addr(ctx))
-        .context("failed to get verifier for contact")
-        .log_err(ctx)
-        .unwrap_or_default()
-        .strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) -> u32 {
     if contact.is_null() {
@@ -4946,8 +4929,8 @@ pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
         return;
     }
 
-    let accounts = &*accounts;
-    block_on(async move { accounts.read().await.start_io().await });
+    let accounts = &mut *accounts;
+    block_on(async move { accounts.write().await.start_io().await });
 }
 
 #[no_mangle]

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.126.1"
+version = "1.131.4"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -19,7 +19,7 @@ schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
 tempfile = "3.8.0"
 log = "0.4"
-async-channel = { version = "1.8.0" }
+async-channel = { version = "2.0.0" }
 futures = { version = "0.3.28" }
 serde_json = "1.0.105"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }

deltachat-jsonrpc/README.md

@@ -108,10 +108,10 @@ This will build the `deltachat-jsonrpc-server` binary and then run a test suite
 
 The test suite includes some tests that need online connectivity and a way to create test email accounts. To run these tests, talk to DeltaChat developers to get a token for the `testrun.org` service, or use a local instance of [`mailadm`](https://github.com/deltachat/docker-mailadm).
 
-Then, set the `DCC_NEW_TMP_EMAIL` environment variable to your mailadm token before running the tests.
+Then, set the `CHATMAIL_DOMAIN` environment variable to your testing email server domain.
 
 ```
-DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=yourtoken npm run test
+CHATMAIL_DOMAIN=chat.example.org npm run test
 ```
 
 #### Test Coverage

deltachat-jsonrpc/src/api.rs

@@ -47,7 +47,7 @@ use types::provider_info::ProviderInfo;
 use types::reactions::JSONRPCReactions;
 use types::webxdc::WebxdcMessageInfo;
 
-use self::types::message::MessageLoadResult;
+use self::types::message::{MessageInfo, MessageLoadResult};
 use self::types::{
     chat::{BasicChat, JSONRPCChatVisibility, MuteDuration},
     location::JsonrpcLocation,
@@ -221,13 +221,13 @@ impl CommandApi {
 
     /// Starts background tasks for all accounts.
     async fn start_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.read().await.start_io().await;
+        self.accounts.write().await.start_io().await;
         Ok(())
     }
 
     /// Stops background tasks for all accounts.
     async fn stop_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.read().await.stop_io().await;
+        self.accounts.write().await.stop_io().await;
         Ok(())
     }
 
@@ -237,7 +237,7 @@ impl CommandApi {
 
     /// Starts background tasks for a single account.
     async fn start_io(&self, account_id: u32) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
+        let mut ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
@@ -383,7 +383,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
+        let mut ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -1126,6 +1126,16 @@ impl CommandApi {
         MsgId::new(message_id).get_info(&ctx).await
     }
 
+    /// Returns additional information for single message.
+    async fn get_message_info_object(
+        &self,
+        account_id: u32,
+        message_id: u32,
+    ) -> Result<MessageInfo> {
+        let ctx = self.get_context(account_id).await?;
+        MessageInfo::from_msg_id(&ctx, MsgId::new(message_id)).await
+    }
+
     /// Returns contacts that sent read receipts and the time of reading.
     async fn get_message_read_receipts(
         &self,
@@ -1381,6 +1391,19 @@ impl CommandApi {
     //                   chat
     // ---------------------------------------------
 
+    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id` if it exists.
+    ///
+    /// If it does not exist, `None` is returned.
+    async fn get_chat_id_by_contact_id(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<Option<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let chat_id = ChatId::lookup_by_contact(&ctx, ContactId::new(contact_id)).await?;
+        Ok(chat_id.map(|id| id.to_u32()))
+    }
+
     /// Returns all message IDs of the given types in a chat.
     /// Typically used to show a gallery.
     ///
@@ -2033,13 +2056,19 @@ impl CommandApi {
         text: Option<String>,
         file: Option<String>,
         quoted_message_id: Option<u32>,
+        view_type: Option<MessageViewtype>,
     ) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        let mut draft = Message::new(if file.is_some() {
-            Viewtype::File
-        } else {
-            Viewtype::Text
-        });
+        let mut draft = Message::new(view_type.map_or_else(
+            || {
+                if file.is_some() {
+                    Viewtype::File
+                } else {
+                    Viewtype::Text
+                }
+            },
+            |v| v.into(),
+        ));
         draft.set_text(text.unwrap_or_default());
         if let Some(file) = file {
             draft.set_file(file, None);

deltachat-jsonrpc/src/api/types/chat.rs

@@ -18,6 +18,17 @@ use super::contact::ContactObject;
 pub struct FullChat {
     id: u32,
     name: String,
+
+    /// True if the chat is protected.
+    ///
+    /// UI should display a green checkmark
+    /// in the chat title,
+    /// in the chat profile title and
+    /// in the chatlist item
+    /// if chat protection is enabled.
+    /// UI should also display a green checkmark
+    /// in the contact profile
+    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -31,6 +42,7 @@ pub struct FullChat {
     fresh_message_counter: usize,
     // is_group - please check over chat.type in frontend instead
     is_contact_request: bool,
+    is_protection_broken: bool,
     is_device_chat: bool,
     self_in_group: bool,
     is_muted: bool,
@@ -100,6 +112,7 @@ impl FullChat {
             color,
             fresh_message_counter,
             is_contact_request: chat.is_contact_request(),
+            is_protection_broken: chat.is_protection_broken(),
             is_device_chat: chat.is_device_talk(),
             self_in_group: contact_ids.contains(&ContactId::SELF),
             is_muted: chat.is_muted(),
@@ -126,6 +139,17 @@ impl FullChat {
 pub struct BasicChat {
     id: u32,
     name: String,
+
+    /// True if the chat is protected.
+    ///
+    /// UI should display a green checkmark
+    /// in the chat title,
+    /// in the chat profile title and
+    /// in the chatlist item
+    /// if chat protection is enabled.
+    /// UI should also display a green checkmark
+    /// in the contact profile
+    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -134,6 +158,7 @@ pub struct BasicChat {
     is_self_talk: bool,
     color: String,
     is_contact_request: bool,
+    is_protection_broken: bool,
     is_device_chat: bool,
     is_muted: bool,
 }
@@ -160,6 +185,7 @@ impl BasicChat {
             is_self_talk: chat.is_self_talk(),
             color,
             is_contact_request: chat.is_contact_request(),
+            is_protection_broken: chat.is_protection_broken(),
             is_device_chat: chat.is_device_talk(),
             is_muted: chat.is_muted(),
         })

deltachat-jsonrpc/src/api/types/contact.rs

@@ -19,11 +19,30 @@ pub struct ContactObject {
     profile_image: Option<String>, // BLOBS
     name_and_addr: String,
     is_blocked: bool,
+
+    /// True if the contact can be added to verified groups.
+    ///
+    /// If this is true
+    /// UI should display green checkmark after the contact name
+    /// in contact list items,
+    /// in chat member list items
+    /// and in profiles if no chat with the contact exist.
     is_verified: bool,
-    /// the address that verified this contact
-    verifier_addr: Option<String>,
-    /// the id of the contact that verified this contact
+
+    /// True if the contact profile title should have a green checkmark.
+    ///
+    /// This indicates whether 1:1 chat has a green checkmark
+    /// or will have a green checkmark if created.
+    is_profile_verified: bool,
+
+    /// The ID of the contact that verified this contact.
+    ///
+    /// If this is present,
+    /// display a green checkmark and "Introduced by ..."
+    /// string followed by the verifier contact name and address
+    /// in the contact profile.
     verifier_id: Option<u32>,
+
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
@@ -39,18 +58,12 @@ impl ContactObject {
             None => None,
         };
         let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+        let is_profile_verified = contact.is_profile_verified(context).await?;
 
-        let (verifier_addr, verifier_id) = if is_verified {
-            (
-                contact.get_verifier_addr(context).await?,
-                contact
-                    .get_verifier_id(context)
-                    .await?
-                    .map(|contact_id| contact_id.to_u32()),
-            )
-        } else {
-            (None, None)
-        };
+        let verifier_id = contact
+            .get_verifier_id(context)
+            .await?
+            .map(|contact_id| contact_id.to_u32());
 
         Ok(ContactObject {
             address: contact.get_addr().to_owned(),
@@ -64,7 +77,7 @@ impl ContactObject {
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
             is_verified,
-            verifier_addr,
+            is_profile_verified,
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),

deltachat-jsonrpc/src/api/types/message.rs

@@ -552,3 +552,71 @@ pub struct MessageReadReceipt {
     pub contact_id: u32,
     pub timestamp: i64,
 }
+
+#[derive(Serialize, TypeDef, schemars::JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct MessageInfo {
+    rawtext: String,
+    ephemeral_timer: EphemeralTimer,
+    /// When message is ephemeral this contains the timestamp of the message expiry
+    ephemeral_timestamp: Option<i64>,
+    error: Option<String>,
+    rfc724_mid: String,
+    server_urls: Vec<String>,
+    hop_info: Option<String>,
+}
+
+impl MessageInfo {
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+        let message = Message::load_from_db(context, msg_id).await?;
+        let rawtext = msg_id.rawtext(context).await?;
+        let ephemeral_timer = message.get_ephemeral_timer().into();
+        let ephemeral_timestamp = match message.get_ephemeral_timer() {
+            deltachat::ephemeral::Timer::Disabled => None,
+            deltachat::ephemeral::Timer::Enabled { .. } => Some(message.get_ephemeral_timestamp()),
+        };
+
+        let server_urls =
+            MsgId::get_info_server_urls(context, message.rfc724_mid().to_owned()).await?;
+
+        let hop_info = msg_id.hop_info(context).await?;
+
+        Ok(Self {
+            rawtext,
+            ephemeral_timer,
+            ephemeral_timestamp,
+            error: message.error(),
+            rfc724_mid: message.rfc724_mid().to_owned(),
+            server_urls,
+            hop_info,
+        })
+    }
+}
+
+#[derive(
+    Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize, TypeDef, schemars::JsonSchema,
+)]
+#[serde(rename_all = "camelCase", tag = "variant")]
+pub enum EphemeralTimer {
+    /// Timer is disabled.
+    Disabled,
+
+    /// Timer is enabled.
+    Enabled {
+        /// Timer duration in seconds.
+        ///
+        /// The value cannot be 0.
+        duration: u32,
+    },
+}
+
+impl From<deltachat::ephemeral::Timer> for EphemeralTimer {
+    fn from(value: deltachat::ephemeral::Timer) -> Self {
+        match value {
+            deltachat::ephemeral::Timer::Disabled => EphemeralTimer::Disabled,
+            deltachat::ephemeral::Timer::Enabled { duration } => {
+                EphemeralTimer::Enabled { duration }
+            }
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -6,6 +6,8 @@ use typescript_type_def::TypeDef;
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct ProviderInfo {
+    /// Unique ID, corresponding to provider database filename.
+    pub id: String,
     pub before_login_hint: String,
     pub overview_page: String,
     pub status: u32, // in reality this is an enum, but for simplicity and because it gets converted into a number anyway, we use an u32 here.
@@ -14,6 +16,7 @@ pub struct ProviderInfo {
 impl ProviderInfo {
     pub fn from_dc_type(provider: Option<&Provider>) -> Option<Self> {
         provider.map(|p| ProviderInfo {
+            id: p.id.to_owned(),
             before_login_hint: p.before_login_hint.to_owned(),
             overview_page: p.overview_page.to_owned(),
             status: p.status.to_u32().unwrap(),

deltachat-jsonrpc/src/lib.rs

@@ -17,7 +17,7 @@ mod tests {
         let accounts = Accounts::new(tmp_dir, writable).await?;
         let api = CommandApi::new(accounts);
 
-        let (sender, mut receiver) = unbounded::<String>();
+        let (sender, receiver) = unbounded::<String>();
 
         let (client, mut rx) = RpcClient::new();
         let session = RpcSession::new(client, api);
@@ -36,17 +36,17 @@ mod tests {
             let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
+            let result = receiver.recv().await?;
             println!("{result:?}");
-            assert_eq!(result, Some(response.to_owned()));
+            assert_eq!(result, response.to_owned());
         }
         {
             let request = r#"{"jsonrpc":"2.0","method":"get_all_account_ids","params":[],"id":2}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
+            let result = receiver.recv().await?;
             println!("{result:?}");
-            assert_eq!(result, Some(response.to_owned()));
+            assert_eq!(result, response.to_owned());
         }
 
         Ok(())
@@ -59,7 +59,7 @@ mod tests {
         let accounts = Accounts::new(tmp_dir, writable).await?;
         let api = CommandApi::new(accounts);
 
-        let (sender, mut receiver) = unbounded::<String>();
+        let (sender, receiver) = unbounded::<String>();
 
         let (client, mut rx) = RpcClient::new();
         let session = RpcSession::new(client, api);
@@ -78,15 +78,15 @@ mod tests {
             let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
-            assert_eq!(result, Some(response.to_owned()));
+            let result = receiver.recv().await?;
+            assert_eq!(result, response.to_owned());
         }
         {
             let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
-            assert_eq!(result, Some(response.to_owned()));
+            let result = receiver.recv().await?;
+            assert_eq!(result, response.to_owned());
         }
 
         Ok(())

deltachat-jsonrpc/src/webserver.rs

@@ -28,7 +28,7 @@ async fn main() -> Result<(), std::io::Error> {
         .layer(Extension(state.clone()));
 
     tokio::spawn(async move {
-        state.accounts.read().await.start_io().await;
+        state.accounts.write().await.start_io().await;
     });
 
     let addr = SocketAddr::from(([127, 0, 0, 1], port));

deltachat-jsonrpc/typescript/package.json

@@ -9,7 +9,6 @@
     "@types/chai": "^4.2.21",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^9.0.0",
-    "@types/node-fetch": "^2.5.7",
     "@types/ws": "^7.2.4",
     "c8": "^7.10.0",
     "chai": "^4.3.4",
@@ -17,7 +16,6 @@
     "esbuild": "^0.17.9",
     "http-server": "^14.1.1",
     "mocha": "^9.1.1",
-    "node-fetch": "^2.6.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.6.2",
     "typedoc": "^0.23.2",
@@ -55,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.126.1"
+  "version": "1.131.4"
 }

deltachat-jsonrpc/typescript/test/basic.ts

@@ -79,6 +79,9 @@ describe("basic tests", () => {
       accountId = await dc.rpc.addAccount();
     });
     it("should block and unblock contact", async function () {
+      // Cannot send sync messages to self as we do not have a self address.
+      await dc.rpc.setConfig(accountId, "sync_msgs", "0");
+
       const contactId = await dc.rpc.createContact(
         accountId,
         "example@delta.chat",

deltachat-jsonrpc/typescript/test/online.ts

@@ -13,16 +13,16 @@ describe("online tests", function () {
 
   before(async function () {
     this.timeout(60000);
-    if (!process.env.DCC_NEW_TMP_EMAIL) {
+    if (!process.env.CHATMAIL_DOMAIN) {
       if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
         console.error(
-          "CAN NOT RUN COVERAGE correctly: Missing DCC_NEW_TMP_EMAIL environment variable!\n\n",
+          "CAN NOT RUN COVERAGE correctly: Missing CHATMAIL_DOMAIN environment variable!\n\n",
           "You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test"
         );
         process.exit(1);
       }
       console.log(
-        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests"
+        "Missing CHATMAIL_DOMAIN environment variable!, skip integration tests"
       );
       this.skip();
     }
@@ -33,7 +33,7 @@ describe("online tests", function () {
       if (kind !== "Info") console.log(contextId, kind);
     });
 
-    account1 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    account1 = createTempUser(process.env.CHATMAIL_DOMAIN);
     if (!account1 || !account1.email || !account1.password) {
       console.log(
         "We didn't got back an account from the api, skip integration tests"
@@ -41,7 +41,7 @@ describe("online tests", function () {
       this.skip();
     }
 
-    account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    account2 = createTempUser(process.env.CHATMAIL_DOMAIN);
     if (!account2 || !account2.email || !account2.password) {
       console.log(
         "We didn't got back an account2 from the api, skip integration tests"

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -2,7 +2,6 @@ import { tmpdir } from "os";
 import { join, resolve } from "path";
 import { mkdtemp, rm } from "fs/promises";
 import { spawn, exec } from "child_process";
-import fetch from "node-fetch";
 import { Readable, Writable } from "node:stream";
 
 export type RpcServerHandle = {
@@ -57,15 +56,14 @@ export async function startServer(): Promise<RpcServerHandle> {
   };
 }
 
-export async function createTempUser(url: string) {
-  const response = await fetch(url, {
-    method: "POST",
-    headers: {
-      "cache-control": "no-cache",
-    },
-  });
-  if (!response.ok) throw new Error("Received invalid response");
-  return response.json();
+export function createTempUser(chatmailDomain: String) {
+  const charset = "2345789acdefghjkmnpqrstuvwxyz";
+  let user = "ci-";
+  for (let i = 0; i < 6; i++) {
+    user += charset[Math.floor(Math.random() * charset.length)];
+  }
+  const email = user + "@" + chatmailDomain;
+  return { email: email, password: user + "$" + user };
 }
 
 function getTargetDir(): Promise<string> {

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.126.1"
+version = "1.131.4"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -11,7 +11,7 @@ deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
 log = "0.4.20"
 pretty_env_logger = "0.5"
-rusqlite = "0.29"
+rusqlite = "0.30"
 rustyline = "12"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 

deltachat-repl/src/main.rs

@@ -401,7 +401,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    ctx: Context,
+    mut ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');

deltachat-rpc-client/README.md

@@ -25,7 +25,7 @@ $ pip install .
 ## Testing
 
 1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
-2. Run `PATH="../target/debug:$PATH" tox`.
+2. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
 
 Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
 

deltachat-rpc-client/pyproject.toml

@@ -71,6 +71,3 @@ line-length = 120
 
 [tool.isort]
 profile = "black"
-
-[tool.pytest.ini_options]
-log_cli = true

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -4,7 +4,7 @@ from warnings import warn
 
 from ._utils import AttrDict
 from .chat import Chat
-from .const import ChatlistFlag, ContactFlag, SpecialContactId
+from .const import ChatlistFlag, ContactFlag, EventType, SpecialContactId
 from .contact import Contact
 from .message import Message
 
@@ -111,6 +111,20 @@ class Account:
         contacts = self._rpc.get_blocked_contacts(self.id)
         return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
 
+    def get_chat_by_contact(self, contact: Union[int, Contact]) -> Optional[Chat]:
+        """Return 1:1 chat for a contact if it exists."""
+        if isinstance(contact, Contact):
+            assert contact.account == self
+            contact_id = contact.id
+        elif isinstance(contact, int):
+            contact_id = contact
+        else:
+            raise ValueError(f"{contact!r} is not a contact")
+        chat_id = self._rpc.get_chat_id_by_contact_id(self.id, contact_id)
+        if chat_id:
+            return Chat(self, chat_id)
+        return None
+
     def get_contacts(
         self,
         query: Optional[str] = None,
@@ -250,6 +264,13 @@ class Account:
         next_msg_ids = self._rpc.wait_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
+    def wait_for_incoming_msg_event(self):
+        """Wait for incoming message event and return it."""
+        while True:
+            event = self.wait_for_event()
+            if event.kind == EventType.INCOMING_MSG:
+                return event
+
     def get_fresh_messages_in_arrival_order(self) -> List[Message]:
         """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
         warn(

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -160,11 +160,12 @@ class Chat:
         text: Optional[str] = None,
         file: Optional[str] = None,
         quoted_msg: Optional[int] = None,
+        viewtype: Optional[str] = None,
     ) -> None:
         """Set draft message."""
         if isinstance(quoted_msg, Message):
             quoted_msg = quoted_msg.id
-        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg)
+        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg, viewtype)
 
     def remove_draft(self) -> None:
         """Remove draft message."""

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -1,6 +1,5 @@
-import json
 import os
-import urllib.request
+import random
 from typing import AsyncGenerator, List, Optional
 
 import pytest
@@ -10,12 +9,11 @@ from .rpc import Rpc
 
 
 def get_temp_credentials() -> dict:
-    url = os.getenv("DCC_NEW_TMP_EMAIL")
-    assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set"
-
-    request = urllib.request.Request(url, method="POST")
-    with urllib.request.urlopen(request, timeout=60) as f:
-        return json.load(f)
+    domain = os.getenv("CHATMAIL_DOMAIN")
+    username = "ci-" + "".join(random.choice("2345789acdefghjkmnpqrstuvwxyz") for i in range(6))
+    password = f"{username}${username}"
+    addr = f"{username}@{domain}"
+    return {"email": addr, "password": password}
 
 
 class ACFactory:
@@ -23,7 +21,9 @@ class ACFactory:
         self.deltachat = deltachat
 
     def get_unconfigured_account(self) -> Account:
-        return self.deltachat.add_account()
+        account = self.deltachat.add_account()
+        account.set_config("verified_one_on_one_chats", "1")
+        return account
 
     def get_unconfigured_bot(self) -> Bot:
         return Bot(self.get_unconfigured_account())
@@ -61,6 +61,16 @@ class ACFactory:
     def get_online_accounts(self, num: int) -> List[Account]:
         return [self.get_online_account() for _ in range(num)]
 
+    def resetup_account(self, ac: Account) -> Account:
+        """Resetup account from scratch, losing the encryption key."""
+        ac.stop_io()
+        ac_clone = self.get_unconfigured_account()
+        for i in ["addr", "mail_pw"]:
+            ac_clone.set_config(i, ac.get_config(i))
+        ac.remove()
+        ac_clone.configure()
+        return ac_clone
+
     def send_message(
         self,
         to_account: Account,

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 import logging
 import os
@@ -5,7 +6,7 @@ import subprocess
 import sys
 from queue import Queue
 from threading import Event, Thread
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Iterator, Optional
 
 
 class JsonRpcError(Exception):
@@ -23,7 +24,7 @@ class Rpc:
 
         self._kwargs = kwargs
         self.process: subprocess.Popen
-        self.id: int
+        self.id_iterator: Iterator[int]
         self.event_queues: Dict[int, Queue]
         # Map from request ID to `threading.Event`.
         self.request_events: Dict[int, Event]
@@ -54,7 +55,7 @@ class Rpc:
                 preexec_fn=os.setpgrp,  # noqa: PLW1509
                 **self._kwargs,
             )
-        self.id = 0
+        self.id_iterator = itertools.count(start=1)
         self.event_queues = {}
         self.request_events = {}
         self.request_results = {}
@@ -131,7 +132,9 @@ class Rpc:
                 event = self.get_next_event()
                 account_id = event["contextId"]
                 queue = self.get_queue(account_id)
-                queue.put(event["event"])
+                event = event["event"]
+                logging.debug("account_id=%d got an event %s", account_id, event)
+                queue.put(event)
         except Exception:
             # Log an exception if the event loop dies.
             logging.exception("Exception in the event loop")
@@ -143,14 +146,12 @@ class Rpc:
 
     def __getattr__(self, attr: str):
         def method(*args) -> Any:
-            self.id += 1
-            request_id = self.id
-
+            request_id = next(self.id_iterator)
             request = {
                 "jsonrpc": "2.0",
                 "method": attr,
                 "params": args,
-                "id": self.id,
+                "id": request_id,
             }
             event = Event()
             self.request_events[request_id] = event

deltachat-rpc-client/tests/test_securejoin.py

@@ -0,0 +1,320 @@
+import logging
+
+from deltachat_rpc_client import Chat, SpecialContactId
+
+
+def test_qr_setup_contact(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+
+    while True:
+        event = alice.wait_for_event()
+        if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+
+
+def test_qr_securejoin(acfactory):
+    alice, bob = acfactory.get_online_accounts(2)
+
+    logging.info("Alice creates a verified group")
+    alice_chat = alice.create_group("Verified group", protect=True)
+
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    while True:
+        event = alice.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+
+
+def test_qr_securejoin_contact_request(acfactory) -> None:
+    """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+    bob_chat_alice = snapshot.chat
+    assert bob_chat_alice.get_basic_snapshot().is_contact_request
+
+    alice_chat = alice.create_group("Verified group", protect=True)
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Chat stays being a contact request.
+    assert bob_chat_alice.get_basic_snapshot().is_contact_request
+
+
+def test_qr_readreceipt(acfactory) -> None:
+    alice, bob, charlie = acfactory.get_online_accounts(3)
+
+    logging.info("Bob and Charlie setup contact with Alice")
+    qr_code, _svg = alice.get_qr_code()
+
+    bob.secure_join(qr_code)
+    charlie.secure_join(qr_code)
+
+    for joiner in [bob, charlie]:
+        while True:
+            event = joiner.wait_for_event()
+            if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+                break
+
+    logging.info("Alice creates a verified group")
+    group = alice.create_group("Group", protect=True)
+
+    bob_addr = bob.get_config("addr")
+    charlie_addr = charlie.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_contact_charlie = alice.create_contact(charlie_addr, "Charlie")
+
+    group.add_contact(alice_contact_bob)
+    group.add_contact(alice_contact_charlie)
+
+    # Promote a group.
+    group.send_message(text="Hello")
+
+    logging.info("Bob and Charlie receive a group")
+
+    bob_msg_id = bob.wait_for_incoming_msg_event().msg_id
+    bob_message = bob.get_message_by_id(bob_msg_id)
+    bob_snapshot = bob_message.get_snapshot()
+    assert bob_snapshot.text == "Hello"
+
+    # Charlie receives the same "Hello" message as Bob.
+    charlie.wait_for_incoming_msg_event()
+
+    logging.info("Bob sends a message to the group")
+
+    bob_out_message = bob_snapshot.chat.send_message(text="Hi from Bob!")
+
+    charlie_msg_id = charlie.wait_for_incoming_msg_event().msg_id
+    charlie_message = charlie.get_message_by_id(charlie_msg_id)
+    charlie_snapshot = charlie_message.get_snapshot()
+    assert charlie_snapshot.text == "Hi from Bob!"
+
+    bob_contact_charlie = bob.create_contact(charlie_addr, "Charlie")
+    assert not bob.get_chat_by_contact(bob_contact_charlie)
+
+    logging.info("Charlie reads Bob's message")
+    charlie_message.mark_seen()
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "MsgRead" and event["msg_id"] == bob_out_message.id:
+            break
+
+    # Receiving a read receipt from Charlie
+    # should not unblock hidden chat with Charlie for Bob.
+    assert not bob.get_chat_by_contact(bob_contact_charlie)
+
+
+def test_verified_group_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifying a member and sending a message in a group."""
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    while True:
+        event = ac3.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hi!"
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Hi!"
+
+    # ac1 contact is verified for ac2 because ac3 gossiped ac1 key in the "Hi!" message.
+    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
+    assert ac1_contact.get_snapshot().is_verified
+
+    # ac2 can write messages to the group.
+    snapshot.chat.send_text("Works again!")
+
+    snapshot = ac3.get_message_by_id(ac3.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1_chat_messages = snapshot.chat.get_messages()
+    ac2_addr = ac2.get_config("addr")
+    assert ac1_chat_messages[-2].get_snapshot().text == f"Changed setup for {ac2_addr}"
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_verified_group_member_added_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifiying than removing and adding a member back."""
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    while True:
+        event = ac3.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("Received message %s", snapshot.text)
+    assert snapshot.text == "Hi!"
+
+    ac1.wait_for_incoming_msg_event()  # Hi!
+
+    ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
+    ac3_chat.remove_contact(ac3_contact_ac2)
+    ac3_chat.add_contact(ac3_contact_ac2)
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert "removed" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "removed" in snapshot.text
+
+    event = ac2.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    chat_id = event.chat_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("ac2 got event message: %s", snapshot.text)
+    assert "added" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "added" in snapshot.text
+
+    chat = Chat(ac2, chat_id)
+    chat.send_text("Works again!")
+
+    msg_id = ac3.wait_for_incoming_msg_event().msg_id
+    message = ac3.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
+    assert ac1_contact_ac2_snapshot.is_verified
+    assert ac1_contact_ac2_snapshot.verifier_id == ac1.get_contact_by_addr(ac3.get_config("addr")).id
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id

deltachat-rpc-client/tests/test_something.py

@@ -358,3 +358,22 @@ def test_openrpc_command_line() -> None:
     openrpc = json.loads(out)
     assert "openrpc" in openrpc
     assert "methods" in openrpc
+
+
+def test_provider_info(rpc) -> None:
+    account_id = rpc.add_account()
+
+    provider_info = rpc.get_provider_info(account_id, "example.org")
+    assert provider_info["id"] == "example.com"
+
+    provider_info = rpc.get_provider_info(account_id, "uep7oiw4ahtaizuloith.org")
+    assert provider_info is None
+
+    # Test MX record resolution.
+    provider_info = rpc.get_provider_info(account_id, "github.com")
+    assert provider_info["id"] == "gmail"
+
+    # Disable MX record resolution.
+    rpc.set_config(account_id, "socks5_enabled", "1")
+    provider_info = rpc.get_provider_info(account_id, "github.com")
+    assert provider_info is None

deltachat-rpc-client/tox.ini

@@ -11,7 +11,7 @@ setenv =
 # Avoid stack overflow when Rust core is built without optimizations.
     RUST_MIN_STACK=8388608
 passenv =
-    DCC_NEW_TMP_EMAIL
+    CHATMAIL_DOMAIN
 deps =
     pytest
     pytest-timeout
@@ -28,4 +28,6 @@ commands =
     ruff src/ examples/ tests/
 
 [pytest]
-timeout = 60
+timeout = 300
+log_cli = true
+log_level = debug

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.126.1"
+version = "1.131.4"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deny.toml

@@ -11,6 +11,7 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
+     { name = "async-channel", version = "1.9.0" },
      { name = "base16ct", version = "0.1.1" },
      { name = "base64", version = "<0.21" },
      { name = "bitflags", version = "1.3.2" },
@@ -24,18 +25,18 @@ skip = [
      { name = "digest", version = "<0.10" },
      { name = "ed25519-dalek", version = "1.0.1" },
      { name = "ed25519", version = "1.5.3" },
+     { name = "event-listener", version = "2.5.3" },
      { name = "getrandom", version = "<0.2" },
-     { name = "hashbrown", version = "<0.14.0" },
-     { name = "indexmap", version = "<2.0.0" },
      { name = "pem-rfc7468", version = "0.6.0" },
      { name = "pkcs8", version = "0.9.0" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
-     { name = "redox_syscall", version = "0.2.16" },
+     { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
+     { name = "ring", version = "0.16.20" },
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },
@@ -44,15 +45,12 @@ skip = [
      { name = "spki", version = "0.6.0" },
      { name = "syn", version = "1.0.109" },
      { name = "time", version = "<0.3" },
+     { name = "untrusted", version = "0.7.1" },
      { name = "wasi", version = "<0.11" },
-     { name = "windows_aarch64_gnullvm", version = "<0.48" },
      { name = "windows_aarch64_msvc", version = "<0.48" },
      { name = "windows_i686_gnu", version = "<0.48" },
      { name = "windows_i686_msvc", version = "<0.48" },
-     { name = "windows-sys", version = "<0.48" },
-     { name = "windows-targets", version = "<0.48" },
      { name = "windows", version = "0.32.0" },
-     { name = "windows_x86_64_gnullvm", version = "<0.48" },
      { name = "windows_x86_64_gnu", version = "<0.48" },
      { name = "windows_x86_64_msvc", version = "<0.48" },
 ]
@@ -86,5 +84,4 @@ license-files = [
 github = [
        "async-email",
        "deltachat",
-       "quinn-rs",
 ]

draft/aeap-mvp.md

@@ -57,7 +57,7 @@ Note that usually a mail is signed by a key that has a UID matching the from add
 ### Notes:
   
 - We treat protected and non-protected chats the same
-- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more pepole know what old addresses you had).
+- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more people know what old addresses you had).
 - As soon as we encrypt read receipts, sending a read receipt will be enough to tell a lot of people that you transitioned
 - AEAP will make the problem of inconsistent group state worse, both because it doesn't work if the message is unencrypted (even if the design allowed it, it would be problematic security-wise) and because some chat partners may have gotten the transition and some not. We should do something against this at some point in the future, like asking the user whether they want to add/remove the members to restore consistent group state.
 

node/README.md

@@ -40,7 +40,7 @@ npm install deltachat-node
 
 ## Dependencies
 
-- Nodejs >= `v16.0.0`
+- Nodejs >= `v18.0.0`
 - rustup (optional if you can't use the prebuilds)
 
 > On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
@@ -113,8 +113,8 @@ Then, in the `deltachat-desktop` repository, run:
 deltachat doesn't support universal (fat) binaries (that contain builds for both cpu architectures) yet, until it does you can use the following workaround to get x86_64 builds:
 
 ```
-$ fnm install 17 --arch x64
-$ fnm use 17
+$ fnm install 19 --arch x64
+$ fnm use 19
 $ node -p process.arch
 # result should be x64
 $ rustup target add x86_64-apple-darwin
@@ -127,8 +127,8 @@ $ npm run test
 If your node and electron are already build for arm64 you can also try building for arm:
 
 ```
-$ fnm install 16 --arch arm64
-$ fnm use 16
+$ fnm install 18 --arch arm64
+$ fnm use 18
 $ node -p process.arch
 # result should be arm64
 $ npm_config_arch=arm64 npm run build
@@ -204,10 +204,10 @@ Running `npm test` ends with showing a code coverage report, which is produced b
 
 The coverage report from `nyc` in the console is rather limited. To get a more detailed coverage report you can run `npm run coverage-html-report`. This will produce a html report from the `nyc` data and display it in a browser on your local machine.
 
-To run the integration tests you need to set the `DCC_NEW_TMP_EMAIL` environment variables. E.g.:
+To run the integration tests you need to set the `CHATMAIL_DOMAIN` environment variables. E.g.:
 
 ```
-$ export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=[token]
+$ export CHATMAIL_DOMAIN=chat.example.org
 $ npm run test
 ```
 

node/constants.js

@@ -28,6 +28,7 @@ module.exports = {
   DC_DOWNLOAD_DONE: 0,
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
@@ -239,6 +240,7 @@ module.exports = {
   DC_STR_MSGGRPNAME: 15,
   DC_STR_MSGLOCATIONDISABLED: 65,
   DC_STR_MSGLOCATIONENABLED: 64,
+  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE: 172,
   DC_STR_NOMESSAGES: 1,
   DC_STR_NOT_CONNECTED: 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER: 113,

node/lib/constants.ts

@@ -28,6 +28,7 @@ export enum C {
   DC_DOWNLOAD_DONE = 0,
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
@@ -239,6 +240,7 @@ export enum C {
   DC_STR_MSGGRPNAME = 15,
   DC_STR_MSGLOCATIONDISABLED = 65,
   DC_STR_MSGLOCATIONENABLED = 64,
+  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE = 172,
   DC_STR_NOMESSAGES = 1,
   DC_STR_NOT_CONNECTED = 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER = 113,

node/test/test.js

@@ -8,26 +8,17 @@ import { EventId2EventName, C } from '../dist/constants'
 import { join } from 'path'
 import { statSync } from 'fs'
 import { Context } from '../dist/context'
-import fetch from 'node-fetch'
 chai.use(chaiAsPromised)
 chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 
-async function createTempUser(url) {
-  async function postData(url = '') {
-    // Default options are marked with *
-    const response = await fetch(url, {
-      method: 'POST', // *GET, POST, PUT, DELETE, etc.
-      headers: {
-        'cache-control': 'no-cache',
-      },
-    })
-    if (!response.ok) {
-      throw new Error('request failed: ' + response.body.read())
-    }
-    return response.json() // parses JSON response into native JavaScript objects
+function createTempUser(chatmailDomain) {
+  const charset = "2345789acdefghjkmnpqrstuvwxyz";
+  let user = "ci-";
+  for (let i = 0; i < 6; i++) {
+    user += charset[Math.floor(Math.random() * charset.length)];
   }
-
-  return await postData(url)
+  const email = user + "@" + chatmailDomain;
+  return { email: email, password: user + "$" + user };
 }
 
 describe('static tests', function () {
@@ -676,9 +667,9 @@ describe('Offline Tests with unconfigured account', function () {
 
     const lot = chatList.getSummary(0)
     strictEqual(lot.getId(), 0, 'lot has no id')
-    strictEqual(lot.getState(), C.DC_STATE_UNDEFINED, 'correct state')
+    strictEqual(lot.getState(), C.DC_STATE_IN_NOTICED, 'correct state')
 
-    const text = 'No messages.'
+    const text = 'Others will only see this group after you sent a first message.'
     context.createGroupChat('groupchat1111')
     chatList = context.getChatList(0, 'groupchat1111', null)
     strictEqual(
@@ -768,14 +759,7 @@ describe('Integration tests', function () {
   })
 
   this.beforeAll(async function () {
-    if (!process.env.DCC_NEW_TMP_EMAIL) {
-      console.log(
-        'Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests'
-      )
-      this.skip()
-    }
-
-    account = await createTempUser(process.env.DCC_NEW_TMP_EMAIL)
+    account = createTempUser(process.env.CHATMAIL_DOMAIN)
     if (!account || !account.email || !account.password) {
       console.log(
         "We didn't got back an account from the api, skip integration tests"

node/tsconfig.json

@@ -13,10 +13,10 @@
     },
     "exclude": ["node_modules", "deltachat-core-rust", "dist", "scripts"],
     "typedocOptions": {
-        "mode": "file",
         "out": "docs",
-        "excludeNotExported": true,
+        "excludePrivate": true,
         "defaultCategory": "index",
-        "includeVersion": true
+        "includeVersion": true,
+        "entryPoints": ["lib/index.ts"]
     }
 }

node/windows.md

@@ -6,7 +6,7 @@ E.g via <https://git-scm.com/download/win>
 
 ## install node
 
-Download and install `v16` from <https://nodejs.org/en/>
+Download and install `v18` from <https://nodejs.org/en/>
 
 ## install rust
 

package.json

@@ -2,28 +2,25 @@
   "dependencies": {
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
-    "node-gyp-build": "^4.1.0"
+    "node-gyp-build": "^4.6.1"
   },
   "description": "node.js bindings for deltachat-core",
   "devDependencies": {
     "@types/debug": "^4.1.7",
-    "@types/node": "^16.11.26",
+    "@types/node": "^20.8.10",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "esm": "^3.2.25",
-    "hallmark": "^2.0.0",
     "mocha": "^8.2.1",
-    "node-fetch": "^2.6.7",
-    "node-gyp": "^9.0.0",
-    "opn-cli": "^5.0.0",
-    "prebuildify": "^3.0.0",
-    "prebuildify-ci": "^1.0.4",
-    "prettier": "^2.0.5",
-    "typedoc": "^0.17.0",
-    "typescript": "^3.9.10"
+    "node-gyp": "^10.0.0",
+    "prebuildify": "^5.0.1",
+    "prebuildify-ci": "^1.0.5",
+    "prettier": "^3.0.3",
+    "typedoc": "^0.25.3",
+    "typescript": "^5.2.2"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "files": [
     "node/scripts/*",
@@ -49,16 +46,15 @@
     "build:core:rust": "node node/scripts/rebuild-core.js",
     "clean": "rm -rf node/dist node/build node/prebuilds node/node_modules ./target",
     "download-prebuilds": "prebuildify-ci download",
-    "hallmark": "hallmark --fix",
     "install": "node node/scripts/install.js",
     "install:prebuilds": "cd node && node-gyp-build \"npm run build:core\" \"npm run build:bindings:c:postinstall\"",
     "lint": "prettier --check \"node/lib/**/*.{ts,tsx}\"",
     "lint-fix": "prettier --write \"node/lib/**/*.{ts,tsx}\" \"node/test/**/*.js\"",
-    "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
+    "prebuildify": "cd node && prebuildify -t 18.0.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.126.1"
+  "version": "1.131.4"
 }

python/README.rst

@@ -53,14 +53,14 @@ end-to-end tests that require accounts on real e-mail servers.
 Running "live" tests with temporary accounts
 --------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLs created and managed by `mailadm <https://mailadm.readthedocs.io/>`_.
+If you want to run live functional tests
+you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
+that creates e-mail accounts like this::
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this::
+    export CHATMAIL_DOMAIN=nine.testrun.org
 
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
-These accounts are removed automatically as they expire.
+With this account-creation setting, pytest runs create ephemeral e-mail accounts on the server.
+These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
 After setting the variable, either rerun `scripts/run-python-test.sh`
 or run offline and online tests with `tox` directly::
 
@@ -159,6 +159,6 @@ find it with::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ docker run -e DCC_NEW_TMP_EMAIL \
+    $ docker run -e CHATMAIL_DOMAIN \
        --rm -it -v $(pwd):/mnt -w /mnt \
        deltachat/coredeps scripts/run_all.sh

python/src/deltachat/account.py

@@ -571,11 +571,11 @@ class Account:
         return ScannedQRCode(lot)
 
     def qr_setup_contact(self, qr):
-        """setup contact and return a Chat after contact is established.
+        """setup contact and return a `Chat` instance after contact is established.
+
+        This function triggers a network protocol in the background between
+        the emitter of the QR code and this account.
 
-        Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
-        is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
         assert self.check_qr(qr).is_ask_verifycontact()
@@ -585,11 +585,11 @@ class Account:
         return Chat(self, chat_id)
 
     def qr_join_chat(self, qr):
-        """join a chat group through a QR code.
+        """return a `Chat` instance for which the securejoin network
+        protocol has been started.
 
-        Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
-        is returned which is the chat that we just joined.
+        This function triggers a network protocol in the background between
+        the emitter of the QR code and this account.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
         """

python/src/deltachat/testplugin.py

@@ -8,11 +8,11 @@ import sys
 import threading
 import time
 import weakref
+import random
 from queue import Queue
 from typing import Callable, Dict, List, Optional, Set
 
 import pytest
-import requests
 from _pytest._code import Source
 
 import deltachat
@@ -24,10 +24,10 @@ from .events import FFIEventLogger, FFIEventTracker
 def pytest_addoption(parser):
     group = parser.getgroup("deltachat testplugin options")
     group.addoption(
-        "--liveconfig",
+        "--chatmail",
         action="store",
         default=None,
-        help="a file with >=2 lines where each line contains NAME=VALUE config settings for one account",
+        help="chatmail server domain name",
     )
     group.addoption(
         "--ignored",
@@ -52,11 +52,11 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    cfg = config.getoption("--liveconfig")
+    cfg = config.getoption("--chatmail")
     if not cfg:
-        cfg = os.getenv("DCC_NEW_TMP_EMAIL")
+        cfg = os.getenv("CHATMAIL_DOMAIN")
         if cfg:
-            config.option.liveconfig = cfg
+            config.option.chatmail = cfg
 
     # Make sure we don't get garbled output because threads keep running
     # collect all ever created accounts in a weakref-set (so we don't
@@ -125,13 +125,10 @@ def pytest_report_header(config, startdir):
         ),
     ]
 
-    cfg = config.option.liveconfig
-    if cfg:
-        if "?" in cfg:
-            url, token = cfg.split("?", 1)
-            summary.append(f"Liveconfig provider: {url}?<token omitted>")
-        else:
-            summary.append(f"Liveconfig file: {cfg}")
+    chatmail_opt = config.getoption("--chatmail")
+    if chatmail_opt:
+        summary.append(f"Chatmail account provider: {chatmail_opt}")
+
     return summary
 
 
@@ -156,38 +153,29 @@ class TestProcess:
     def get_liveconfig_producer(self):
         """provide live account configs, cached on a per-test-process scope
         so that test functions can re-use already known live configs.
-        Depending on the --liveconfig option this comes from
-        a HTTP provider or a file with a line specifying each accounts config.
         """
-        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
-        if not liveconfig_opt:
-            pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts")
-
-        if not liveconfig_opt.startswith("http"):
-            for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith("#"):
-                    d = {}
-                    for part in line.split():
-                        name, value = part.split("=")
-                        d[name] = value
-                    self._configlist.append(d)
-
-            yield from iter(self._configlist)
-        else:
+        chatmail_opt = self.pytestconfig.getoption("--chatmail")
+        if chatmail_opt:
+            # Use a chatmail instance.
+            domain = chatmail_opt
             MAX_LIVE_CREATED_ACCOUNTS = 10
             for index in range(MAX_LIVE_CREATED_ACCOUNTS):
                 try:
                     yield self._configlist[index]
                 except IndexError:
-                    res = requests.post(liveconfig_opt, timeout=60)
-                    if res.status_code != 200:
-                        pytest.fail(f"newtmpuser count={index} code={res.status_code}: '{res.text}'")
-                    d = res.json()
-                    config = {"addr": d["email"], "mail_pw": d["password"]}
+                    part = "".join(random.choices("2345789acdefghjkmnpqrstuvwxyz", k=6))
+                    username = f"ci-{part}"
+                    password = f"{username}${username}"
+                    addr = f"{username}@{domain}"
+                    config = {"addr": addr, "mail_pw": password}
                     print("newtmpuser {}: addr={}".format(index, config["addr"]))
                     self._configlist.append(config)
                     yield config
             pytest.fail(f"more than {MAX_LIVE_CREATED_ACCOUNTS} live accounts requested.")
+        else:
+            pytest.skip(
+                "specify CHATMAIL_DOMAIN or --chatmail to provide live accounts",
+            )
 
     def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
         db_target_path = pathlib.Path(db_target_path)
@@ -533,6 +521,7 @@ class ACFactory:
         configdict.setdefault("bcc_self", False)
         configdict.setdefault("mvbox_move", False)
         configdict.setdefault("sentbox_watch", False)
+        configdict.setdefault("sync_msgs", False)
         ac.update_config(configdict)
         self._preconfigure_key(ac, configdict["addr"])
         return ac

python/tests/test_0_complex_or_slow.py

@@ -140,6 +140,10 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
     assert msg.is_system_message()
     assert "added" in msg.text.lower()
 
+    assert any(
+        m.is_system_message() and m.text == "Messages are guaranteed to be end-to-end encrypted from now on."
+        for m in msg.chat.get_messages()
+    )
     lp.sec("ac1: send message")
     msg_out = chat1.send_text("hello")
     assert msg_out.is_encrypted()
@@ -580,6 +584,7 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.get_sender_contact().addr == ac1.get_config("addr")
     chat2 = msg_in.chat
     assert chat2.is_protected()
+    assert chat2.get_messages()[0].text == "Messages are guaranteed to be end-to-end encrypted from now on."
 
     lp.sec("ac2_offl: sending message")
     msg_out = chat2.send_text("hello")
@@ -647,6 +652,15 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     chat2.send_text("hi2")
 
     lp.sec("ac2_offl: receiving message")
+    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+    msg_in = ac2_offl.get_message_by_id(ev.data2)
+    assert msg_in.is_system_message()
+    assert msg_in.text == "Messages are guaranteed to be end-to-end encrypted from now on."
+
+    # We need to consume one event that has data2=0
+    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+    assert ev.data2 == 0
+
     ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
     msg_in = ac2_offl.get_message_by_id(ev.data2)
     assert not msg_in.is_system_message()

python/tests/test_1_online.py

@@ -367,7 +367,7 @@ def test_webxdc_download_on_demand(acfactory, data, lp):
 
     lp.sec("ac2 sets download limit")
     ac2.set_config("download_limit", "100")
-    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(50000))}, "some test data")
+    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data")
     ac2_update = ac2._evtracker.wait_next_incoming_message()
     assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE
     assert not msg2.get_status_updates()
@@ -489,7 +489,7 @@ def test_forward_messages(acfactory, lp):
     lp.sec("ac2: check new chat has a forwarded message")
     assert chat3.is_promoted()
     messages = chat3.get_messages()
-    assert len(messages) == 1
+    assert len(messages) == 2
     msg = messages[-1]
     assert msg.is_forwarded()
     ac2.delete_messages(messages)
@@ -1445,7 +1445,7 @@ def test_reaction_to_partially_fetched_msg(acfactory, lp, tmp_path):
       message, then processes a partially downloaded message.
     - As a result, Bob does not see a reaction
     """
-    download_limit = 32768
+    download_limit = 300000
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_addr = ac1.get_config("addr")
     chat = ac1.create_chat(ac2)
@@ -1699,6 +1699,59 @@ def test_qr_join_chat(acfactory, lp, verified_one_on_one_chats):
     assert not ch.is_protected()
 
 
+def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory, lp):
+    ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
+
+    lp.sec("ac3: verify with ac2")
+    ac3.qr_setup_contact(ac2.get_setup_contact_qr())
+    ac2._evtracker.wait_securejoin_inviter_progress(1000)
+
+    # in order for ac2 to have pending bobstate with a verified group
+    # we first create a fully joined verified group, and then start
+    # joining a second time but interrupt it, to create pending bob state
+
+    lp.sec("ac1: create verified group that ac2 fully joins")
+    ch1 = ac1.create_group_chat("ac1-shutoff group", verified=True)
+    ac2.qr_join_chat(ch1.get_join_qr())
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    # ensure ac1 can write and ac2 receives messages in verified chat
+    ch1.send_text("ac1 says hello")
+    while 1:
+        msg = ac2.wait_next_incoming_message()
+        if msg.text == "ac1 says hello":
+            assert msg.chat.is_protected()
+            break
+
+    lp.sec("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
+    ac2.qr_join_chat(ch1.get_join_qr())
+    ac1.shutdown()
+    lp.sec("ac2 now has pending bobstate but ac1 is shutoff")
+
+    # we meanwhile expect ac3/ac2 verification started in the beginning to have completed
+    assert ac3.get_contact(ac2).is_verified()
+    assert ac2.get_contact(ac3).is_verified()
+
+    lp.sec("ac3: create a verified group VG with ac2")
+    vg = ac3.create_group_chat("ac3-created", [ac2], verified=True)
+
+    # ensure ac2 receives message in VG
+    vg.send_text("hello")
+    while 1:
+        msg = ac2.wait_next_incoming_message()
+        if msg.text == "hello":
+            assert msg.chat.is_protected()
+            break
+
+    lp.sec("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
+    ac4.qr_join_chat(vg.get_join_qr())
+    ac3._evtracker.wait_securejoin_inviter_progress(1000)
+    while 1:
+        ev = ac2._evtracker.get()
+        if "added by unrelated SecureJoin" in str(ev):
+            return
+
+
 def test_qr_new_group_unblocked(acfactory, lp):
     """Regression test for a bug intoduced in core v1.113.0.
     ac2 scans a verified group QR code created by ac1.
@@ -1904,7 +1957,7 @@ def test_system_group_msg_from_blocked_user(acfactory, lp):
     chat_on_ac2.send_text("This will arrive")
     msg = ac1._evtracker.wait_next_incoming_message()
     assert msg.text == "This will arrive"
-    message_texts = [m.text for m in chat_on_ac1.get_messages()]
+    message_texts = [m.text for m in chat_on_ac1.get_messages() if not m.is_system_message()]
     assert len(message_texts) == 2
     assert "First group message" in message_texts
     assert "This will arrive" in message_texts
@@ -2208,7 +2261,17 @@ def test_delete_multiple_messages(acfactory, lp):
 
 def test_trash_multiple_messages(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
+
+    # Create the Trash folder on IMAP server
+    # and recreate the account so Trash folder is configured.
+    lp.sec("Creating trash folder")
+    ac2.direct_imap.create_folder("Trash")
+    lp.sec("Creating new accounts")
+    ac2 = acfactory.new_online_configuring_account(cloned_from=ac2)
+    acfactory.bring_accounts_online()
+
     ac2.set_config("delete_to_trash", "1")
+    assert ac2.get_config("configured_trash_folder")
     chat12 = acfactory.get_accepted_chat(ac1, ac2)
 
     lp.sec("ac1: sending 3 messages")
@@ -2239,13 +2302,15 @@ def test_trash_multiple_messages(acfactory, lp):
 
 
 def test_configure_error_msgs_wrong_pw(acfactory):
-    configdict = acfactory.get_next_liveconfig()
-    ac1 = acfactory.get_unconfigured_account()
-    ac1.update_config(configdict)
-    ac1.set_config("mail_pw", "abc")  # Wrong mail pw
-    ac1.configure()
+    (ac1,) = acfactory.get_online_accounts(1)
+
+    ac2 = acfactory.get_unconfigured_account()
+    ac2.set_config("addr", ac1.get_config("addr"))
+    ac2.set_config("mail_pw", "abc")  # Wrong mail pw
+    ac2.configure()
     while True:
-        ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        ev = ac2._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        print(f"Configuration progress: {ev.data1}")
         if ev.data1 == 0:
             break
     # Password is wrong so it definitely has to say something about "password"
@@ -2548,7 +2613,7 @@ class TestOnlineConfigureFails:
     def test_invalid_user(self, acfactory):
         configdict = acfactory.get_next_liveconfig()
         ac1 = acfactory.get_unconfigured_account()
-        configdict["addr"] = "x" + configdict["addr"]
+        configdict["addr"] = "$" + configdict["addr"]
         ac1.update_config(configdict)
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
@@ -2557,7 +2622,7 @@ class TestOnlineConfigureFails:
     def test_invalid_domain(self, acfactory):
         configdict = acfactory.get_next_liveconfig()
         ac1 = acfactory.get_unconfigured_account()
-        configdict["addr"] += "x"
+        configdict["addr"] += "$"
         ac1.update_config(configdict)
         configtracker = ac1.configure()
         configtracker.wait_progress(500)

python/tests/test_2_increation.py

@@ -26,7 +26,7 @@ def wait_msgs_changed(account, msgs_list):
                     break
         else:
             account.log(f"waiting mismatch data1={data1} data2={data2}")
-    return ev.data1, ev.data2
+    return ev.data2
 
 
 class TestOnlineInCreation:
@@ -68,14 +68,17 @@ class TestOnlineInCreation:
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
 
-        lp.sec("forward the message while still in creation")
+        lp.sec("create a new group")
         chat2 = ac1.create_group_chat("newgroup")
+        wait_msgs_changed(ac1, [(0, 0)])
+
+        lp.sec("add a contact to new group")
         chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
+        wait_msgs_changed(ac1, [(chat2.id, None)])
+
+        lp.sec("forward the message while still in creation")
         ac1.forward_messages([prepared_original], chat2)
-        # XXX there might be two EVENT_MSGS_CHANGED and only one of them
-        # is the one caused by forwarding
-        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
+        forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 

python/tests/test_3_offline.py

@@ -5,8 +5,6 @@ from datetime import datetime, timedelta, timezone
 import pytest
 
 import deltachat as dc
-from deltachat.capi import ffi, lib
-from deltachat.cutil import iter_array
 from deltachat.tracker import ImexFailed
 from deltachat import Account, account_hookimpl, Message
 
@@ -802,6 +800,7 @@ class TestOfflineChat:
     def test_audit_log_view_without_daymarker(self, ac1, lp):
         lp.sec("ac1: test audit log (show only system messages)")
         chat = ac1.create_group_chat(name="audit log sample data")
+
         # promote chat
         chat.send_text("hello")
         assert chat.is_promoted()
@@ -811,12 +810,6 @@ class TestOfflineChat:
         chat.set_name("audit log test group")
         chat.send_text("a message in between")
 
-        lp.sec("check message count of all messages")
-        assert len(chat.get_messages()) == 4
-
         lp.sec("check message count of only system messages (without daymarkers)")
-        dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, dc.const.DC_GCM_INFO_ONLY, 0),
-            lib.dc_array_unref,
-        )
-        assert len(list(iter_array(dc_array, lambda x: x))) == 2
+        sysmessages = [x for x in chat.get_messages() if x.is_system_message()]
+        assert len(sysmessages) == 3

python/tox.ini

@@ -16,7 +16,7 @@ setenv =
 passenv = 
     DCC_RS_DEV
     DCC_RS_TARGET
-    DCC_NEW_TMP_EMAIL 
+    CHATMAIL_DOMAIN
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -85,7 +85,7 @@ commands =
 addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 150
+timeout = 300
 timeout_func_only = True
 markers = 
     ignored: ignore this test in default test runs, use --ignored to run.

release-date.in

@@ -1 +1 @@
-2023-10-24
+2023-11-16

scripts/README.md

@@ -14,16 +14,23 @@ and an own build machine.
 
 - `../.github/workflows` contains jobs run by GitHub Actions.
 
+- `run-python-test.sh` runs CFFI Python tests.
+
+- `run-rpc-test.sh` runs JSON-RPC Python tests.
+
+- `make-python-testenv.sh` creates a local python test development environment with CFFI bindings.
+  Reusing the same environment is faster than running `run-python-test.sh` which always
+  recreates environment from scratch and runs additional lints.
+
+- `make-rpc-testenv.sh` creates a local python development environment with JSON-RPC bindings,
+   i.e. `deltachat-rpc-client` and `deltachat-rpc-server`.
+
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
 
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `make-python-testenv.sh` creates local python test development environment.
-  Reusing the same environment is faster than running `run-python-test.sh` which always
-  recreates environment from scratch and runs additional lints.
-
 - `run-doxygen.sh` generates C-docs which are then uploaded to https://c.delta.chat/
 
 - `run_all.sh` builds Python wheels

scripts/make-rpc-testenv.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+tox -c deltachat-rpc-client -e py --devenv venv
+venv/bin/pip install --upgrade pip
+cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug

scripts/remote_tests_python.sh

@@ -28,7 +28,7 @@ ssh $SSHTARGET <<_HERE
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     export TARGET=release
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    export CHATMAIL_DOMAIN=$CHATMAIL_DOMAIN
 
     #we rely on tox/virtualenv being available in the host
     #rm -rf virtualenv venv

scripts/run-rpc-test.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+PATH="$PWD/venv/bin:$PATH" tox -c deltachat-rpc-client

scripts/run_all.sh

@@ -27,7 +27,7 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
-unset DCC_NEW_TMP_EMAIL
+unset CHATMAIL_DOMAIN
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=3c8f7e846c915a183dc44536fb5480d1f25d7c42
+REV=18f714cf73d0bdfb8b013fa344494ab80c92b477
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

spec.md

@@ -192,7 +192,7 @@ in different orders, esp. on creating new groups.
 To remove a member, a `Chat-Group-Member-Removed` header must be sent
 with the value set to the email-address of the member to remove.
 When receiving a `Chat-Group-Member-Removed` header,
-only exaxtly the given member has to be removed from the member list.
+only exactly the given member has to be removed from the member list.
 
 Messenger clients MUST NOT construct the member list
 on other group messages
@@ -339,7 +339,7 @@ only on image changes.
 
 In older specs, the profile-image was sent as an attachment
 and `Chat-User-Avatar:` specified its name.
-However, it turned out that these attachments are kind of unuexpected to users,
+However, it turned out that these attachments are kind of unexpected to users,
 therefore the profile-image go to the header now.
 
 

src/accounts.rs

@@ -1,6 +1,7 @@
 //! # Account manager module.
 
 use std::collections::BTreeMap;
+use std::future::Future;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
@@ -155,7 +156,7 @@ impl Accounts {
         if let Some(cfg) = self.config.get_account(id) {
             let account_path = self.dir.join(cfg.dir);
 
-            fs::remove_dir_all(&account_path)
+            try_many_times(|| fs::remove_dir_all(&account_path))
                 .await
                 .context("failed to remove account data")?;
         }
@@ -191,10 +192,10 @@ impl Accounts {
             fs::create_dir_all(self.dir.join(&account_config.dir))
                 .await
                 .context("failed to create dir")?;
-            fs::rename(&dbfile, &new_dbfile)
+            try_many_times(|| fs::rename(&dbfile, &new_dbfile))
                 .await
                 .context("failed to rename dbfile")?;
-            fs::rename(&blobdir, &new_blobdir)
+            try_many_times(|| fs::rename(&blobdir, &new_blobdir))
                 .await
                 .context("failed to rename blobdir")?;
             if walfile.exists() {
@@ -219,7 +220,7 @@ impl Accounts {
             }
             Err(err) => {
                 let account_path = std::path::PathBuf::from(&account_config.dir);
-                fs::remove_dir_all(&account_path)
+                try_many_times(|| fs::remove_dir_all(&account_path))
                     .await
                     .context("failed to remove account data")?;
                 self.config.remove_account(account_config.id).await?;
@@ -257,8 +258,8 @@ impl Accounts {
     }
 
     /// Starts background tasks such as IMAP and SMTP loops for all accounts.
-    pub async fn start_io(&self) {
-        for account in self.accounts.values() {
+    pub async fn start_io(&mut self) {
+        for account in self.accounts.values_mut() {
             account.start_io().await;
         }
     }
@@ -560,6 +561,37 @@ impl Config {
     }
 }
 
+/// Spend up to 1 minute trying to do the operation.
+///
+/// Even if Delta Chat itself does not hold the file lock,
+/// there may be other processes such as antivirus,
+/// or the filesystem may be network-mounted.
+///
+/// Without this workaround removing account may fail on Windows with an error
+/// "The process cannot access the file because it is being used by another process. (os error 32)".
+async fn try_many_times<F, Fut, T>(f: F) -> std::result::Result<(), T>
+where
+    F: Fn() -> Fut,
+    Fut: Future<Output = std::result::Result<(), T>>,
+{
+    let mut counter = 0;
+    loop {
+        counter += 1;
+
+        if let Err(err) = f().await {
+            if counter > 60 {
+                return Err(err);
+            }
+
+            // Wait 1 second and try again.
+            tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
+        } else {
+            break;
+        }
+    }
+    Ok(())
+}
+
 /// Configuration of a single account.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct AccountConfig {

src/chatlist.rs

@@ -761,4 +761,40 @@ mod tests {
         let summary = chats.get_summary(&t, 0, None).await.unwrap();
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_load_broken() {
+        let t = TestContext::new_bob().await;
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+            .await
+            .unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
+            .await
+            .unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
+            .await
+            .unwrap();
+
+        // check that the chatlist starts with the most recent message
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 3);
+
+        // obfuscated one chat
+        t.sql
+            .execute("UPDATE chats SET type=10 WHERE id=?", (chat_id1,))
+            .await
+            .unwrap();
+
+        // obfuscated chat can't be loaded
+        assert!(Chat::load_from_db(&t, chat_id1).await.is_err());
+
+        // chatlist loads fine
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+
+        // only corrupted chat fails to create summary
+        assert!(chats.get_summary(&t, 0, None).await.is_ok());
+        assert!(chats.get_summary(&t, 1, None).await.is_ok());
+        assert!(chats.get_summary(&t, 2, None).await.is_err());
+        assert_eq!(chats.get_index_for_id(chat_id1).unwrap(), 2);
+    }
 }

src/config.rs

@@ -266,6 +266,10 @@ pub enum Config {
     /// True if it is a bot account.
     Bot,
 
+    /// True when to skip initial start messages in groups.
+    #[strum(props(default = "0"))]
+    SkipStartMessages,
+
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
     #[strum(props(default = "0"))]
@@ -298,7 +302,7 @@ pub enum Config {
     DownloadLimit,
 
     /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set.
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "1"))]
     SyncMsgs,
 
     /// Space-separated list of all the authserv-ids which we believe

src/configure.rs

@@ -31,7 +31,6 @@ use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
-use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
 use crate::socks::Socks5Config;
 use crate::stock_str;
@@ -481,9 +480,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     ctx.set_config_bool(Config::FetchedExistingMsgs, false)
         .await?;
-    ctx.scheduler
-        .interrupt_inbox(InterruptInfo::new(false))
-        .await;
+    ctx.scheduler.interrupt_inbox().await;
 
     progress!(ctx, 940);
     update_device_chats_handle.await??;

src/contact.rs

@@ -19,7 +19,7 @@ use tokio::task;
 use tokio::time::{timeout, Duration};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::ChatId;
+use crate::chat::{ChatId, ProtectionStatus};
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
@@ -32,6 +32,7 @@ use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::sql::{self, params_iter};
+use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
     EmailAddress,
@@ -148,6 +149,22 @@ impl ContactId {
             .await?;
         Ok(())
     }
+
+    /// Reset gossip timestamp in all chats with this contact.
+    pub(crate) async fn regossip_keys(&self, context: &Context) -> Result<()> {
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+                 SET gossiped_timestamp=0
+                 WHERE EXISTS (SELECT 1 FROM chats_contacts
+                               WHERE chats_contacts.chat_id=chats.id
+                               AND chats_contacts.contact_id=?)",
+                (self,),
+            )
+            .await?;
+        Ok(())
+    }
 }
 
 impl fmt::Display for ContactId {
@@ -459,12 +476,12 @@ impl Contact {
 
     /// Block the given contact.
     pub async fn block(context: &Context, id: ContactId) -> Result<()> {
-        set_block_contact(context, id, true).await
+        set_blocked(context, Sync, id, true).await
     }
 
     /// Unblock the given contact.
     pub async fn unblock(context: &Context, id: ContactId) -> Result<()> {
-        set_block_contact(context, id, false).await
+        set_blocked(context, Sync, id, false).await
     }
 
     /// Add a single contact as a result of an _explicit_ user action.
@@ -494,7 +511,7 @@ impl Contact {
             }
         }
         if blocked {
-            Contact::unblock(context, contact_id).await?;
+            set_blocked(context, Nosync, contact_id, false).await?;
         }
 
         Ok(contact_id)
@@ -523,10 +540,24 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `may_be_valid_addr()`.
+    ///
+    /// Returns the contact ID of the contact belonging to the e-mail address or 0 if there is no
+    /// contact that is or was introduced by an accepted contact.
     pub async fn lookup_id_by_addr(
         context: &Context,
         addr: &str,
         min_origin: Origin,
+    ) -> Result<Option<ContactId>> {
+        Self::lookup_id_by_addr_ex(context, addr, min_origin, Some(Blocked::Not)).await
+    }
+
+    /// The same as `lookup_id_by_addr()`, but internal function. Currently also allows looking up
+    /// not unblocked contacts.
+    pub(crate) async fn lookup_id_by_addr_ex(
+        context: &Context,
+        addr: &str,
+        min_origin: Origin,
+        blocked: Option<Blocked>,
     ) -> Result<Option<ContactId>> {
         if addr.is_empty() {
             bail!("lookup_id_by_addr: empty address");
@@ -543,8 +574,14 @@ impl Contact {
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
-            AND id>?2 AND origin>=?3 AND blocked=0;",
-                (&addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32),
+            AND id>?2 AND origin>=?3 AND (? OR blocked=?)",
+                (
+                    &addr_normalized,
+                    ContactId::LAST_SPECIAL,
+                    min_origin as u32,
+                    blocked.is_none(),
+                    blocked.unwrap_or_default(),
+                ),
             )
             .await?;
         Ok(id)
@@ -1230,10 +1267,20 @@ impl Contact {
         self.status.as_str()
     }
 
-    /// Check if a contact was verified. E.g. by a secure-join QR code scan
-    /// and if the key has not changed since this verification.
+    /// Returns true if the contact
+    /// can be added to verified chats,
+    /// i.e. has a verified key
+    /// and Autocrypt key matches the verified key.
     ///
-    /// The UI may draw a checkbox or something like that beside verified contacts.
+    /// If contact is verified
+    /// UI should display green checkmark after the contact name
+    /// in contact list items and
+    /// in chat member list items.
+    ///
+    /// In contact profile view, us this function only if there is no chat with the contact,
+    /// otherwise use is_chat_protected().
+    /// Use [Self::get_verifier_id] to display the verifier contact
+    /// in the info section of the contact profile.
     pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
@@ -1250,16 +1297,23 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
-    /// Returns the address that verified the contact.
-    pub async fn get_verifier_addr(&self, context: &Context) -> Result<Option<String>> {
-        Ok(Peerstate::from_addr(context, self.get_addr())
-            .await?
-            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned())))
-    }
-
-    /// Returns the ContactId that verified the contact.
+    /// Returns the `ContactId` that verified the contact.
+    ///
+    /// If the function returns non-zero result,
+    /// display green checkmark in the profile and "Introduced by ..." line
+    /// with the name and address of the contact
+    /// formatted by [Self::get_name_n_addr].
+    ///
+    /// If this function returns a verifier,
+    /// this does not necessarily mean
+    /// you can add the contact to verified chats.
+    /// Use [Self::is_verified] to check
+    /// if a contact can be added to a verified chat instead.
     pub async fn get_verifier_id(&self, context: &Context) -> Result<Option<ContactId>> {
-        let Some(verifier_addr) = self.get_verifier_addr(context).await? else {
+        let Some(verifier_addr) = Peerstate::from_addr(context, self.get_addr())
+            .await?
+            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned()))
+        else {
             return Ok(None);
         };
 
@@ -1278,6 +1332,27 @@ impl Contact {
         }
     }
 
+    /// Returns if the contact profile title should display a green checkmark.
+    ///
+    /// This generally should be consistent with the 1:1 chat with the contact
+    /// so 1:1 chat with the contact and the contact profile
+    /// either both display the green checkmark or both don't display a green checkmark.
+    ///
+    /// UI often knows beforehand if a chat exists and can also call
+    /// `chat.is_protected()` (if there is a chat)
+    /// or `contact.is_verified()` (if there is no chat) directly.
+    /// This is often easier and also skips some database calls.
+    pub async fn is_profile_verified(&self, context: &Context) -> Result<bool> {
+        let contact_id = self.id;
+
+        if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
+            Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
+        } else {
+            // 1:1 chat does not exist.
+            Ok(self.is_verified(context).await? == VerifiedStatus::BidirectVerified)
+        }
+    }
+
     /// Returns the number of real (i.e. non-special) contacts in the database.
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
@@ -1363,8 +1438,9 @@ fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     }
 }
 
-async fn set_block_contact(
+pub(crate) async fn set_blocked(
     context: &Context,
+    sync: sync::Sync,
     contact_id: ContactId,
     new_blocking: bool,
 ) -> Result<()> {
@@ -1373,7 +1449,6 @@ async fn set_block_contact(
         "Can't block special contact {}",
         contact_id
     );
-
     let contact = Contact::get_by_id(context, contact_id).await?;
 
     if contact.blocked != new_blocking {
@@ -1415,9 +1490,23 @@ WHERE type=? AND id IN (
             if let Some((chat_id, _, _)) =
                 chat::get_chat_id_by_grpid(context, &contact.addr).await?
             {
-                chat_id.unblock(context).await?;
+                chat_id.unblock_ex(context, Nosync).await?;
             }
         }
+
+        if sync.into() {
+            let action = match new_blocking {
+                true => chat::SyncAction::Block,
+                false => chat::SyncAction::Unblock,
+            };
+            context
+                .add_sync_item(SyncData::AlterChat {
+                    id: chat::SyncId::ContactAddr(contact.addr.clone()),
+                    action,
+                })
+                .await?;
+            context.send_sync_msg().await?;
+        }
     }
 
     Ok(())
@@ -2710,7 +2799,6 @@ Hi."#;
 
         let contact_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
         let contact = Contact::get_by_id(&alice, contact_id).await?;
-        assert!(contact.get_verifier_addr(&alice).await?.is_none());
         assert!(contact.get_verifier_id(&alice).await?.is_none());
 
         // Receive a message from Bob to create a peerstate.
@@ -2719,7 +2807,6 @@ Hi."#;
         alice.recv_msg(&sent_msg).await;
 
         let contact = Contact::get_by_id(&alice, contact_id).await?;
-        assert!(contact.get_verifier_addr(&alice).await?.is_none());
         assert!(contact.get_verifier_id(&alice).await?.is_none());
 
         Ok(())

src/context.rs

@@ -23,7 +23,7 @@ use crate::key::{load_self_public_key, DcKey as _};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
-use crate::scheduler::{InterruptInfo, SchedulerState};
+use crate::scheduler::SchedulerState;
 use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
@@ -398,11 +398,24 @@ impl Context {
     }
 
     /// Starts the IO scheduler.
-    pub async fn start_io(&self) {
-        if let Ok(false) = self.is_configured().await {
+    pub async fn start_io(&mut self) {
+        if !self.is_configured().await.unwrap_or_default() {
             warn!(self, "can not start io on a context that is not configured");
             return;
         }
+
+        {
+            if self
+                .get_config(Config::ConfiguredAddr)
+                .await
+                .unwrap_or_default()
+                .filter(|s| s.ends_with(".testrun.org"))
+                .is_some()
+            {
+                let mut lock = self.ratelimit.write().await;
+                *lock = Ratelimit::new(Duration::new(40, 0), 6.0);
+            }
+        }
         self.scheduler.start(self.clone()).await;
     }
 
@@ -424,11 +437,7 @@ impl Context {
 
     pub(crate) async fn schedule_resync(&self) -> Result<()> {
         self.resync_request.store(true, Ordering::Relaxed);
-        self.scheduler
-            .interrupt_inbox(InterruptInfo {
-                probe_network: false,
-            })
-            .await;
+        self.scheduler.interrupt_inbox().await;
         Ok(())
     }
 
@@ -1302,6 +1311,7 @@ mod tests {
             "send_port",
             "send_security",
             "server_flags",
+            "skip_start_messages",
             "smtp_certificate_checks",
             "socks5_host",
             "socks5_port",

src/download.rs

@@ -12,18 +12,16 @@ use crate::context::Context;
 use crate::imap::{Imap, ImapActionResult};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
-use crate::scheduler::InterruptInfo;
 use crate::tools::time;
 use crate::{stock_str, EventType};
 
 /// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
 ///
-/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
-/// need to be downloaded completely to handle them correctly,
-/// eg. to assign them to the correct chat.
-/// As these messages are typically small,
-/// they're caught by `MIN_DOWNLOAD_LIMIT`.
-pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN)
+/// should always be downloaded completely to handle them correctly,
+/// also in larger groups and if group and contact avatar are attached.
+/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`.
+pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840;
 
 /// If a message is downloaded only partially
 /// and `delete_server_after` is set to small timeouts (eg. "at once"),
@@ -94,10 +92,7 @@ impl MsgId {
                     .sql
                     .execute("INSERT INTO download (msg_id) VALUES (?)", (self,))
                     .await?;
-                context
-                    .scheduler
-                    .interrupt_inbox(InterruptInfo::new(false))
-                    .await;
+                context.scheduler.interrupt_inbox().await;
             }
         }
         Ok(())

src/e2ee.rs

@@ -67,13 +67,8 @@ impl EncryptHelper {
                         "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
                     );
                     match peerstate.prefer_encrypt {
-                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::NoPreference | EncryptPreference::Reset => {}
                         EncryptPreference::Mutual => prefer_encrypt_count += 1,
-                        EncryptPreference::Reset => {
-                            if !e2ee_guaranteed {
-                                return Ok(false);
-                            }
-                        }
                     };
                 }
                 None => {
@@ -105,16 +100,39 @@ impl EncryptHelper {
     ) -> Result<String> {
         let mut keyring: Vec<SignedPublicKey> = Vec::new();
 
+        let mut verifier_addresses: Vec<&str> = Vec::new();
+
         for (peerstate, addr) in peerstates
-            .into_iter()
-            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
+            .iter()
+            .filter_map(|(state, addr)| state.clone().map(|s| (s, addr)))
         {
             let key = peerstate
                 .take_key(min_verified)
                 .with_context(|| format!("proper enc-key for {addr} missing, cannot encrypt"))?;
             keyring.push(key);
+            verifier_addresses.push(addr);
         }
+
+        // Encrypt to self.
         keyring.push(self.public_key.clone());
+
+        // Encrypt to secondary verified keys
+        // if we also encrypt to the introducer ("verifier") of the key.
+        if min_verified == PeerstateVerifiedStatus::BidirectVerified {
+            for (peerstate, _addr) in peerstates {
+                if let Some(peerstate) = peerstate {
+                    if let (Some(key), Some(verifier)) = (
+                        peerstate.secondary_verified_key.as_ref(),
+                        peerstate.secondary_verifier.as_deref(),
+                    ) {
+                        if verifier_addresses.contains(&verifier) {
+                            keyring.push(key.clone());
+                        }
+                    }
+                }
+            }
+        }
+
         let sign_key = load_self_secret_key(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
@@ -301,8 +319,11 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             gossip_key_fingerprint: Some(pub_key.fingerprint()),
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
         vec![(Some(peerstate), addr)]
     }

src/events.rs

@@ -1,6 +1,7 @@
 //! # Events specification.
 
 use async_channel::{self as channel, Receiver, Sender, TrySendError};
+use pin_project::pin_project;
 
 mod payload;
 
@@ -64,7 +65,8 @@ impl Events {
 /// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
 /// [`Stream`]: futures::stream::Stream
 #[derive(Debug, Clone)]
-pub struct EventEmitter(Receiver<Event>);
+#[pin_project]
+pub struct EventEmitter(#[pin] Receiver<Event>);
 
 impl EventEmitter {
     /// Async recv of an event. Return `None` if the `Sender` has been dropped.
@@ -77,10 +79,10 @@ impl futures::stream::Stream for EventEmitter {
     type Item = Event;
 
     fn poll_next(
-        mut self: std::pin::Pin<&mut Self>,
+        self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self.0).poll_next(cx)
+        self.project().0.poll_next(cx)
     }
 }
 

src/events/payload.rs

@@ -250,6 +250,7 @@ pub enum EventType {
         /// Progress as:
         /// 400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
         /// (Bob has verified alice and waits until Alice does the same for him)
+        /// 1000=vg-member-added/vc-contact-confirm received
         progress: usize,
     },
 

src/imap.rs

@@ -35,7 +35,6 @@ use crate::receive_imf::{
     from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner, ReceivedMsg,
 };
 use crate::scheduler::connectivity::ConnectivityStore;
-use crate::scheduler::InterruptInfo;
 use crate::socks::Socks5Config;
 use crate::sql;
 use crate::stock_str;
@@ -86,7 +85,7 @@ const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
 
 #[derive(Debug)]
 pub struct Imap {
-    pub(crate) idle_interrupt_receiver: Receiver<InterruptInfo>,
+    pub(crate) idle_interrupt_receiver: Receiver<()>,
     config: ImapConfig,
     pub(crate) session: Option<Session>,
     login_failed_once: bool,
@@ -195,7 +194,7 @@ impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
 
                 while let Some((next_rowid, next_uid, _)) =
                     self.inner.next_if(|(_, next_uid, next_folder)| {
-                        next_folder == &folder && *next_uid == end_uid + 1
+                        next_folder == &folder && (*next_uid == end_uid + 1 || *next_uid == end_uid)
                     })
                 {
                     end_uid = next_uid;
@@ -228,7 +227,7 @@ impl Imap {
         socks5_config: Option<Socks5Config>,
         addr: &str,
         provider_strict_tls: bool,
-        idle_interrupt_receiver: Receiver<InterruptInfo>,
+        idle_interrupt_receiver: Receiver<()>,
     ) -> Result<Self> {
         if lp.server.is_empty() || lp.user.is_empty() || lp.password.is_empty() {
             bail!("Incomplete IMAP connection parameters");
@@ -261,7 +260,7 @@ impl Imap {
     /// Creates new disconnected IMAP client using configured parameters.
     pub async fn new_configured(
         context: &Context,
-        idle_interrupt_receiver: Receiver<InterruptInfo>,
+        idle_interrupt_receiver: Receiver<()>,
     ) -> Result<Self> {
         if !context.is_configured().await? {
             bail!("IMAP Connect without configured params");
@@ -568,9 +567,14 @@ impl Imap {
         Ok(())
     }
 
-    /// Select a folder and take care of uidvalidity changes.
-    /// Also, when selecting a folder for the first time, sets the uid_next to the current
+    /// Selects a folder and takes care of UIDVALIDITY changes.
+    ///
+    /// When selecting a folder for the first time, sets the uid_next to the current
     /// mailbox.uid_next so that no old emails are fetched.
+    ///
+    /// Makes sure that UIDNEXT is known for `selected_mailbox`
+    /// and errors out if UIDNEXT cannot be determined.
+    ///
     /// Returns Result<new_emails> (i.e. whether new emails arrived),
     /// if in doubt, returns new_emails=true so emails are fetched.
     pub(crate) async fn select_with_uidvalidity(
@@ -591,6 +595,37 @@ impl Imap {
         let new_uid_validity = mailbox
             .uid_validity
             .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
+        let new_uid_next = if let Some(uid_next) = mailbox.uid_next {
+            uid_next
+        } else {
+            warn!(
+                context,
+                "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
+            );
+
+            // RFC 3501 says STATUS command SHOULD NOT be used
+            // on the currently selected mailbox because the same
+            // information can be obtained by other means,
+            // such as reading SELECT response.
+            //
+            // However, it also says that UIDNEXT is REQUIRED
+            // in the SELECT response and if we are here,
+            // it is actually not returned.
+            //
+            // In particular, Winmail Pro Mail Server 5.1.0616
+            // never returns UIDNEXT in SELECT response,
+            // but responds to "STATUS INBOX (UIDNEXT)" command.
+            let status = session
+                .inner
+                .status(folder, "(UIDNEXT)")
+                .await
+                .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
+
+            status
+                .uid_next
+                .with_context(|| format!("STATUS {folder} (UIDNEXT) did not return UIDNEXT"))?
+        };
+        mailbox.uid_next = Some(new_uid_next);
 
         let old_uid_validity = get_uidvalidity(context, folder)
             .await
@@ -606,18 +641,16 @@ impl Imap {
                 // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
                 // new messages is only one command, just as a SELECT command)
                 true
-            } else if let Some(uid_next) = mailbox.uid_next {
-                if uid_next < old_uid_next {
+            } else {
+                if new_uid_next < old_uid_next {
                     warn!(
                         context,
-                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
+                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
                     );
-                    set_uid_next(context, folder, uid_next).await?;
+                    set_uid_next(context, folder, new_uid_next).await?;
                     context.schedule_resync().await?;
                 }
-                uid_next != old_uid_next // If uid_next changed, there are new emails
-            } else {
-                true // We have no uid_next and if in doubt, return true
+                new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
             };
             return Ok(new_emails);
         }
@@ -627,43 +660,6 @@ impl Imap {
 
         // ==============  uid_validity has changed or is being set the first time.  ==============
 
-        let new_uid_next = match mailbox.uid_next {
-            Some(uid_next) => uid_next,
-            None => {
-                warn!(
-                    context,
-                    "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
-                );
-
-                // RFC 3501 says STATUS command SHOULD NOT be used
-                // on the currently selected mailbox because the same
-                // information can be obtained by other means,
-                // such as reading SELECT response.
-                //
-                // However, it also says that UIDNEXT is REQUIRED
-                // in the SELECT response and if we are here,
-                // it is actually not returned.
-                //
-                // In particular, Winmail Pro Mail Server 5.1.0616
-                // never returns UIDNEXT in SELECT response,
-                // but responds to "SELECT INBOX (UIDNEXT)" command.
-                let status = session
-                    .inner
-                    .status(folder, "(UIDNEXT)")
-                    .await
-                    .context("STATUS (UIDNEXT) error for {folder:?}")?;
-
-                if let Some(uid_next) = status.uid_next {
-                    uid_next
-                } else {
-                    warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT");
-
-                    // Set UIDNEXT to 1 as a last resort fallback.
-                    1
-                }
-            }
-        };
-
         set_uid_next(context, folder, new_uid_next).await?;
         set_uidvalidity(context, folder, new_uid_validity).await?;
 
@@ -867,14 +863,28 @@ impl Imap {
             uids_fetch_in_batch.push(uid);
         }
 
-        // determine which uid_next to use to update to
-        // receive_imf() returns an `Err` value only on recoverable errors, otherwise it just logs an error.
-        // `largest_uid_processed` is the largest uid where receive_imf() did NOT return an error.
-
-        // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was
-        // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times.
-        let largest_uid_without_errors = max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0));
-        let new_uid_next = largest_uid_without_errors + 1;
+        // Advance uid_next to the maximum of the largest known UID plus 1
+        // and mailbox UIDNEXT.
+        // Largest known UID is normally less than UIDNEXT,
+        // but a message may have arrived between determining UIDNEXT
+        // and executing the FETCH command.
+        let mailbox_uid_next = self
+            .session
+            .as_ref()
+            .context("No IMAP session")?
+            .selected_mailbox
+            .as_ref()
+            .with_context(|| format!("Expected {folder:?} to be selected"))?
+            .uid_next
+            .with_context(|| {
+                format!(
+                    "Expected UIDNEXT to be determined for {folder:?} by select_with_uidvalidity"
+                )
+            })?;
+        let new_uid_next = max(
+            max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0)) + 1,
+            mailbox_uid_next,
+        );
 
         if new_uid_next > old_uid_next {
             set_uid_next(context, folder, new_uid_next).await?;
@@ -2279,10 +2289,7 @@ pub(crate) async fn markseen_on_imap_table(context: &Context, message_id: &str)
             (message_id,),
         )
         .await?;
-    context
-        .scheduler
-        .interrupt_inbox(InterruptInfo::new(false))
-        .await;
+    context.scheduler.interrupt_inbox().await;
 
     Ok(())
 }
@@ -2830,4 +2837,31 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_uid_grouper() {
+        // Input: sequence of (rowid: i64, uid: u32, target: String)
+        // Output: sequence of (target: String, rowid_set: Vec<i64>, uid_set: String)
+        let grouper = UidGrouper::from([(1, 2, "INBOX".to_string())]);
+        let res: Vec<(String, Vec<i64>, String)> = grouper.into_iter().collect();
+        assert_eq!(res, vec![("INBOX".to_string(), vec![1], "2".to_string())]);
+
+        let grouper = UidGrouper::from([(1, 2, "INBOX".to_string()), (2, 3, "INBOX".to_string())]);
+        let res: Vec<(String, Vec<i64>, String)> = grouper.into_iter().collect();
+        assert_eq!(
+            res,
+            vec![("INBOX".to_string(), vec![1, 2], "2:3".to_string())]
+        );
+
+        let grouper = UidGrouper::from([
+            (1, 2, "INBOX".to_string()),
+            (2, 2, "INBOX".to_string()),
+            (3, 3, "INBOX".to_string()),
+        ]);
+        let res: Vec<(String, Vec<i64>, String)> = grouper.into_iter().collect();
+        assert_eq!(
+            res,
+            vec![("INBOX".to_string(), vec![1, 2, 3], "2:3".to_string())]
+        );
+    }
 }

src/imap/idle.rs

@@ -8,9 +8,9 @@ use futures_lite::FutureExt;
 use super::session::Session;
 use super::Imap;
 use crate::config::Config;
-use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
+use crate::context::Context;
+use crate::imap::{client::IMAP_TIMEOUT, get_uid_next, FolderMeaning};
 use crate::log::LogExt;
-use crate::{context::Context, scheduler::InterruptInfo};
 
 const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
 
@@ -18,30 +18,43 @@ impl Session {
     pub async fn idle(
         mut self,
         context: &Context,
-        idle_interrupt_receiver: Receiver<InterruptInfo>,
-        watch_folder: Option<String>,
-    ) -> Result<(Self, InterruptInfo)> {
+        idle_interrupt_receiver: Receiver<()>,
+        folder: &str,
+    ) -> Result<Self> {
         use futures::future::FutureExt;
 
-        if context.get_config_bool(Config::DisableIdle).await? {
-            bail!("IMAP IDLE is disabled");
-        }
-
-        if !self.can_idle() {
-            bail!("IMAP server does not have IDLE capability");
-        }
-
-        let mut info = Default::default();
-
-        self.select_folder(context, watch_folder.as_deref()).await?;
+        self.select_folder(context, Some(folder)).await?;
 
         if self.server_sent_unsolicited_exists(context)? {
-            return Ok((self, info));
+            return Ok(self);
         }
 
-        if let Ok(info) = idle_interrupt_receiver.try_recv() {
-            info!(context, "skip idle, got interrupt {:?}", info);
-            return Ok((self, info));
+        // Despite checking for unsolicited EXISTS above,
+        // we may have missed EXISTS if the message was
+        // received when the folder was not selected.
+        let status = self
+            .status(folder, "(UIDNEXT)")
+            .await
+            .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
+        if let Some(uid_next) = status.uid_next {
+            let expected_uid_next = get_uid_next(context, folder)
+                .await
+                .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
+            if uid_next > expected_uid_next {
+                info!(
+                    context,
+                    "Skipping IDLE on {folder:?} because UIDNEXT {uid_next}>{expected_uid_next} indicates there are new messages."
+                );
+                return Ok(self);
+            }
+        } else {
+            warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT");
+            // Go to IDLE anyway if STATUS is broken.
+        }
+
+        if let Ok(()) = idle_interrupt_receiver.try_recv() {
+            info!(context, "skip idle, got interrupt");
+            return Ok(self);
         }
 
         let mut handle = self.inner.idle();
@@ -58,59 +71,45 @@ impl Session {
 
         enum Event {
             IdleResponse(IdleResponse),
-            Interrupt(InterruptInfo),
+            Interrupt,
         }
 
-        let folder_name = watch_folder.as_deref().unwrap_or("None");
-        info!(
-            context,
-            "{}: Idle entering wait-on-remote state", folder_name
-        );
+        info!(context, "{folder}: Idle entering wait-on-remote state");
         let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
-            let info = idle_interrupt_receiver.recv().await;
+            idle_interrupt_receiver.recv().await.ok();
 
             // cancel imap idle connection properly
             drop(interrupt);
 
-            Ok(Event::Interrupt(info.unwrap_or_default()))
+            Ok(Event::Interrupt)
         });
 
         match fut.await {
             Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
-                info!(context, "{}: Idle has NewData {:?}", folder_name, x);
+                info!(context, "{folder}: Idle has NewData {:?}", x);
             }
             Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                info!(
-                    context,
-                    "{}: Idle-wait timeout or interruption", folder_name
-                );
+                info!(context, "{folder}: Idle-wait timeout or interruption");
             }
             Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                info!(
-                    context,
-                    "{}: Idle wait was interrupted manually", folder_name
-                );
+                info!(context, "{folder}: Idle wait was interrupted manually");
             }
-            Ok(Event::Interrupt(i)) => {
-                info!(
-                    context,
-                    "{}: Idle wait was interrupted: {:?}", folder_name, &i
-                );
-                info = i;
+            Ok(Event::Interrupt) => {
+                info!(context, "{folder}: Idle wait was interrupted");
             }
             Err(err) => {
-                warn!(context, "{}: Idle wait errored: {:?}", folder_name, err);
+                warn!(context, "{folder}: Idle wait errored: {err:?}");
             }
         }
 
         let mut session = tokio::time::timeout(Duration::from_secs(15), handle.done())
             .await
-            .with_context(|| format!("{folder_name}: IMAP IDLE protocol timed out"))?
-            .with_context(|| format!("{folder_name}: IMAP IDLE failed"))?;
+            .with_context(|| format!("{folder}: IMAP IDLE protocol timed out"))?
+            .with_context(|| format!("{folder}: IMAP IDLE failed"))?;
         session.as_mut().set_read_timeout(Some(IMAP_TIMEOUT));
         self.inner = session;
 
-        Ok((self, info))
+        Ok(self)
     }
 }
 
@@ -120,7 +119,7 @@ impl Imap {
         context: &Context,
         watch_folder: Option<String>,
         folder_meaning: FolderMeaning,
-    ) -> InterruptInfo {
+    ) {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
@@ -131,32 +130,33 @@ impl Imap {
             watch_folder
         } else {
             info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
-            return self
-                .idle_interrupt_receiver
-                .recv()
-                .await
-                .unwrap_or_default();
+            self.idle_interrupt_receiver.recv().await.ok();
+            return;
         };
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-        // check every minute if there are new messages
-        // TODO: grow sleep durations / make them more flexible
-        let mut interval = tokio::time::interval(Duration::from_secs(60));
-
+        const TIMEOUT_INIT_MS: u64 = 60_000;
+        let mut timeout_ms: u64 = TIMEOUT_INIT_MS;
         enum Event {
             Tick,
-            Interrupt(InterruptInfo),
+            Interrupt,
         }
         // loop until we are interrupted or if we fetched something
-        let info = loop {
+        loop {
             use futures::future::FutureExt;
+            use rand::Rng;
+
+            let mut interval = tokio::time::interval(Duration::from_millis(timeout_ms));
+            timeout_ms = timeout_ms
+                .saturating_add(rand::thread_rng().gen_range((timeout_ms / 2)..=timeout_ms));
+            interval.tick().await; // The first tick completes immediately.
             match interval
                 .tick()
                 .map(|_| Event::Tick)
                 .race(
                     self.idle_interrupt_receiver
                         .recv()
-                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                        .map(|_| Event::Interrupt),
                 )
                 .await
             {
@@ -178,7 +178,7 @@ impl Imap {
                                 .unwrap_or_default()
                         {
                             // we only fake-idled because network was gone during IDLE, probably
-                            break InterruptInfo::new(false);
+                            break;
                         }
                     }
                     info!(context, "fake_idle is connected");
@@ -192,8 +192,9 @@ impl Imap {
                     {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
+                            timeout_ms = TIMEOUT_INIT_MS;
                             if res {
-                                break InterruptInfo::new(false);
+                                break;
                             }
                         }
                         Err(err) => {
@@ -202,13 +203,12 @@ impl Imap {
                         }
                     }
                 }
-                Event::Interrupt(info) => {
-                    // Interrupt
+                Event::Interrupt => {
                     info!(context, "Fake IDLE interrupted");
-                    break info;
+                    break;
                 }
             }
-        };
+        }
 
         info!(
             context,
@@ -219,7 +219,5 @@ impl Imap {
                 .as_millis() as f64
                 / 1000.,
         );
-
-        info
     }
 }

src/message.rs

@@ -24,7 +24,6 @@ use crate::mimeparser::{parse_message_id, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
 use crate::reaction::get_msg_reactions;
-use crate::scheduler::InterruptInfo;
 use crate::sql;
 use crate::summary::Summary;
 use crate::tools::{
@@ -150,21 +149,53 @@ WHERE id=?;
         self.0
     }
 
+    /// Returns raw text of a message, used for message info
+    pub async fn rawtext(self, context: &Context) -> Result<String> {
+        Ok(context
+            .sql
+            .query_get_value("SELECT txt_raw FROM msgs WHERE id=?", (self,))
+            .await?
+            .unwrap_or_default())
+    }
+
+    /// Returns server foldernames and UIDs of a message, used for message info
+    pub async fn get_info_server_urls(
+        context: &Context,
+        rfc724_mid: String,
+    ) -> Result<Vec<String>> {
+        context
+            .sql
+            .query_map(
+                "SELECT folder, uid FROM imap WHERE rfc724_mid=?",
+                (rfc724_mid,),
+                |row| {
+                    let folder: String = row.get("folder")?;
+                    let uid: u32 = row.get("uid")?;
+                    Ok(format!("</{folder}/;UID={uid}>"))
+                },
+                |rows| {
+                    rows.collect::<std::result::Result<Vec<_>, _>>()
+                        .map_err(Into::into)
+                },
+            )
+            .await
+    }
+
+    /// Returns information about hops of a message, used for message info
+    pub async fn hop_info(self, context: &Context) -> Result<Option<String>> {
+        context
+            .sql
+            .query_get_value("SELECT hop_info FROM msgs WHERE id=?", (self,))
+            .await
+    }
+
     /// Returns detailed message information in a multi-line text form.
     pub async fn get_info(self, context: &Context) -> Result<String> {
         let msg = Message::load_from_db(context, self).await?;
-        let rawtxt: Option<String> = context
-            .sql
-            .query_get_value("SELECT txt_raw FROM msgs WHERE id=?", (self,))
-            .await?;
+        let rawtxt: String = self.rawtext(context).await?;
 
         let mut ret = String::new();
 
-        if rawtxt.is_none() {
-            ret += &format!("Cannot load message {self}.");
-            return Ok(ret);
-        }
-        let rawtxt = rawtxt.unwrap_or_default();
         let rawtxt = truncate(rawtxt.trim(), DC_DESIRED_TEXT_LEN);
 
         let fts = timestamp_to_str(msg.get_timestamp());
@@ -282,32 +313,13 @@ WHERE id=?;
         if !msg.rfc724_mid.is_empty() {
             ret += &format!("\nMessage-ID: {}", msg.rfc724_mid);
 
-            let server_uids = context
-                .sql
-                .query_map(
-                    "SELECT folder, uid FROM imap WHERE rfc724_mid=?",
-                    (msg.rfc724_mid,),
-                    |row| {
-                        let folder: String = row.get("folder")?;
-                        let uid: u32 = row.get("uid")?;
-                        Ok((folder, uid))
-                    },
-                    |rows| {
-                        rows.collect::<std::result::Result<Vec<_>, _>>()
-                            .map_err(Into::into)
-                    },
-                )
-                .await?;
-
-            for (folder, uid) in server_uids {
+            let server_urls = Self::get_info_server_urls(context, msg.rfc724_mid).await?;
+            for server_url in server_urls {
                 // Format as RFC 5092 relative IMAP URL.
-                ret += &format!("\n</{folder}/;UID={uid}>");
+                ret += &format!("\n{server_url}");
             }
         }
-        let hop_info: Option<String> = context
-            .sql
-            .query_get_value("SELECT hop_info FROM msgs WHERE id=?;", (self,))
-            .await?;
+        let hop_info = self.hop_info(context).await?;
 
         ret += "\n\n";
         ret += &hop_info.unwrap_or_else(|| "No Hop Info".to_owned());
@@ -649,6 +661,12 @@ impl Message {
         self.id
     }
 
+    /// Returns the rfc724 message ID
+    /// May be empty
+    pub fn rfc724_mid(&self) -> &str {
+        &self.rfc724_mid
+    }
+
     /// Returns the ID of the contact who wrote the message.
     pub fn get_from_id(&self) -> ContactId {
         self.from_id
@@ -1508,10 +1526,7 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     }
 
     // Interrupt Inbox loop to start message deletion and run housekeeping.
-    context
-        .scheduler
-        .interrupt_inbox(InterruptInfo::new(false))
-        .await;
+    context.scheduler.interrupt_inbox().await;
     Ok(())
 }
 
@@ -1629,10 +1644,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
                         )
                         .await
                         .context("failed to insert into smtp_mdns")?;
-                    context
-                        .scheduler
-                        .interrupt_smtp(InterruptInfo::new(false))
-                        .await;
+                    context.scheduler.interrupt_smtp().await;
                 }
             }
             updated_chat_ids.insert(curr_chat_id);
@@ -1799,6 +1811,14 @@ pub async fn estimate_deletion_cnt(
 pub(crate) async fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
+) -> Result<Option<MsgId>> {
+    rfc724_mid_exists_and(context, rfc724_mid, "1").await
+}
+
+pub(crate) async fn rfc724_mid_exists_and(
+    context: &Context,
+    rfc724_mid: &str,
+    cond: &str,
 ) -> Result<Option<MsgId>> {
     let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>');
     if rfc724_mid.is_empty() {
@@ -1809,7 +1829,7 @@ pub(crate) async fn rfc724_mid_exists(
     let res = context
         .sql
         .query_row_optional(
-            "SELECT id FROM msgs WHERE rfc724_mid=?",
+            &("SELECT id FROM msgs WHERE rfc724_mid=? AND ".to_string() + cond),
             (rfc724_mid,),
             |row| {
                 let msg_id: MsgId = row.get(0)?;

src/mimefactory.rs

@@ -14,7 +14,7 @@ use crate::chat::Chat;
 use crate::config::Config;
 use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
 use crate::contact::Contact;
-use crate::context::{get_version_str, Context};
+use crate::context::Context;
 use crate::e2ee::EncryptHelper;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::html::new_html_mimepart;
@@ -316,7 +316,15 @@ impl<'a> MimeFactory<'a> {
         match &self.loaded {
             Loaded::Message { chat } => {
                 if chat.is_protected() {
-                    PeerstateVerifiedStatus::BidirectVerified
+                    if self.msg.get_info_type() == SystemMessage::SecurejoinMessage {
+                        // Securejoin messages are supposed to verify a key.
+                        // In order to do this, it is necessary that they can be sent
+                        // to a key that is not yet verified.
+                        // This has to work independently of whether the chat is protected right now.
+                        PeerstateVerifiedStatus::Unverified
+                    } else {
+                        PeerstateVerifiedStatus::BidirectVerified
+                    }
                 } else {
                     PeerstateVerifiedStatus::Unverified
                 }
@@ -596,9 +604,10 @@ impl<'a> MimeFactory<'a> {
 
         if let Loaded::Message { chat } = &self.loaded {
             if chat.typ == Chattype::Broadcast {
+                let encoded_chat_name = encode_words(&chat.name);
                 headers.protected.push(Header::new(
                     "List-ID".into(),
-                    format!("{} <{}>", chat.name, chat.grpid),
+                    format!("{encoded_chat_name} <{}>", chat.grpid),
                 ));
             }
         }
@@ -1357,14 +1366,12 @@ impl<'a> MimeFactory<'a> {
         );
 
         // second body part: machine-readable, always REQUIRED by RFC 6522
-        let version = get_version_str();
         let message_text2 = format!(
-            "Reporting-UA: Delta Chat {}\r\n\
-             Original-Recipient: rfc822;{}\r\n\
+            "Original-Recipient: rfc822;{}\r\n\
              Final-Recipient: rfc822;{}\r\n\
              Original-Message-ID: <{}>\r\n\
              Disposition: manual-action/MDN-sent-automatically; displayed\r\n",
-            version, self.from_addr, self.from_addr, self.msg.rfc724_mid
+            self.from_addr, self.from_addr, self.msg.rfc724_mid
         );
 
         let extension_fields = if additional_msg_ids.is_empty() {

src/mimeparser.rs

@@ -59,7 +59,7 @@ pub(crate) struct MimeMessage {
     /// Message headers.
     headers: HashMap<String, String>,
 
-    /// Addresses are normalized and lowercased:
+    /// Addresses are normalized and lowercase
     pub recipients: Vec<SingleInfo>,
 
     /// `From:` address.
@@ -163,10 +163,10 @@ pub enum SystemMessage {
     /// Chat ephemeral message timer is changed.
     EphemeralTimerChanged = 10,
 
-    /// Chat protection is enabled.
+    /// "Messages are guaranteed to be end-to-end encrypted from now on."
     ChatProtectionEnabled = 11,
 
-    /// Chat protection is disabled.
+    /// "%1$s sent a message from another device."
     ChatProtectionDisabled = 12,
 
     /// Self-sent-message that contains only json used for multi-device-sync;
@@ -376,6 +376,12 @@ impl MimeMessage {
         if !encrypted {
             signatures.clear();
         }
+        if let Some(peerstate) = &mut decryption_info.peerstate {
+            if peerstate.prefer_encrypt != EncryptPreference::Mutual && !signatures.is_empty() {
+                peerstate.prefer_encrypt = EncryptPreference::Mutual;
+                peerstate.save_to_db(&context.sql).await?;
+            }
+        }
 
         // Auto-submitted is also set by holiday-notices so we also check `chat-version`
         let is_bot = headers.contains_key("auto-submitted") && headers.contains_key("chat-version");
@@ -1358,7 +1364,7 @@ impl MimeMessage {
         self.get_mailinglist_header().is_some()
     }
 
-    pub fn repl_msg_by_error(&mut self, error_msg: &str) {
+    pub fn replace_msg_by_error(&mut self, error_msg: &str) {
         self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
             part.typ = Viewtype::Text;

src/net.rs

@@ -1,10 +1,11 @@
 //! # Common network utilities.
+use std::net::Ipv4Addr;
 use std::net::{IpAddr, SocketAddr};
 use std::pin::Pin;
 use std::str::FromStr;
 use std::time::Duration;
 
-use anyhow::{Context as _, Error, Result};
+use anyhow::{format_err, Context as _, Result};
 use tokio::net::{lookup_host, TcpStream};
 use tokio::time::timeout;
 use tokio_io_timeout::TimeoutStream;
@@ -119,6 +120,22 @@ async fn lookup_host_with_cache(
                 }
             }
         }
+
+        if resolved_addrs.is_empty() {
+            // Load hardcoded cache if everything else fails.
+            //
+            // See <https://support.delta.chat/t/no-dns-resolution-result/2778> and
+            // <https://github.com/deltachat/deltachat-core-rust/issues/4920> for reasons.
+            //
+            // In the future we may pre-resolve all provider database addresses
+            // and build them in.
+            if hostname == "mail.sangham.net" {
+                resolved_addrs.push(SocketAddr::new(
+                    IpAddr::V4(Ipv4Addr::new(159, 69, 186, 85)),
+                    port,
+                ));
+            }
+        }
     }
 
     Ok(resolved_addrs)
@@ -178,7 +195,9 @@ pub(crate) async fn connect_tcp(
     let tcp_stream = match tcp_stream {
         Some(tcp_stream) => tcp_stream,
         None => {
-            return Err(last_error.unwrap_or_else(|| Error::msg("no DNS resolution results")));
+            return Err(
+                last_error.unwrap_or_else(|| format_err!("no DNS resolution results for {host}"))
+            );
         }
     };
 

src/peerstate.rs

@@ -1,7 +1,5 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module.
 
-use std::collections::HashSet;
-
 use anyhow::{Context as _, Error, Result};
 use num_traits::FromPrimitive;
 
@@ -40,7 +38,7 @@ pub enum PeerstateVerifiedStatus {
 }
 
 /// Peerstate represents the state of an Autocrypt peer.
-#[derive(Debug, PartialEq, Eq)]
+#[derive(Debug, PartialEq, Eq, Clone)]
 pub struct Peerstate {
     /// E-mail address of the contact.
     pub addr: String,
@@ -82,13 +80,24 @@ pub struct Peerstate {
     /// Fingerprint of the verified public key.
     pub verified_key_fingerprint: Option<Fingerprint>,
 
+    /// The address that introduced this verified key.
+    pub verifier: Option<String>,
+
+    /// Secondary public verified key of the contact.
+    /// It could be a contact gossiped by another verified contact in a shared group
+    /// or a key that was previously used as a verified key.
+    pub secondary_verified_key: Option<SignedPublicKey>,
+
+    /// Fingerprint of the secondary verified public key.
+    pub secondary_verified_key_fingerprint: Option<Fingerprint>,
+
+    /// The address that introduced secondary verified key.
+    pub secondary_verifier: Option<String>,
+
     /// True if it was detected
     /// that the fingerprint of the key used in chats with
     /// opportunistic encryption was changed after Peerstate creation.
     pub fingerprint_changed: bool,
-
-    /// The address that verified this contact
-    pub verifier: Option<String>,
 }
 
 impl Peerstate {
@@ -106,8 +115,11 @@ impl Peerstate {
             gossip_timestamp: 0,
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         }
     }
 
@@ -132,8 +144,11 @@ impl Peerstate {
             gossip_timestamp: message_time,
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         }
     }
 
@@ -141,7 +156,10 @@ impl Peerstate {
     pub async fn from_addr(context: &Context, addr: &str) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint, verifier \
+                     verified_key, verified_key_fingerprint, \
+                     verifier, \
+                     secondary_verified_key, secondary_verified_key_fingerprint, \
+                     secondary_verifier \
                      FROM acpeerstates \
                      WHERE addr=? COLLATE NOCASE LIMIT 1;";
         Self::from_stmt(context, query, (addr,)).await
@@ -154,7 +172,10 @@ impl Peerstate {
     ) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint, verifier \
+                     verified_key, verified_key_fingerprint, \
+                     verifier, \
+                     secondary_verified_key, secondary_verified_key_fingerprint, \
+                     secondary_verifier \
                      FROM acpeerstates  \
                      WHERE public_key_fingerprint=? \
                      OR gossip_key_fingerprint=? \
@@ -174,7 +195,10 @@ impl Peerstate {
     ) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint, verifier \
+                     verified_key, verified_key_fingerprint, \
+                     verifier, \
+                     secondary_verified_key, secondary_verified_key_fingerprint, \
+                     secondary_verifier \
                      FROM acpeerstates  \
                      WHERE verified_key_fingerprint=? \
                      OR addr=? COLLATE NOCASE \
@@ -191,11 +215,6 @@ impl Peerstate {
         let peerstate = context
             .sql
             .query_row_optional(query, params, |row| {
-                // all the above queries start with this: SELECT
-                //   addr, last_seen, last_seen_autocrypt, prefer_encrypted,
-                //   public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
-                //   gossip_key_fingerprint, verified_key, verified_key_fingerprint
-
                 let res = Peerstate {
                     addr: row.get("addr")?,
                     last_seen: row.get("last_seen")?,
@@ -230,11 +249,24 @@ impl Peerstate {
                         .map(|s| s.parse::<Fingerprint>())
                         .transpose()
                         .unwrap_or_default(),
-                    fingerprint_changed: false,
                     verifier: {
                         let verifier: Option<String> = row.get("verifier")?;
-                        verifier.filter(|verifier| !verifier.is_empty())
+                        verifier.filter(|s| !s.is_empty())
                     },
+                    secondary_verified_key: row
+                        .get("secondary_verified_key")
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    secondary_verified_key_fingerprint: row
+                        .get::<_, Option<String>>("secondary_verified_key_fingerprint")?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    secondary_verifier: {
+                        let secondary_verifier: Option<String> = row.get("secondary_verifier")?;
+                        secondary_verifier.filter(|s| !s.is_empty())
+                    },
+                    fingerprint_changed: false,
                 };
 
                 Ok(res)
@@ -421,48 +453,56 @@ impl Peerstate {
     /// Make sure to call `self.save_to_db` to save these changes
     /// Params:
     /// verifier:
-    ///   The address which verifies the given contact
-    ///   If we are verifying the contact, use that contacts address
+    ///   The address which introduces the given contact.
+    ///   If we are verifying the contact, use that contacts address.
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
         fingerprint: Fingerprint,
-        verified: PeerstateVerifiedStatus,
         verifier: String,
     ) -> Result<()> {
-        if verified == PeerstateVerifiedStatus::BidirectVerified {
-            match which_key {
-                PeerstateKeyType::PublicKey => {
-                    if self.public_key_fingerprint.is_some()
-                        && self.public_key_fingerprint.as_ref().unwrap() == &fingerprint
-                    {
-                        self.verified_key = self.public_key.clone();
-                        self.verified_key_fingerprint = Some(fingerprint);
-                        self.verifier = Some(verifier);
-                        Ok(())
-                    } else {
-                        Err(Error::msg(format!(
-                            "{fingerprint} is not peer's public key fingerprint",
-                        )))
-                    }
-                }
-                PeerstateKeyType::GossipKey => {
-                    if self.gossip_key_fingerprint.is_some()
-                        && self.gossip_key_fingerprint.as_ref().unwrap() == &fingerprint
-                    {
-                        self.verified_key = self.gossip_key.clone();
-                        self.verified_key_fingerprint = Some(fingerprint);
-                        self.verifier = Some(verifier);
-                        Ok(())
-                    } else {
-                        Err(Error::msg(format!(
-                            "{fingerprint} is not peer's gossip key fingerprint",
-                        )))
-                    }
+        match which_key {
+            PeerstateKeyType::PublicKey => {
+                if self.public_key_fingerprint.is_some()
+                    && self.public_key_fingerprint.as_ref().unwrap() == &fingerprint
+                {
+                    self.verified_key = self.public_key.clone();
+                    self.verified_key_fingerprint = Some(fingerprint);
+                    self.verifier = Some(verifier);
+                    Ok(())
+                } else {
+                    Err(Error::msg(format!(
+                        "{fingerprint} is not peer's public key fingerprint",
+                    )))
                 }
             }
-        } else {
-            Err(Error::msg("BidirectVerified required"))
+            PeerstateKeyType::GossipKey => {
+                if self.gossip_key_fingerprint.is_some()
+                    && self.gossip_key_fingerprint.as_ref().unwrap() == &fingerprint
+                {
+                    self.verified_key = self.gossip_key.clone();
+                    self.verified_key_fingerprint = Some(fingerprint);
+                    self.verifier = Some(verifier);
+                    Ok(())
+                } else {
+                    Err(Error::msg(format!(
+                        "{fingerprint} is not peer's gossip key fingerprint",
+                    )))
+                }
+            }
+        }
+    }
+
+    /// Sets current gossiped key as the secondary verified key.
+    ///
+    /// If gossiped key is the same as the current verified key,
+    /// do nothing to avoid overwriting secondary verified key
+    /// which may be different.
+    pub fn set_secondary_verified_key_from_gossip(&mut self, verifier: String) {
+        if self.gossip_key_fingerprint != self.verified_key_fingerprint {
+            self.secondary_verified_key = self.gossip_key.clone();
+            self.secondary_verified_key_fingerprint = self.gossip_key_fingerprint.clone();
+            self.secondary_verifier = Some(verifier);
         }
     }
 
@@ -480,9 +520,12 @@ impl Peerstate {
                 gossip_key_fingerprint,
                 verified_key,
                 verified_key_fingerprint,
-                addr,
-                verifier)
-                VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
+                verifier,
+                secondary_verified_key,
+                secondary_verified_key_fingerprint,
+                secondary_verifier,
+                addr)
+                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                 ON CONFLICT (addr)
                 DO UPDATE SET
                   last_seen = excluded.last_seen,
@@ -495,7 +538,10 @@ impl Peerstate {
                   gossip_key_fingerprint = excluded.gossip_key_fingerprint,
                   verified_key = excluded.verified_key,
                   verified_key_fingerprint = excluded.verified_key_fingerprint,
-                  verifier = excluded.verifier",
+                  verifier = excluded.verifier,
+                  secondary_verified_key = excluded.secondary_verified_key,
+                  secondary_verified_key_fingerprint = excluded.secondary_verified_key_fingerprint,
+                  secondary_verifier = excluded.secondary_verifier",
             (
                 self.last_seen,
                 self.last_seen_autocrypt,
@@ -507,23 +553,19 @@ impl Peerstate {
                 self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()),
                 self.verified_key.as_ref().map(|k| k.to_bytes()),
                 self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
-                &self.addr,
                 self.verifier.as_deref().unwrap_or(""),
+                self.secondary_verified_key.as_ref().map(|k| k.to_bytes()),
+                self.secondary_verified_key_fingerprint
+                    .as_ref()
+                    .map(|fp| fp.hex()),
+                self.secondary_verifier.as_deref().unwrap_or(""),
+                &self.addr,
             ),
         )
         .await?;
         Ok(())
     }
 
-    /// Returns true if verified key is contained in the given set of fingerprints.
-    pub fn has_verified_key(&self, fingerprints: &HashSet<Fingerprint>) -> bool {
-        if let Some(vkc) = &self.verified_key_fingerprint {
-            fingerprints.contains(vkc) && self.verified_key.is_some()
-        } else {
-            false
-        }
-    }
-
     /// Returns the address that verified the contact
     pub fn get_verifier(&self) -> Option<&str> {
         self.verifier.as_deref()
@@ -774,8 +816,11 @@ mod tests {
             gossip_key_fingerprint: Some(pub_key.fingerprint()),
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -814,8 +859,11 @@ mod tests {
             gossip_key_fingerprint: None,
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -847,8 +895,11 @@ mod tests {
             gossip_key_fingerprint: None,
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
 
         assert!(
@@ -910,8 +961,11 @@ mod tests {
             gossip_key_fingerprint: None,
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
 
         peerstate.apply_header(&header, 100);

src/provider/data.rs

@@ -222,6 +222,99 @@ static P_BUZON_UY: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// c1.testrun.org.md: c1.testrun.org
+static P_C1_TESTRUN_ORG: Provider = Provider {
+    id: "c1.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/c1-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "c1.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "c1.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
+// c2.testrun.org.md: c2.testrun.org
+static P_C2_TESTRUN_ORG: Provider = Provider {
+    id: "c2.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/c2-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "c2.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "c2.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
+// c3.testrun.org.md: c3.testrun.org
+static P_C3_TESTRUN_ORG: Provider = Provider {
+    id: "c3.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/c3-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "c3.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "c3.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
 // chello.at.md: chello.at
 static P_CHELLO_AT: Provider = Provider {
     id: "chello.at",
@@ -867,6 +960,37 @@ static P_NAVER: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// nine.testrun.org.md: nine.testrun.org
+static P_NINE_TESTRUN_ORG: Provider = Provider {
+    id: "nine.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/nine-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "nine.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "nine.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
 // nubo.coop.md: nubo.coop
 static P_NUBO_COOP: Provider = Provider {
     id: "nubo.coop",
@@ -1495,6 +1619,9 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("delta.blindzeln.org", &P_BLINDZELN_ORG),
         ("bluewin.ch", &P_BLUEWIN_CH),
         ("buzon.uy", &P_BUZON_UY),
+        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
+        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
+        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
         ("chello.at", &P_CHELLO_AT),
         ("xfinity.com", &P_COMCAST),
         ("comcast.net", &P_COMCAST),
@@ -1708,6 +1835,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("mailo.com", &P_MAILO_COM),
         ("nauta.cu", &P_NAUTA_CU),
         ("naver.com", &P_NAVER),
+        ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
         ("nubo.coop", &P_NUBO_COOP),
         ("hotmail.com", &P_OUTLOOK_COM),
         ("outlook.com", &P_OUTLOOK_COM),
@@ -1860,6 +1988,9 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("blindzeln.org", &P_BLINDZELN_ORG),
         ("bluewin.ch", &P_BLUEWIN_CH),
         ("buzon.uy", &P_BUZON_UY),
+        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
+        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
+        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
         ("chello.at", &P_CHELLO_AT),
         ("comcast", &P_COMCAST),
         ("dismail.de", &P_DISMAIL_DE),
@@ -1888,6 +2019,7 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("mailo.com", &P_MAILO_COM),
         ("nauta.cu", &P_NAUTA_CU),
         ("naver", &P_NAVER),
+        ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
         ("nubo.coop", &P_NUBO_COOP),
         ("outlook.com", &P_OUTLOOK_COM),
         ("ouvaton.coop", &P_OUVATON_COOP),
@@ -1918,4 +2050,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static _PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 20).unwrap());
+    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 11, 5).unwrap());

src/qr.rs

@@ -1052,8 +1052,11 @@ mod tests {
             gossip_key_fingerprint: None,
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
         assert!(
             peerstate.save_to_db(&ctx.ctx.sql).await.is_ok(),

src/receive_imf.rs

@@ -26,16 +26,18 @@ use crate::imap::{markseen_on_imap_table, GENERATED_PREFIX};
 use crate::location;
 use crate::log::LogExt;
 use crate::message::{
-    self, rfc724_mid_exists, Message, MessageState, MessengerMessage, MsgId, Viewtype,
+    self, rfc724_mid_exists, rfc724_mid_exists_and, Message, MessageState, MessengerMessage, MsgId,
+    Viewtype,
 };
 use crate::mimeparser::{parse_message_ids, AvatarAction, MimeMessage, SystemMessage};
 use crate::param::{Param, Params};
-use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus};
+use crate::peerstate::{Peerstate, PeerstateKeyType};
 use crate::reaction::{set_msg_reaction, Reaction};
 use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
 use crate::simplify;
 use crate::sql;
 use crate::stock_str;
+use crate::sync::Sync::*;
 use crate::tools::{
     buf_compress, extract_grpid_from_rfc724_mid, smeared_time, strip_rtlo_characters,
 };
@@ -225,6 +227,8 @@ pub(crate) async fn receive_imf_inner(
         .and_then(|value| mailparse::dateparse(value).ok())
         .map_or(rcvd_timestamp, |value| min(value, rcvd_timestamp + 60));
 
+    update_verified_keys(context, &mut mime_parser, from_id).await?;
+
     // Add parts
     let received_msg = add_parts(
         context,
@@ -281,9 +285,7 @@ pub(crate) async fn receive_imf_inner(
     if let Some(ref sync_items) = mime_parser.sync_items {
         if from_id == ContactId::SELF {
             if mime_parser.was_encrypted() {
-                if let Err(err) = context.execute_sync_items(sync_items).await {
-                    warn!(context, "receive_imf cannot execute sync items: {err:#}.");
-                }
+                context.execute_sync_items(sync_items).await;
             } else {
                 warn!(context, "Sync items are not encrypted.");
             }
@@ -455,6 +457,7 @@ async fn add_parts(
     let mut chat_id_blocked = Blocked::Not;
 
     let mut better_msg = None;
+    let mut group_changes_msgs = (Vec::new(), None);
     if mime_parser.is_system_message == SystemMessage::LocationStreamingEnabled {
         better_msg = Some(stock_str::msg_location_enabled_by(context, from_id).await);
     }
@@ -502,7 +505,6 @@ async fn add_parts(
     // - incoming messages introduce a chat only for known contacts if they are sent by a messenger
     // (of course, the user can add other chats manually later)
     let to_id: ContactId;
-
     let state: MessageState;
     let mut needs_delete_job = false;
     if incoming {
@@ -514,25 +516,23 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
-            match handle_securejoin_handshake(context, mime_parser, from_id).await {
-                Ok(securejoin::HandshakeMessage::Done) => {
+            match handle_securejoin_handshake(context, mime_parser, from_id)
+                .await
+                .context("error in Secure-Join message handling")?
+            {
+                securejoin::HandshakeMessage::Done => {
                     chat_id = Some(DC_CHAT_ID_TRASH);
                     needs_delete_job = true;
                     securejoin_seen = true;
                 }
-                Ok(securejoin::HandshakeMessage::Ignore) => {
+                securejoin::HandshakeMessage::Ignore => {
                     chat_id = Some(DC_CHAT_ID_TRASH);
                     securejoin_seen = true;
                 }
-                Ok(securejoin::HandshakeMessage::Propagate) => {
+                securejoin::HandshakeMessage::Propagate => {
                     // process messages as "member added" normally
                     securejoin_seen = false;
                 }
-                Err(err) => {
-                    warn!(context, "Error in Secure-Join message handling: {err:#}.");
-                    chat_id = Some(DC_CHAT_ID_TRASH);
-                    securejoin_seen = true;
-                }
             }
             // Peerstate could be updated by handling the Securejoin handshake.
             let contact = Contact::get_by_id(context, from_id).await?;
@@ -554,6 +554,11 @@ async fn add_parts(
             markseen_on_imap_table(context, rfc724_mid).await.ok();
         }
 
+        if chat_id.is_none() && is_mdn {
+            chat_id = Some(DC_CHAT_ID_TRASH);
+            info!(context, "Message is an MDN (TRASH).",);
+        }
+
         if chat_id.is_none() {
             // try to assign to a chat based on In-Reply-To/References:
 
@@ -637,7 +642,7 @@ async fn add_parts(
                         chat_id = None;
                     } else {
                         let s = stock_str::unknown_sender_for_chat(context).await;
-                        mime_parser.repl_msg_by_error(&s);
+                        mime_parser.replace_msg_by_error(&s);
                     }
                 } else {
                     // In non-protected chats, just mark the sender as overridden. Therefore, the UI will prepend `~`
@@ -650,15 +655,16 @@ async fn add_parts(
                 }
             }
 
-            better_msg = better_msg.or(apply_group_changes(
+            group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
                 sent_timestamp,
                 group_chat_id,
                 from_id,
                 to_ids,
+                is_partial_download.is_some(),
             )
-            .await?);
+            .await?;
         }
 
         if chat_id.is_none() {
@@ -808,19 +814,17 @@ async fn add_parts(
 
         // handshake may mark contacts as verified and must be processed before chats are created
         if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
-            match observe_securejoin_on_other_device(context, mime_parser, to_id).await {
-                Ok(securejoin::HandshakeMessage::Done)
-                | Ok(securejoin::HandshakeMessage::Ignore) => {
+            match observe_securejoin_on_other_device(context, mime_parser, to_id)
+                .await
+                .context("error in Secure-Join watching")?
+            {
+                securejoin::HandshakeMessage::Done | securejoin::HandshakeMessage::Ignore => {
                     chat_id = Some(DC_CHAT_ID_TRASH);
                 }
-                Ok(securejoin::HandshakeMessage::Propagate) => {
+                securejoin::HandshakeMessage::Propagate => {
                     // process messages as "member added" normally
                     chat_id = None;
                 }
-                Err(err) => {
-                    warn!(context, "Error in Secure-Join watching: {err:#}.");
-                    chat_id = Some(DC_CHAT_ID_TRASH);
-                }
             }
         } else if mime_parser.sync_items.is_some() && self_sent {
             chat_id = Some(DC_CHAT_ID_TRASH);
@@ -887,22 +891,23 @@ async fn add_parts(
             // automatically unblock chat when the user sends a message
             if chat_id_blocked != Blocked::Not {
                 if let Some(chat_id) = chat_id {
-                    chat_id.unblock(context).await?;
+                    chat_id.unblock_ex(context, Nosync).await?;
                     chat_id_blocked = Blocked::Not;
                 }
             }
         }
 
         if let Some(chat_id) = chat_id {
-            better_msg = better_msg.or(apply_group_changes(
+            group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
                 sent_timestamp,
                 chat_id,
                 from_id,
                 to_ids,
+                is_partial_download.is_some(),
             )
-            .await?);
+            .await?;
         }
 
         if chat_id.is_none() && self_sent {
@@ -919,11 +924,27 @@ async fn add_parts(
 
             if let Some(chat_id) = chat_id {
                 if Blocked::Not != chat_id_blocked {
-                    chat_id.unblock(context).await?;
+                    chat_id.unblock_ex(context, Nosync).await?;
                     // Not assigning `chat_id_blocked = Blocked::Not` to avoid unused_assignments warning.
                 }
             }
         }
+
+        if chat_id.is_none() {
+            // Check if the message belongs to a broadcast list.
+            if let Some(mailinglist_header) = mime_parser.get_mailinglist_header() {
+                let listid = mailinglist_header_listid(mailinglist_header)?;
+                chat_id = Some(
+                    if let Some((id, ..)) = chat::get_chat_id_by_grpid(context, &listid).await? {
+                        id
+                    } else {
+                        let name =
+                            compute_mailinglist_name(mailinglist_header, &listid, mime_parser);
+                        chat::create_broadcast_list_ex(context, Nosync, listid, name).await?
+                    },
+                );
+            }
+        }
     }
 
     if fetching_existing_messages && mime_parser.decrypting_failed {
@@ -1055,7 +1076,7 @@ async fn add_parts(
             {
                 warn!(context, "Verification problem: {err:#}.");
                 let s = format!("{err}. See 'Info' for more details");
-                mime_parser.repl_msg_by_error(&s);
+                mime_parser.replace_msg_by_error(&s);
             }
         }
     }
@@ -1116,7 +1137,29 @@ async fn add_parts(
 
     let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len());
 
-    for part in &mut mime_parser.parts {
+    if let Some(msg) = group_changes_msgs.1 {
+        match &better_msg {
+            None => better_msg = Some(msg),
+            Some(_) => group_changes_msgs.0.push(msg),
+        }
+    }
+
+    for group_changes_msg in group_changes_msgs.0 {
+        // Currently all additional group changes messages are "Member added".
+        chat::add_info_msg_with_cmd(
+            context,
+            chat_id,
+            &group_changes_msg,
+            SystemMessage::MemberAddedToGroup,
+            sort_timestamp,
+            None,
+            None,
+            None,
+        )
+        .await?;
+    }
+
+    for part in &mime_parser.parts {
         if part.is_reaction {
             let reaction_str = simplify::remove_footers(part.msg.as_str());
             set_msg_reaction(
@@ -1596,7 +1639,7 @@ async fn create_or_lookup_group(
         {
             warn!(context, "Verification problem: {err:#}.");
             let s = format!("{err}. See 'Info' for more details");
-            mime_parser.repl_msg_by_error(&s);
+            mime_parser.replace_msg_by_error(&s);
         }
         ProtectionStatus::Protected
     } else {
@@ -1657,20 +1700,10 @@ async fn create_or_lookup_group(
             members.push(from_id);
         }
         members.extend(to_ids);
+        members.sort_unstable();
         members.dedup();
         chat::add_to_chat_contacts_table(context, new_chat_id, &members).await?;
 
-        // once, we have protected-chats explained in UI, we can uncomment the following lines.
-        // ("verified groups" did not add a message anyway)
-        //
-        //if create_protected == ProtectionStatus::Protected {
-        // set from_id=0 as it is not clear that the sender of this random group message
-        // actually really has enabled chat-protection at some point.
-        //new_chat_id
-        //    .add_protection_msg(context, ProtectionStatus::Protected, false, 0)
-        //    .await?;
-        //}
-
         context.emit_event(EventType::ChatModified(new_chat_id));
     }
 
@@ -1695,6 +1728,7 @@ async fn create_or_lookup_group(
 /// Apply group member list, name, avatar and protection status changes from the MIME message.
 ///
 /// Optionally returns better message to replace the original system message.
+/// is_partial_download: whether the message is not fully downloaded.
 async fn apply_group_changes(
     context: &Context,
     mime_parser: &mut MimeMessage,
@@ -1702,15 +1736,21 @@ async fn apply_group_changes(
     chat_id: ChatId,
     from_id: ContactId,
     to_ids: &[ContactId],
-) -> Result<Option<String>> {
+    is_partial_download: bool,
+) -> Result<(Vec<String>, Option<String>)> {
+    if chat_id.is_special() {
+        // Do not apply group changes to the trash chat.
+        return Ok((Vec::new(), None));
+    }
     let mut chat = Chat::load_from_db(context, chat_id).await?;
     if chat.typ != Chattype::Group {
-        return Ok(None);
+        return Ok((Vec::new(), None));
     }
 
     let mut send_event_chat_modified = false;
     let (mut removed_id, mut added_id) = (None, None);
     let mut better_msg = None;
+    let mut group_changes_msgs = Vec::new();
 
     // True if a Delta Chat client has explicitly added our current primary address.
     let self_added =
@@ -1720,12 +1760,14 @@ async fn apply_group_changes(
             false
         };
 
-    let mut chat_contacts = HashSet::from_iter(chat::get_chat_contacts(context, chat_id).await?);
+    let mut chat_contacts =
+        HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat_id).await?);
     let is_from_in_chat =
         !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
 
     // Reject group membership changes from non-members and old changes.
-    let allow_member_list_changes = is_from_in_chat
+    let allow_member_list_changes = !is_partial_download
+        && is_from_in_chat
         && chat_id
             .update_timestamp(context, Param::MemberListTimestamp, sent_timestamp)
             .await?;
@@ -1739,7 +1781,9 @@ async fn apply_group_changes(
             || match mime_parser.get_header(HeaderDef::InReplyTo) {
                 // If we don't know the referenced message, we missed some messages.
                 // Maybe they added/removed members, so we need to recreate our member list.
-                Some(reply_to) => rfc724_mid_exists(context, reply_to).await?.is_none(),
+                Some(reply_to) => rfc724_mid_exists_and(context, reply_to, "download_state=0")
+                    .await?
+                    .is_none(),
                 None => false,
             }
     } && {
@@ -1758,12 +1802,17 @@ async fn apply_group_changes(
         {
             warn!(context, "Verification problem: {err:#}.");
             let s = format!("{err}. See 'Info' for more details");
-            mime_parser.repl_msg_by_error(&s);
+            mime_parser.replace_msg_by_error(&s);
         }
 
         if !chat.is_protected() {
             chat_id
-                .inner_set_protection(context, ProtectionStatus::Protected)
+                .set_protection(
+                    context,
+                    ProtectionStatus::Protected,
+                    smeared_time(context),
+                    Some(from_id),
+                )
                 .await?;
         }
     }
@@ -1856,33 +1905,43 @@ async fn apply_group_changes(
         }
 
         if !recreate_member_list {
-            let diff: HashSet<ContactId> =
-                chat_contacts.difference(&new_members).copied().collect();
-            // Only delete old contacts if the sender is not a classical MUA user:
-            // Classical MUA users usually don't intend to remove users from an email
-            // thread, so if they removed a recipient then it was probably by accident.
-            if mime_parser.has_chat_version() {
-                // This is what provides group membership consistency: we remove group members
-                // locally if we see a discrepancy with the "To" list in the received message as it
-                // is better for privacy than adding absent members locally. But it shouldn't be a
-                // big problem if somebody missed a member addition, because they will likely
-                // recreate the member list from the next received message. The problem occurs only
-                // if that "somebody" managed to reply earlier. Really, it's a problem for big
-                // groups with high message rate, but let it be for now.
-                if !diff.is_empty() {
-                    warn!(context, "Implicit removal of {diff:?} from chat {chat_id}.");
-                }
-                new_members = chat_contacts.difference(&diff).copied().collect();
-            } else {
-                new_members.extend(diff);
+            // Don't delete any members locally, but instead add absent ones to provide group
+            // membership consistency for all members:
+            // - Classical MUA users usually don't intend to remove users from an email thread, so
+            //   if they removed a recipient then it was probably by accident.
+            // - DC users could miss new member additions and then better to handle this in the same
+            //   way as for classical MUA messages. Moreover, if we remove a member implicitly, they
+            //   will never know that and continue to think they're still here.
+            // But it shouldn't be a big problem if somebody missed a member removal, because they
+            // will likely recreate the member list from the next received message. The problem
+            // occurs only if that "somebody" managed to reply earlier. Really, it's a problem for
+            // big groups with high message rate, but let it be for now.
+            let mut diff: HashSet<ContactId> =
+                new_members.difference(&chat_contacts).copied().collect();
+            new_members = chat_contacts.clone();
+            new_members.extend(diff.clone());
+            if let Some(added_id) = added_id {
+                diff.remove(&added_id);
+            }
+            if !diff.is_empty() {
+                warn!(context, "Implicit addition of {diff:?} to chat {chat_id}.");
+            }
+            group_changes_msgs.reserve(diff.len());
+            for contact_id in diff {
+                let contact = Contact::get_by_id(context, contact_id).await?;
+                group_changes_msgs.push(
+                    stock_str::msg_add_member_local(
+                        context,
+                        contact.get_addr(),
+                        ContactId::UNDEFINED,
+                    )
+                    .await,
+                );
             }
         }
         if let Some(removed_id) = removed_id {
             new_members.remove(&removed_id);
         }
-        if let Some(added_id) = added_id {
-            new_members.insert(added_id);
-        }
         if recreate_member_list {
             info!(
                 context,
@@ -1891,21 +1950,7 @@ async fn apply_group_changes(
         }
 
         if new_members != chat_contacts {
-            let new_members_ref = &new_members;
-            context
-                .sql
-                .transaction(move |transaction| {
-                    transaction
-                        .execute("DELETE FROM chats_contacts WHERE chat_id=?", (chat_id,))?;
-                    for contact_id in new_members_ref {
-                        transaction.execute(
-                            "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
-                            (chat_id, contact_id),
-                        )?;
-                    }
-                    Ok(())
-                })
-                .await?;
+            chat::update_chat_contacts_table(context, chat_id, &new_members).await?;
             chat_contacts = new_members;
             send_event_chat_modified = true;
         }
@@ -1945,11 +1990,22 @@ async fn apply_group_changes(
     if send_event_chat_modified {
         context.emit_event(EventType::ChatModified(chat_id));
     }
-    Ok(better_msg)
+    Ok((group_changes_msgs, better_msg))
 }
 
 static LIST_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^(.+)<(.+)>$").unwrap());
 
+fn mailinglist_header_listid(list_id_header: &str) -> Result<String> {
+    Ok(match LIST_ID_REGEX.captures(list_id_header) {
+        Some(cap) => cap.get(2).context("no match??")?.as_str().trim(),
+        None => list_id_header
+            .trim()
+            .trim_start_matches('<')
+            .trim_end_matches('>'),
+    }
+    .to_string())
+}
+
 /// Create or lookup a mailing list chat.
 ///
 /// `list_id_header` contains the Id that must be used for the mailing list
@@ -1959,21 +2015,13 @@ static LIST_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^(.+)<(.+)>$").unw
 ///
 /// `mime_parser` is the corresponding message
 /// and is used to figure out the mailing list name from different header fields.
-#[allow(clippy::indexing_slicing)]
 async fn create_or_lookup_mailinglist(
     context: &Context,
     allow_creation: bool,
     list_id_header: &str,
     mime_parser: &MimeMessage,
 ) -> Result<Option<(ChatId, Blocked)>> {
-    let listid = match LIST_ID_REGEX.captures(list_id_header) {
-        Some(cap) => cap[2].trim().to_string(),
-        None => list_id_header
-            .trim()
-            .trim_start_matches('<')
-            .trim_end_matches('>')
-            .to_string(),
-    };
+    let listid = mailinglist_header_listid(list_id_header)?;
 
     if let Some((chat_id, _, blocked)) = chat::get_chat_id_by_grpid(context, &listid).await? {
         return Ok(Some((chat_id, blocked)));
@@ -2245,10 +2293,72 @@ enum VerifiedEncryption {
     NotVerified(String), // The string contains the reason why it's not verified
 }
 
+/// Moves secondary verified key to primary verified key
+/// if the message is signed with a secondary verified key.
+/// Removes secondary verified key if the message is signed with primary key.
+async fn update_verified_keys(
+    context: &Context,
+    mimeparser: &mut MimeMessage,
+    from_id: ContactId,
+) -> Result<Option<String>> {
+    if from_id == ContactId::SELF {
+        return Ok(None);
+    }
+
+    if !mimeparser.was_encrypted() {
+        return Ok(None);
+    }
+
+    let Some(peerstate) = &mut mimeparser.decryption_info.peerstate else {
+        // No peerstate means no verified keys.
+        return Ok(None);
+    };
+
+    let signed_with_primary_verified_key = peerstate
+        .verified_key_fingerprint
+        .as_ref()
+        .filter(|fp| mimeparser.signatures.contains(fp))
+        .is_some();
+    let signed_with_secondary_verified_key = peerstate
+        .secondary_verified_key_fingerprint
+        .as_ref()
+        .filter(|fp| mimeparser.signatures.contains(fp))
+        .is_some();
+
+    if signed_with_primary_verified_key {
+        // Remove secondary key if it exists.
+        if peerstate.secondary_verified_key.is_some()
+            || peerstate.secondary_verified_key_fingerprint.is_some()
+            || peerstate.secondary_verifier.is_some()
+        {
+            peerstate.secondary_verified_key = None;
+            peerstate.secondary_verified_key_fingerprint = None;
+            peerstate.secondary_verifier = None;
+            peerstate.save_to_db(&context.sql).await?;
+        }
+
+        // No need to notify about secondary key removal.
+        Ok(None)
+    } else if signed_with_secondary_verified_key {
+        peerstate.verified_key = peerstate.secondary_verified_key.take();
+        peerstate.verified_key_fingerprint = peerstate.secondary_verified_key_fingerprint.take();
+        peerstate.verifier = peerstate.secondary_verifier.take();
+        peerstate.fingerprint_changed = true;
+        peerstate.save_to_db(&context.sql).await?;
+
+        // Primary verified key changed.
+        Ok(None)
+    } else {
+        Ok(None)
+    }
+}
+
 /// Checks whether the message is allowed to appear in a protected chat.
 ///
 /// This means that it is encrypted, signed with a verified key,
 /// and if it's a group, all the recipients are verified.
+///
+/// Also propagates gossiped keys to verified if needed.
 async fn has_verified_encryption(
     context: &Context,
     mimeparser: &MimeMessage,
@@ -2285,7 +2395,13 @@ async fn has_verified_encryption(
             ));
         };
 
-        if !peerstate.has_verified_key(&mimeparser.signatures) {
+        let signed_with_verified_key = peerstate
+            .verified_key_fingerprint
+            .as_ref()
+            .filter(|fp| mimeparser.signatures.contains(fp))
+            .is_some();
+
+        if !signed_with_verified_key {
             return Ok(NotVerified(
                 "The message was sent with non-verified encryption".to_string(),
             ));
@@ -2303,6 +2419,16 @@ async fn has_verified_encryption(
         return Ok(Verified);
     }
 
+    mark_recipients_as_verified(context, from_id, to_ids, mimeparser).await?;
+    Ok(Verified)
+}
+
+async fn mark_recipients_as_verified(
+    context: &Context,
+    from_id: ContactId,
+    to_ids: Vec<ContactId>,
+    mimeparser: &MimeMessage,
+) -> Result<(), anyhow::Error> {
     let rows = context
         .sql
         .query_map(
@@ -2311,7 +2437,7 @@ async fn has_verified_encryption(
              LEFT JOIN acpeerstates ps ON c.addr=ps.addr  WHERE c.id IN({}) ",
                 sql::repeat_vars(to_ids.len())
             ),
-            rusqlite::params_from_iter(to_ids),
+            rusqlite::params_from_iter(&to_ids),
             |row| {
                 let to_addr: String = row.get(0)?;
                 let is_verified: i32 = row.get(1).unwrap_or(0);
@@ -2326,50 +2452,48 @@ async fn has_verified_encryption(
 
     let contact = Contact::get_by_id(context, from_id).await?;
 
-    for (to_addr, mut is_verified) in rows {
-        info!(
-            context,
-            "has_verified_encryption: {:?} self={:?}.",
-            to_addr,
-            context.is_self_addr(&to_addr).await
-        );
-        let peerstate = Peerstate::from_addr(context, &to_addr).await?;
-
+    for (to_addr, is_verified) in rows {
         // mark gossiped keys (if any) as verified
         if mimeparser.gossiped_addr.contains(&to_addr.to_lowercase()) {
-            if let Some(mut peerstate) = peerstate {
-                // if we're here, we know the gossip key is verified:
-                // - use the gossip-key as verified-key if there is no verified-key
-                // - OR if the verified-key does not match public-key or gossip-key
-                //   (otherwise a verified key can _only_ be updated through QR scan which might be annoying,
-                //   see <https://github.com/nextleap-project/countermitm/issues/46> for a discussion about this point)
-                if !is_verified
-                    || peerstate.verified_key_fingerprint != peerstate.public_key_fingerprint
-                        && peerstate.verified_key_fingerprint != peerstate.gossip_key_fingerprint
-                {
-                    info!(context, "{} has verified {}.", contact.get_addr(), to_addr);
-                    let fp = peerstate.gossip_key_fingerprint.clone();
-                    if let Some(fp) = fp {
-                        peerstate.set_verified(
-                            PeerstateKeyType::GossipKey,
-                            fp,
-                            PeerstateVerifiedStatus::BidirectVerified,
-                            contact.get_addr().to_owned(),
-                        )?;
+            if let Some(mut peerstate) = Peerstate::from_addr(context, &to_addr).await? {
+                // If we're here, we know the gossip key is verified.
+                //
+                // Use the gossip-key as verified-key if there is no verified-key.
+                //
+                // Store gossip key as secondary verified key if there is a verified key and
+                // gossiped key is different.
+                //
+                // See <https://github.com/nextleap-project/countermitm/issues/46>
+                // and <https://github.com/deltachat/deltachat-core-rust/issues/4541> for discussion.
+                let verifier_addr = contact.get_addr().to_owned();
+                if !is_verified {
+                    info!(context, "{verifier_addr} has verified {to_addr}.");
+                    if let Some(fp) = peerstate.gossip_key_fingerprint.clone() {
+                        peerstate.set_verified(PeerstateKeyType::GossipKey, fp, verifier_addr)?;
                         peerstate.save_to_db(&context.sql).await?;
-                        is_verified = true;
+
+                        if !is_verified {
+                            let (to_contact_id, _) = Contact::add_or_lookup(
+                                context,
+                                "",
+                                ContactAddress::new(&to_addr)?,
+                                Origin::Hidden,
+                            )
+                            .await?;
+                            ChatId::set_protection_for_contact(context, to_contact_id).await?;
+                        }
                     }
+                } else {
+                    // The contact already has a verified key.
+                    // Store gossiped key as the secondary verified key.
+                    peerstate.set_secondary_verified_key_from_gossip(verifier_addr);
+                    peerstate.save_to_db(&context.sql).await?;
                 }
             }
         }
-        if !is_verified {
-            return Ok(NotVerified(format!(
-                "{} is not a member of this protected chat",
-                to_addr
-            )));
-        }
     }
-    Ok(Verified)
+
+    Ok(())
 }
 
 /// Returns the last message referenced from `References` header if it is in the database.

src/receive_imf/tests.rs

@@ -3117,6 +3117,46 @@ async fn test_thunderbird_autocrypt() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_prefer_encrypt_mutual_if_encrypted() -> Result<()> {
+    let t = TestContext::new_bob().await;
+
+    let raw =
+        include_bytes!("../../test-data/message/thunderbird_encrypted_signed_with_pubkey.eml");
+    receive_imf(&t, raw, false).await?;
+    let peerstate = Peerstate::from_addr(&t, "alice@example.org")
+        .await?
+        .unwrap();
+    assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
+
+    receive_imf(
+        &t,
+        b"From: alice@example.org\n\
+          To: bob@example.net\n\
+          Subject: foo\n\
+          Message-ID: <message@example.org>\n\
+          Date: Thu, 2 Nov 2023 02:20:28 -0300\n\
+          \n\
+          unencrypted\n",
+        false,
+    )
+    .await?;
+    let peerstate = Peerstate::from_addr(&t, "alice@example.org")
+        .await?
+        .unwrap();
+    assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+    let raw = include_bytes!("../../test-data/message/thunderbird_encrypted_signed.eml");
+    receive_imf(&t, raw, false).await?;
+    let peerstate = Peerstate::from_addr(&t, "alice@example.org")
+        .await?
+        .unwrap();
+    assert!(peerstate.public_key.is_some());
+    assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_thunderbird_autocrypt_unencrypted() -> Result<()> {
     let t = TestContext::new_bob().await;
@@ -3371,14 +3411,24 @@ async fn test_dont_recreate_contacts_on_add_remove() -> Result<()> {
 
     alice.recv_msg(&bob.pop_sent_msg().await).await;
 
-    // Bob didn't receive the addition of Fiona, so Alice must remove Fiona from the members list
-    // back to make their group members view consistent.
-    assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 3);
+    // Bob didn't receive the addition of Fiona, but Alice mustn't remove Fiona from the members
+    // list back. Instead, Bob must add Fiona from the next Alice's message to make their group
+    // members view consistent.
+    assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 4);
 
     // Just a dumb check for remove_contact_from_chat(). Let's have it in this only place.
     remove_contact_from_chat(&bob, bob_chat_id, bob_blue).await?;
     alice.recv_msg(&bob.pop_sent_msg().await).await;
-    assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2);
+    assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 3);
+
+    send_text_msg(
+        &alice,
+        alice_chat_id,
+        "Finally add Fiona please".to_string(),
+    )
+    .await?;
+    bob.recv_msg(&alice.pop_sent_msg().await).await;
+    assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 3);
 
     Ok(())
 }
@@ -3462,8 +3512,11 @@ async fn test_dont_readd_with_normal_msg() -> Result<()> {
 
     bob.recv_msg(&alice.pop_sent_msg().await).await;
 
-    // Alice didn't receive Bobs leave message, but bob shouldn't readded himself just because of that.
-    assert!(!is_contact_in_chat(&bob, bob_chat_id, ContactId::SELF).await?);
+    // Alice didn't receive Bob's leave message, so Bob must readd themselves otherwise other
+    // members would think Bob is still here while they aren't, and then retry to leave if they
+    // think that Alice didn't re-add them on purpose (which is possible if Alice uses a classical
+    // MUA).
+    assert!(is_contact_in_chat(&bob, bob_chat_id, ContactId::SELF).await?);
     Ok(())
 }
 
@@ -3633,6 +3686,34 @@ async fn test_mua_can_readd() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_member_left_does_not_create_chat() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+    let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "Group").await?;
+    add_contact_to_chat(
+        &alice,
+        alice_chat_id,
+        Contact::create(&alice, "bob", &bob.get_config(Config::Addr).await?.unwrap()).await?,
+    )
+    .await?;
+    send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
+    alice.pop_sent_msg().await;
+
+    // Bob only received a message of Alice leaving the group.
+    // This should not create the group.
+    //
+    // The reason is to avoid recreating deleted chats,
+    // especially the chats that were created due to "split group" bugs
+    // which some members simply deleted and some members left,
+    // recreating the chat for others.
+    remove_contact_from_chat(&alice, alice_chat_id, ContactId::SELF).await?;
+    let bob_chat_id = bob.recv_msg(&alice.pop_sent_msg().await).await.chat_id;
+    assert!(bob_chat_id.is_trash());
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_recreate_member_list_on_missing_add_of_self() -> Result<()> {
     let alice = TestContext::new_alice().await;
@@ -3646,11 +3727,14 @@ async fn test_recreate_member_list_on_missing_add_of_self() -> Result<()> {
     .await?;
     send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
     alice.pop_sent_msg().await;
-    remove_contact_from_chat(&alice, alice_chat_id, ContactId::SELF).await?;
+
+    send_text_msg(&alice, alice_chat_id, "second message".to_string()).await?;
+
     let bob_chat_id = bob.recv_msg(&alice.pop_sent_msg().await).await.chat_id;
+    assert!(!bob_chat_id.is_special());
 
     // Bob missed the message adding them, but must recreate the member list.
-    assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 1);
+    assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 2);
     assert!(is_contact_in_chat(&bob, bob_chat_id, ContactId::SELF).await?);
     Ok(())
 }
@@ -3810,3 +3894,113 @@ async fn test_create_group_with_big_msg() -> Result<()> {
 
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_partial_group_consistency() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    let bob_id = Contact::create(&alice, "", "bob@example.net").await?;
+    let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "foos").await?;
+    add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
+
+    send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
+    let add = alice.pop_sent_msg().await;
+    let bob = tcm.bob().await;
+    bob.recv_msg(&add).await;
+    let bob_chat_id = bob.get_last_msg().await.chat_id;
+    let contacts = get_chat_contacts(&bob, bob_chat_id).await?;
+    assert_eq!(contacts.len(), 2);
+
+    // Get initial timestamp.
+    let timestamp = bob_chat_id
+        .get_param(&bob)
+        .await?
+        .get_i64(Param::MemberListTimestamp)
+        .unwrap();
+
+    // Bob receives partial message.
+    let msg_id = receive_imf_inner(
+        &bob,
+        "first@example.org",
+        b"From: Alice <alice@example.org>\n\
+To: <bob@example.net>, <charlie@example.com>\n\
+Chat-Version: 1.0\n\
+Subject: subject\n\
+Message-ID: <first@example.org>\n\
+Date: Sun, 14 Nov 2021 00:10:00 +0000\
+Content-Type: text/plain
+Chat-Group-Member-Added: charlie@example.com",
+        false,
+        Some(100000),
+        false,
+    )
+    .await?
+    .context("no received message")?;
+
+    let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?;
+    let timestamp2 = bob_chat_id
+        .get_param(&bob)
+        .await?
+        .get_i64(Param::MemberListTimestamp)
+        .unwrap();
+
+    // Partial download does not change the member list.
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert_eq!(timestamp, timestamp2);
+    assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts);
+
+    // Alice sends normal message to bob, adding fiona.
+    add_contact_to_chat(
+        &alice,
+        alice_chat_id,
+        Contact::create(&alice, "fiona", "fiona@example.net").await?,
+    )
+    .await?;
+
+    bob.recv_msg(&alice.pop_sent_msg().await).await;
+
+    let timestamp3 = bob_chat_id
+        .get_param(&bob)
+        .await?
+        .get_i64(Param::MemberListTimestamp)
+        .unwrap();
+
+    // Receiving a message after a partial download recreates the member list because we treat
+    // such messages as if we have not seen them.
+    assert_ne!(timestamp, timestamp3);
+    let contacts = get_chat_contacts(&bob, bob_chat_id).await?;
+    assert_eq!(contacts.len(), 3);
+
+    // Bob fully reives the partial message.
+    let msg_id = receive_imf_inner(
+        &bob,
+        "first@example.org",
+        b"From: Alice <alice@example.org>\n\
+To: Bob <bob@example.net>\n\
+Chat-Version: 1.0\n\
+Subject: subject\n\
+Message-ID: <first@example.org>\n\
+Date: Sun, 14 Nov 2021 00:10:00 +0000\
+Content-Type: text/plain
+Chat-Group-Member-Added: charlie@example.com",
+        false,
+        None,
+        false,
+    )
+    .await?
+    .context("no received message")?;
+
+    let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?;
+    let timestamp4 = bob_chat_id
+        .get_param(&bob)
+        .await?
+        .get_i64(Param::MemberListTimestamp)
+        .unwrap();
+
+    // After full download, the old message should not change group state.
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(timestamp3, timestamp4);
+    assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts);
+
+    Ok(())
+}

src/scheduler.rs

@@ -1,3 +1,4 @@
+use std::cmp;
 use std::iter::{self, once};
 use std::num::NonZeroUsize;
 use std::sync::atomic::Ordering;
@@ -6,6 +7,7 @@ use anyhow::{bail, Context as _, Error, Result};
 use async_channel::{self as channel, Receiver, Sender};
 use futures::future::try_join_all;
 use futures_lite::FutureExt;
+use rand::Rng;
 use tokio::sync::{oneshot, RwLock, RwLockWriteGuard};
 use tokio::task;
 
@@ -233,17 +235,17 @@ impl SchedulerState {
         connectivity::maybe_network_lost(context, stores).await;
     }
 
-    pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
+    pub(crate) async fn interrupt_inbox(&self) {
         let inner = self.inner.read().await;
         if let InnerSchedulerState::Started(ref scheduler) = *inner {
-            scheduler.interrupt_inbox(info);
+            scheduler.interrupt_inbox();
         }
     }
 
-    pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
+    pub(crate) async fn interrupt_smtp(&self) {
         let inner = self.inner.read().await;
         if let InnerSchedulerState::Started(ref scheduler) = *inner {
-            scheduler.interrupt_smtp(info);
+            scheduler.interrupt_smtp();
         }
     }
 
@@ -463,18 +465,15 @@ async fn inbox_loop(
 /// handling all the errors. In case of an error, it is logged, but not propagated upwards. If
 /// critical operation fails such as fetching new messages fails, connection is reset via
 /// `trigger_reconnect`, so a fresh one can be opened.
-async fn fetch_idle(
-    ctx: &Context,
-    connection: &mut Imap,
-    folder_meaning: FolderMeaning,
-) -> InterruptInfo {
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: FolderMeaning) {
     let folder_config = match folder_meaning.to_config() {
         Some(c) => c,
         None => {
             error!(ctx, "Bad folder meaning: {}", folder_meaning);
-            return connection
+            connection
                 .fake_idle(ctx, None, FolderMeaning::Unknown)
                 .await;
+            return;
         }
     };
     let folder = match ctx.get_config(folder_config).await {
@@ -484,9 +483,10 @@ async fn fetch_idle(
                 ctx,
                 "Can not watch {} folder, failed to retrieve config: {:#}", folder_config, err
             );
-            return connection
+            connection
                 .fake_idle(ctx, None, FolderMeaning::Unknown)
                 .await;
+            return;
         }
     };
 
@@ -495,9 +495,10 @@ async fn fetch_idle(
     } else {
         connection.connectivity.set_not_configured(ctx).await;
         info!(ctx, "Can not watch {} folder, not set", folder_config);
-        return connection
+        connection
             .fake_idle(ctx, None, FolderMeaning::Unknown)
             .await;
+        return;
     };
 
     // connect and fake idle if unable to connect
@@ -508,9 +509,10 @@ async fn fetch_idle(
     {
         warn!(ctx, "{:#}", err);
         connection.trigger_reconnect(ctx);
-        return connection
+        connection
             .fake_idle(ctx, Some(watch_folder), folder_meaning)
             .await;
+        return;
     }
 
     if folder_config == Config::ConfiguredInboxFolder {
@@ -534,7 +536,7 @@ async fn fetch_idle(
     {
         connection.trigger_reconnect(ctx);
         warn!(ctx, "{:#}", err);
-        return InterruptInfo::new(false);
+        return;
     }
 
     // Mark expired messages for deletion. Marked messages will be deleted from the server
@@ -573,7 +575,7 @@ async fn fetch_idle(
                 {
                     connection.trigger_reconnect(ctx);
                     warn!(ctx, "{:#}", err);
-                    return InterruptInfo::new(false);
+                    return;
                 }
             }
             Ok(false) => {}
@@ -591,55 +593,56 @@ async fn fetch_idle(
     connection.connectivity.set_connected(ctx).await;
 
     ctx.emit_event(EventType::ImapInboxIdle);
-    if let Some(session) = connection.session.take() {
-        if !session.can_idle() {
-            info!(
-                ctx,
-                "IMAP session does not support IDLE, going to fake idle."
-            );
-            return connection
-                .fake_idle(ctx, Some(watch_folder), folder_meaning)
-                .await;
-        }
-
-        if ctx
-            .get_config_bool(Config::DisableIdle)
-            .await
-            .context("Failed to get disable_idle config")
-            .log_err(ctx)
-            .unwrap_or_default()
-        {
-            info!(ctx, "IMAP IDLE is disabled, going to fake idle.");
-            return connection
-                .fake_idle(ctx, Some(watch_folder), folder_meaning)
-                .await;
-        }
-
-        info!(ctx, "IMAP session supports IDLE, using it.");
-        match session
-            .idle(
-                ctx,
-                connection.idle_interrupt_receiver.clone(),
-                Some(watch_folder),
-            )
-            .await
-            .context("idle")
-        {
-            Ok((session, info)) => {
-                connection.session = Some(session);
-                info
-            }
-            Err(err) => {
-                connection.trigger_reconnect(ctx);
-                warn!(ctx, "{:#}", err);
-                InterruptInfo::new(false)
-            }
-        }
-    } else {
+    let Some(session) = connection.session.take() else {
         warn!(ctx, "No IMAP session, going to fake idle.");
         connection
             .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await
+            .await;
+        return;
+    };
+
+    if !session.can_idle() {
+        info!(
+            ctx,
+            "IMAP session does not support IDLE, going to fake idle."
+        );
+        connection
+            .fake_idle(ctx, Some(watch_folder), folder_meaning)
+            .await;
+        return;
+    }
+
+    if ctx
+        .get_config_bool(Config::DisableIdle)
+        .await
+        .context("Failed to get disable_idle config")
+        .log_err(ctx)
+        .unwrap_or_default()
+    {
+        info!(ctx, "IMAP IDLE is disabled, going to fake idle.");
+        connection
+            .fake_idle(ctx, Some(watch_folder), folder_meaning)
+            .await;
+        return;
+    }
+
+    info!(ctx, "IMAP session supports IDLE, using it.");
+    match session
+        .idle(
+            ctx,
+            connection.idle_interrupt_receiver.clone(),
+            &watch_folder,
+        )
+        .await
+        .context("idle")
+    {
+        Ok(session) => {
+            connection.session = Some(session);
+        }
+        Err(err) => {
+            connection.trigger_reconnect(ctx);
+            warn!(ctx, "{:#}", err);
+        }
     }
 }
 
@@ -706,8 +709,9 @@ async fn smtp_loop(
         loop {
             if let Err(err) = send_smtp_messages(&ctx, &mut connection).await {
                 warn!(ctx, "send_smtp_messages failed: {:#}", err);
-                timeout = Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
+                timeout = Some(timeout.unwrap_or(30));
             } else {
+                timeout = None;
                 let duration_until_can_send = ctx.ratelimit.read().await.until_can_send();
                 if !duration_until_can_send.is_zero() {
                     info!(
@@ -715,14 +719,9 @@ async fn smtp_loop(
                         "smtp got rate limited, waiting for {} until can send again",
                         duration_to_str(duration_until_can_send)
                     );
-                    tokio::time::timeout(duration_until_can_send, async {
-                        idle_interrupt_receiver.recv().await.unwrap_or_default()
-                    })
-                    .await
-                    .unwrap_or_default();
+                    tokio::time::sleep(duration_until_can_send).await;
                     continue;
                 }
-                timeout = None;
             }
 
             // Fake Idle
@@ -736,17 +735,23 @@ async fn smtp_loop(
             // sending is retried (at the latest) after the timeout. If sending fails
             // again, we increase the timeout exponentially, in order not to do lots of
             // unnecessary retries.
-            if let Some(timeout) = timeout {
+            if let Some(t) = timeout {
+                let now = tokio::time::Instant::now();
                 info!(
                     ctx,
-                    "smtp has messages to retry, planning to retry {} seconds later", timeout
+                    "smtp has messages to retry, planning to retry {} seconds later", t,
                 );
-                let duration = std::time::Duration::from_secs(timeout);
+                let duration = std::time::Duration::from_secs(t);
                 tokio::time::timeout(duration, async {
                     idle_interrupt_receiver.recv().await.unwrap_or_default()
                 })
                 .await
                 .unwrap_or_default();
+                let slept = (tokio::time::Instant::now() - now).as_secs();
+                timeout = Some(cmp::max(
+                    t,
+                    slept.saturating_add(rand::thread_rng().gen_range((slept / 2)..=slept)),
+                ));
             } else {
                 info!(ctx, "smtp has no messages to retry, waiting for interrupt");
                 idle_interrupt_receiver.recv().await.unwrap_or_default();
@@ -860,24 +865,24 @@ impl Scheduler {
 
     fn maybe_network(&self) {
         for b in self.boxes() {
-            b.conn_state.interrupt(InterruptInfo::new(true));
+            b.conn_state.interrupt();
         }
-        self.interrupt_smtp(InterruptInfo::new(true));
+        self.interrupt_smtp();
     }
 
     fn maybe_network_lost(&self) {
         for b in self.boxes() {
-            b.conn_state.interrupt(InterruptInfo::new(false));
+            b.conn_state.interrupt();
         }
-        self.interrupt_smtp(InterruptInfo::new(false));
+        self.interrupt_smtp();
     }
 
-    fn interrupt_inbox(&self, info: InterruptInfo) {
-        self.inbox.conn_state.interrupt(info);
+    fn interrupt_inbox(&self) {
+        self.inbox.conn_state.interrupt();
     }
 
-    fn interrupt_smtp(&self, info: InterruptInfo) {
-        self.smtp.interrupt(info);
+    fn interrupt_smtp(&self) {
+        self.smtp.interrupt();
     }
 
     fn interrupt_ephemeral_task(&self) {
@@ -927,7 +932,7 @@ struct ConnectionState {
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
-    idle_interrupt_sender: Sender<InterruptInfo>,
+    idle_interrupt_sender: Sender<()>,
     /// Mutex to pass connectivity info between IMAP/SMTP threads and the API
     connectivity: ConnectivityStore,
 }
@@ -943,9 +948,9 @@ impl ConnectionState {
         Ok(())
     }
 
-    fn interrupt(&self, info: InterruptInfo) {
+    fn interrupt(&self) {
         // Use try_send to avoid blocking on interrupts.
-        self.idle_interrupt_sender.try_send(info).ok();
+        self.idle_interrupt_sender.try_send(()).ok();
     }
 }
 
@@ -977,8 +982,8 @@ impl SmtpConnectionState {
     }
 
     /// Interrupt any form of idle.
-    fn interrupt(&self, info: InterruptInfo) {
-        self.state.interrupt(info);
+    fn interrupt(&self) {
+        self.state.interrupt();
     }
 
     /// Shutdown this connection completely.
@@ -991,7 +996,7 @@ impl SmtpConnectionState {
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
-    idle_interrupt_receiver: Receiver<InterruptInfo>,
+    idle_interrupt_receiver: Receiver<()>,
 }
 
 #[derive(Debug)]
@@ -1022,8 +1027,8 @@ impl ImapConnectionState {
     }
 
     /// Interrupt any form of idle.
-    fn interrupt(&self, info: InterruptInfo) {
-        self.state.interrupt(info);
+    fn interrupt(&self) {
+        self.state.interrupt();
     }
 
     /// Shutdown this connection completely.
@@ -1038,14 +1043,3 @@ struct ImapConnectionHandlers {
     connection: Imap,
     stop_receiver: Receiver<()>,
 }
-
-#[derive(Default, Debug)]
-pub struct InterruptInfo {
-    pub probe_network: bool,
-}
-
-impl InterruptInfo {
-    pub fn new(probe_network: bool) -> Self {
-        Self { probe_network }
-    }
-}

src/scheduler/connectivity.rs

@@ -2,7 +2,7 @@ use core::fmt;
 use std::cmp::min;
 use std::{iter::once, ops::Deref, sync::Arc};
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use humansize::{format_size, BINARY};
 use tokio::sync::Mutex;
 
@@ -299,6 +299,10 @@ impl Context {
                     .yellow {
                         background-color: #fdc625;
                     }
+                    .not-started-error {
+                        font-size: 2em;
+                        color: red;
+                    }
                 </style>
             </head>
             <body>"#
@@ -318,7 +322,8 @@ impl Context {
                 sched.smtp.state.connectivity.clone(),
             ),
             _ => {
-                return Err(anyhow!("Not started"));
+                ret += "<div class=\"not-started-error\">Error: IO Not Started</div><p>Please report this issue to the app developer.</p>\n</body></html>\n";
+                return Ok(ret);
             }
         };
         drop(lock);

src/securejoin.rs

@@ -8,8 +8,8 @@ use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::{Blocked, Chattype};
-use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
+use crate::constants::Blocked;
+use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::e2ee::ensure_secret_key_exists;
 use crate::events::EventType;
@@ -18,9 +18,10 @@ use crate::key::{load_self_public_key, DcKey, Fingerprint};
 use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
-use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus};
+use crate::peerstate::{Peerstate, PeerstateKeyType};
 use crate::qr::check_qr;
 use crate::stock_str;
+use crate::sync::Sync::*;
 use crate::token;
 use crate::tools::time;
 
@@ -36,17 +37,15 @@ use crate::token::Namespace;
 /// Set of characters to percent-encode in email addresses and names.
 pub const NON_ALPHANUMERIC_WITHOUT_DOT: &AsciiSet = &NON_ALPHANUMERIC.remove(b'.');
 
-macro_rules! inviter_progress {
-    ($context:tt, $contact_id:expr, $progress:expr) => {
-        assert!(
-            $progress >= 0 && $progress <= 1000,
-            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
-        );
-        $context.emit_event($crate::events::EventType::SecurejoinInviterProgress {
-            contact_id: $contact_id,
-            progress: $progress,
-        });
-    };
+fn inviter_progress(context: &Context, contact_id: ContactId, progress: usize) {
+    debug_assert!(
+        progress <= 1000,
+        "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
+    );
+    context.emit_event(EventType::SecurejoinInviterProgress {
+        contact_id,
+        progress,
+    });
 }
 
 /// Generates a Secure Join QR code.
@@ -318,7 +317,7 @@ pub(crate) async fn handle_securejoin_handshake(
             }
             info!(context, "Secure-join requested.",);
 
-            inviter_progress!(context, contact_id, 300);
+            inviter_progress(context, contact_id, 300);
 
             // for setup-contact, make Alice's one-to-one chat with Bob visible
             // (secure-join-information are shown in the group chat)
@@ -415,10 +414,9 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?
                 .get_addr()
                 .to_owned();
-            if mark_peer_as_verified(context, fingerprint.clone(), contact_addr)
-                .await
-                .is_err()
-            {
+            let fingerprint_found =
+                mark_peer_as_verified(context, fingerprint.clone(), contact_addr).await?;
+            if !fingerprint_found {
                 could_not_establish_secure_connection(
                     context,
                     contact_id,
@@ -428,10 +426,11 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
+            contact_id.regossip_keys(context).await?;
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await?;
             info!(context, "Auth verified.",);
             context.emit_event(EventType::ContactsChanged(Some(contact_id)));
-            inviter_progress!(context, contact_id, 600);
+            inviter_progress(context, contact_id, 600);
             if join_vg {
                 // the vg-member-added message is special:
                 // this is a normal Chat-Group-Member-Added message
@@ -446,17 +445,19 @@ pub(crate) async fn handle_securejoin_handshake(
                 match chat::get_chat_id_by_grpid(context, field_grpid).await? {
                     Some((group_chat_id, _, _)) => {
                         secure_connection_established(context, contact_id, group_chat_id).await?;
-                        if let Err(err) =
-                            chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
-                                .await
-                        {
-                            error!(context, "failed to add contact: {}", err);
-                        }
+                        chat::add_contact_to_chat_ex(
+                            context,
+                            Nosync,
+                            group_chat_id,
+                            contact_id,
+                            true,
+                        )
+                        .await?;
                     }
                     None => bail!("Chat {} not found", &field_grpid),
                 }
-                inviter_progress!(context, contact_id, 800);
-                inviter_progress!(context, contact_id, 1000);
+                inviter_progress(context, contact_id, 800);
+                inviter_progress(context, contact_id, 1000);
             } else {
                 // Alice -> Bob
                 secure_connection_established(
@@ -474,53 +475,52 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await
                 .context("failed sending vc-contact-confirm message")?;
 
-                inviter_progress!(context, contact_id, 1000);
+                inviter_progress(context, contact_id, 1000);
             }
             Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed)
         }
-        "vg-member-added" | "vc-contact-confirm" => {
-            /*=======================================================
-            ====             Bob - the joiner's side             ====
-            ====   Step 7 in "Setup verified contact" protocol   ====
-            =======================================================*/
+        /*=======================================================
+        ====             Bob - the joiner's side             ====
+        ====   Step 7 in "Setup verified contact" protocol   ====
+        =======================================================*/
+        "vc-contact-confirm" => match BobState::from_db(&context.sql).await? {
+            Some(bobstate) => bob::handle_contact_confirm(context, bobstate, mime_message).await,
+            None => Ok(HandshakeMessage::Ignore),
+        },
+
+        "vg-member-added" => {
+            let Some(member_added) = mime_message
+                .get_header(HeaderDef::ChatGroupMemberAdded)
+                .map(|s| s.as_str())
+            else {
+                warn!(
+                    context,
+                    "vg-member-added without Chat-Group-Member-Added header"
+                );
+                return Ok(HandshakeMessage::Propagate);
+            };
+            if !context.is_self_addr(member_added).await? {
+                info!(
+                    context,
+                    "Member {member_added} added by unrelated SecureJoin process"
+                );
+                return Ok(HandshakeMessage::Propagate);
+            }
             match BobState::from_db(&context.sql).await? {
                 Some(bobstate) => {
                     bob::handle_contact_confirm(context, bobstate, mime_message).await
                 }
-                None => match join_vg {
-                    true => Ok(HandshakeMessage::Propagate),
-                    false => Ok(HandshakeMessage::Ignore),
-                },
+                None => Ok(HandshakeMessage::Propagate),
             }
         }
+
         "vg-member-added-received" | "vc-contact-confirm-received" => {
             /*==========================================================
             ====              Alice - the inviter side              ====
             ====  Step 8 in "Out-of-band verified groups" protocol  ====
             ==========================================================*/
 
-            if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
-                if contact.is_verified(context).await? == VerifiedStatus::Unverified {
-                    warn!(context, "{} invalid.", step);
-                    return Ok(HandshakeMessage::Ignore);
-                }
-                if join_vg {
-                    let field_grpid = mime_message
-                        .get_header(HeaderDef::SecureJoinGroup)
-                        .map(|s| s.as_str())
-                        .unwrap_or_else(|| "");
-                    if let Err(err) = chat::get_chat_id_by_grpid(context, field_grpid).await {
-                        warn!(context, "Failed to lookup chat_id from grpid: {}", err);
-                        return Err(
-                            err.context(format!("Chat for group {} not found", &field_grpid))
-                        );
-                    }
-                }
-                Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device
-            } else {
-                warn!(context, "{} invalid.", step);
-                Ok(HandshakeMessage::Ignore)
-            }
+            Ok(HandshakeMessage::Done) // "Done" deletes the message
         }
         _ => {
             warn!(context, "invalid step: {}", step);
@@ -614,30 +614,18 @@ pub(crate) async fn observe_securejoin_on_other_device(
                         return Ok(HandshakeMessage::Ignore);
                     }
                 };
-                if let Err(err) = peerstate.set_verified(
-                    PeerstateKeyType::GossipKey,
-                    fingerprint,
-                    PeerstateVerifiedStatus::BidirectVerified,
-                    addr,
-                ) {
-                    could_not_establish_secure_connection(
-                        context,
-                        contact_id,
-                        info_chat_id(context, contact_id).await?,
-                        &format!("Could not mark peer as verified at step {step}: {err}"),
-                    )
-                    .await?;
-                    return Ok(HandshakeMessage::Ignore);
-                }
+                peerstate.set_verified(PeerstateKeyType::GossipKey, fingerprint, addr)?;
                 peerstate.prefer_encrypt = EncryptPreference::Mutual;
                 peerstate.save_to_db(&context.sql).await.unwrap_or_default();
+
+                ChatId::set_protection_for_contact(context, contact_id).await?;
             } else if let Some(fingerprint) =
                 mime_message.get_header(HeaderDef::SecureJoinFingerprint)
             {
                 // FIXME: Old versions of DC send this header instead of gossips. Remove this
                 // eventually.
                 let fingerprint = fingerprint.parse()?;
-                if mark_peer_as_verified(
+                let fingerprint_found = mark_peer_as_verified(
                     context,
                     fingerprint,
                     Contact::get_by_id(context, contact_id)
@@ -645,9 +633,8 @@ pub(crate) async fn observe_securejoin_on_other_device(
                         .get_addr()
                         .to_owned(),
                 )
-                .await
-                .is_err()
-                {
+                .await?;
+                if !fingerprint_found {
                     could_not_establish_secure_connection(
                         context,
                         contact_id,
@@ -672,10 +659,10 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 return Ok(HandshakeMessage::Ignore);
             }
             if step.as_str() == "vg-member-added" {
-                inviter_progress!(context, contact_id, 800);
+                inviter_progress(context, contact_id, 800);
             }
             if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" {
-                inviter_progress!(context, contact_id, 1000);
+                inviter_progress(context, contact_id, 1000);
             }
             if step.as_str() == "vg-request-with-auth" || step.as_str() == "vc-request-with-auth" {
                 // This actually reflects what happens on the first device (which does the secure
@@ -698,24 +685,21 @@ async fn secure_connection_established(
     contact_id: ContactId,
     chat_id: ChatId,
 ) -> Result<()> {
-    let contact = Contact::get_by_id(context, contact_id).await?;
-    let msg = stock_str::contact_verified(context, &contact).await;
-    chat::add_info_msg(context, chat_id, &msg, time()).await?;
     if context
         .get_config_bool(Config::VerifiedOneOnOneChats)
         .await?
     {
-        let chat = Chat::load_from_db(context, chat_id).await?;
-        if chat.typ == Chattype::Single {
-            chat_id
-                .set_protection(
-                    context,
-                    ProtectionStatus::Protected,
-                    time(),
-                    Some(contact_id),
-                )
-                .await?;
-        }
+        let private_chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Yes)
+            .await?
+            .id;
+        private_chat_id
+            .set_protection(
+                context,
+                ProtectionStatus::Protected,
+                time(),
+                Some(contact_id),
+            )
+            .await?;
     }
     context.emit_event(EventType::ChatModified(chat_id));
     Ok(())
@@ -737,27 +721,21 @@ async fn could_not_establish_secure_connection(
     Ok(())
 }
 
+/// Tries to mark peer with provided key fingerprint as verified.
+///
+/// Returns true if such key was found, false otherwise.
 async fn mark_peer_as_verified(
     context: &Context,
     fingerprint: Fingerprint,
     verifier: String,
-) -> Result<()> {
-    if let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, &fingerprint).await? {
-        if let Err(err) = peerstate.set_verified(
-            PeerstateKeyType::PublicKey,
-            fingerprint,
-            PeerstateVerifiedStatus::BidirectVerified,
-            verifier,
-        ) {
-            error!(context, "Could not mark peer as verified: {}", err);
-            return Err(err);
-        }
-        peerstate.prefer_encrypt = EncryptPreference::Mutual;
-        peerstate.save_to_db(&context.sql).await.unwrap_or_default();
-        Ok(())
-    } else {
-        bail!("no peerstate in db for fingerprint {}", fingerprint.hex());
-    }
+) -> Result<bool> {
+    let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, &fingerprint).await? else {
+        return Ok(false);
+    };
+    peerstate.set_verified(PeerstateKeyType::PublicKey, fingerprint, verifier)?;
+    peerstate.prefer_encrypt = EncryptPreference::Mutual;
+    peerstate.save_to_db(&context.sql).await.unwrap_or_default();
+    Ok(true)
 }
 
 /* ******************************************************************************
@@ -794,6 +772,7 @@ mod tests {
     use crate::chat;
     use crate::chat::ProtectionStatus;
     use crate::chatlist::Chatlist;
+    use crate::constants::Chattype;
     use crate::contact::ContactAddress;
     use crate::contact::VerifiedStatus;
     use crate::peerstate::Peerstate;
@@ -943,25 +922,10 @@ mod tests {
         // Check Alice got the verified message in her 1:1 chat.
         {
             let chat = alice.create_chat(&bob).await;
-            let msg_ids: Vec<_> = chat::get_chat_msgs(&alice.ctx, chat.get_id())
-                .await
-                .unwrap()
-                .into_iter()
-                .filter_map(|item| match item {
-                    chat::ChatItem::Message { msg_id } => Some(msg_id),
-                    _ => None,
-                })
-                .collect();
-            assert_eq!(msg_ids.len(), 2);
-
-            let msg0 = Message::load_from_db(&alice.ctx, msg_ids[0]).await.unwrap();
-            assert!(msg0.is_info());
-            assert!(msg0.get_text().contains("bob@example.net verified"));
-
-            let msg1 = Message::load_from_db(&alice.ctx, msg_ids[1]).await.unwrap();
-            assert!(msg1.is_info());
+            let msg = get_chat_msg(&alice, chat.get_id(), 0, 1).await;
+            assert!(msg.is_info());
             let expected_text = chat_protection_enabled(&alice).await;
-            assert_eq!(msg1.get_text(), expected_text);
+            assert_eq!(msg.get_text(), expected_text);
         }
 
         // Check Alice sent the right message to Bob.
@@ -997,24 +961,10 @@ mod tests {
         // Check Bob got the verified message in his 1:1 chat.
         {
             let chat = bob.create_chat(&alice).await;
-            let msg_ids: Vec<_> = chat::get_chat_msgs(&bob.ctx, chat.get_id())
-                .await
-                .unwrap()
-                .into_iter()
-                .filter_map(|item| match item {
-                    chat::ChatItem::Message { msg_id } => Some(msg_id),
-                    _ => None,
-                })
-                .collect();
-
-            let msg0 = Message::load_from_db(&bob.ctx, msg_ids[0]).await.unwrap();
-            assert!(msg0.is_info());
-            assert!(msg0.get_text().contains("alice@example.org verified"));
-
-            let msg1 = Message::load_from_db(&bob.ctx, msg_ids[1]).await.unwrap();
-            assert!(msg1.is_info());
+            let msg = get_chat_msg(&bob, chat.get_id(), 0, 1).await;
+            assert!(msg.is_info());
             let expected_text = chat_protection_enabled(&bob).await;
-            assert_eq!(msg1.get_text(), expected_text);
+            assert_eq!(msg.get_text(), expected_text);
         }
 
         // Check Bob sent the final message
@@ -1054,8 +1004,11 @@ mod tests {
             gossip_key_fingerprint: Some(alice_pubkey.fingerprint()),
             verified_key: None,
             verified_key_fingerprint: None,
-            fingerprint_changed: false,
             verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            fingerprint_changed: false,
         };
         peerstate.save_to_db(&bob.ctx.sql).await?;
 
@@ -1310,11 +1263,11 @@ mod tests {
             );
             // There should be 3 messages in the chat:
             // - The ChatProtectionEnabled message
-            // - bob@example.net verified
             // - You added member bob@example.net
-            let msg = get_chat_msg(&alice, alice_chatid, 1, 3).await;
+            let msg = get_chat_msg(&alice, alice_chatid, 0, 2).await;
             assert!(msg.is_info());
-            assert!(msg.get_text().contains("bob@example.net verified"));
+            let expected_text = chat_protection_enabled(&alice).await;
+            assert_eq!(msg.get_text(), expected_text);
         }
 
         // Bob should not yet have Alice verified
@@ -1350,27 +1303,6 @@ mod tests {
                     println!("msg {msg_id} text: {text}");
                 }
             }
-            let mut msg_iter = chat::get_chat_msgs(&bob.ctx, bob_chatid)
-                .await
-                .unwrap()
-                .into_iter();
-            loop {
-                match msg_iter.next() {
-                    Some(chat::ChatItem::Message { msg_id }) => {
-                        let msg = Message::load_from_db(&bob.ctx, msg_id).await.unwrap();
-                        let text = msg.get_text();
-                        match text.contains("alice@example.org verified") {
-                            true => {
-                                assert!(msg.is_info());
-                                break;
-                            }
-                            false => continue,
-                        }
-                    }
-                    Some(_) => continue,
-                    None => panic!("Verified message not found in Bob's group chat"),
-                }
-            }
         }
 
         let sent = bob.pop_sent_msg().await;

src/securejoin/bob.rs

@@ -15,6 +15,7 @@ use crate::contact::Contact;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::mimeparser::MimeMessage;
+use crate::sync::Sync::*;
 use crate::tools::time;
 use crate::{chat, stock_str};
 
@@ -110,7 +111,7 @@ pub(super) async fn handle_auth_required(
 /// Handles `vc-contact-confirm` and `vg-member-added` handshake messages.
 ///
 /// # Bob - the joiner's side
-/// ## Step 4 in the "Setup Contact protocol"
+/// ## Step 7 in the "Setup Contact protocol"
 pub(super) async fn handle_contact_confirm(
     context: &Context,
     mut bobstate: BobState,
@@ -131,6 +132,7 @@ pub(super) async fn handle_contact_confirm(
             // verify both contacts (this could be a bug/security issue, see
             // e.g. https://github.com/deltachat/deltachat-core-rust/issues/1177).
             bobstate.notify_peer_verified(context).await?;
+            bobstate.emit_progress(context, JoinerProgress::Succeeded);
             Ok(retval)
         }
         Some(_) => {
@@ -179,7 +181,7 @@ impl BobState {
             } => {
                 let group_chat_id = match chat::get_chat_id_by_grpid(context, grpid).await? {
                     Some((chat_id, _protected, _blocked)) => {
-                        chat_id.unblock(context).await?;
+                        chat_id.unblock_ex(context, Nosync).await?;
                         chat_id
                     }
                     None => {
@@ -220,16 +222,13 @@ impl BobState {
     /// This creates an info message in the chat being joined.
     async fn notify_peer_verified(&self, context: &Context) -> Result<()> {
         let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        let msg = stock_str::contact_verified(context, &contact).await;
         let chat_id = self.joining_chat_id(context).await?;
-        chat::add_info_msg(context, chat_id, &msg, time()).await?;
 
         if context
             .get_config_bool(Config::VerifiedOneOnOneChats)
             .await?
-            && chat_id == self.alice_chat()
         {
-            chat_id
+            self.alice_chat()
                 .set_protection(
                     context,
                     ProtectionStatus::Protected,
@@ -255,8 +254,8 @@ enum JoinerProgress {
     ///
     /// Typically shows as "alice@addr verified, introducing myself."
     RequestWithAuthSent,
-    // /// Completed securejoin.
-    // Succeeded,
+    /// Completed securejoin.
+    Succeeded,
 }
 
 impl From<JoinerProgress> for usize {
@@ -264,7 +263,7 @@ impl From<JoinerProgress> for usize {
         match progress {
             JoinerProgress::Error => 0,
             JoinerProgress::RequestWithAuthSent => 400,
-            // JoinerProgress::Succeeded => 1000,
+            JoinerProgress::Succeeded => 1000,
         }
     }
 }

src/securejoin/bobstate.rs

@@ -7,7 +7,7 @@
 //! The [`BobState`] is only directly used to initially create it when starting the
 //! protocol.
 
-use anyhow::{Error, Result};
+use anyhow::Result;
 use rusqlite::Connection;
 
 use super::qrinvite::QrInvite;
@@ -335,36 +335,6 @@ impl BobState {
             context,
             "Bob Step 7 - handling vc-contact-confirm/vg-member-added message"
         );
-        let vg_expect_encrypted = match self.invite {
-            QrInvite::Contact { .. } => {
-                // setup-contact is always encrypted
-                true
-            }
-            QrInvite::Group { ref grpid, .. } => {
-                // This is buggy, result will always be
-                // false since the group is created by receive_imf for
-                // the very handshake message we're handling now.  But
-                // only after we have returned.  It does not impact
-                // the security invariants of secure-join however.
-
-                chat::get_chat_id_by_grpid(context, grpid)
-                    .await?
-                    .map_or(false, |(_chat_id, is_protected, _blocked)| is_protected)
-                // when joining a non-verified group
-                // the vg-member-added message may be unencrypted
-                // when not all group members have keys or prefer encryption.
-                // So only expect encryption if this is a verified group
-            }
-        };
-        if vg_expect_encrypted
-            && !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint()))
-        {
-            self.update_next(&context.sql, SecureJoinStep::Terminated)
-                .await?;
-            return Ok(Some(BobHandshakeStage::Terminated(
-                "Contact confirm message not encrypted",
-            )));
-        }
         mark_peer_as_verified(
             context,
             self.invite.fingerprint().clone(),
@@ -375,17 +345,6 @@ impl BobState {
             .await?;
         context.emit_event(EventType::ContactsChanged(None));
 
-        if let QrInvite::Group { .. } = self.invite {
-            let member_added = mime_message
-                .get_header(HeaderDef::ChatGroupMemberAdded)
-                .map(|s| s.as_str())
-                .ok_or_else(|| Error::msg("Missing Chat-Group-Member-Added header"))?;
-            if !context.is_self_addr(member_added).await? {
-                info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
-                return Ok(None);
-            }
-        }
-
         self.send_handshake_message(context, BobHandshakeMsg::ContactConfirmReceived)
             .await
             .map_err(|_| {

src/sql/migrations.rs

@@ -750,6 +750,19 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         .await?;
     }
 
+    if dbversion < 104 {
+        sql.execute_migration(
+            "ALTER TABLE acpeerstates
+             ADD COLUMN secondary_verified_key;
+             ALTER TABLE acpeerstates
+             ADD COLUMN secondary_verified_key_fingerprint TEXT DEFAULT '';
+             ALTER TABLE acpeerstates
+             ADD COLUMN secondary_verifier TEXT DEFAULT ''",
+            104,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?

src/stock_str.rs

@@ -413,6 +413,12 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "%1$s sent a message from another device."))]
     ChatProtectionDisabled = 171,
+
+    #[strum(props(fallback = "Others will only see this group after you sent a first message."))]
+    NewGroupSendFirstMessage = 172,
+
+    #[strum(props(fallback = "Member %1$s added."))]
+    MsgAddMember = 173,
 }
 
 impl StockMessage {
@@ -600,6 +606,7 @@ pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: ContactId
 }
 
 /// Stock string: `I added member %1$s.`.
+/// This one is for sending in group chats.
 ///
 /// The `added_member_addr` parameter should be an email address and is looked up in the
 /// contacts to combine with the authorized display name.
@@ -634,7 +641,11 @@ pub(crate) async fn msg_add_member_local(
             .unwrap_or_else(|_| addr.to_string()),
         _ => addr.to_string(),
     };
-    if by_contact == ContactId::SELF {
+    if by_contact == ContactId::UNDEFINED {
+        translated(context, StockMessage::MsgAddMember)
+            .await
+            .replace1(whom)
+    } else if by_contact == ContactId::SELF {
         translated(context, StockMessage::MsgYouAddMember)
             .await
             .replace1(whom)
@@ -813,6 +824,7 @@ pub(crate) async fn secure_join_group_qr_description(context: &Context, chat: &C
 }
 
 /// Stock string: `%1$s verified.`.
+#[allow(dead_code)]
 pub(crate) async fn contact_verified(context: &Context, contact: &Contact) -> String {
     let addr = &contact.get_name_n_addr();
     translated(context, StockMessage::ContactVerified)
@@ -1284,6 +1296,11 @@ pub(crate) async fn aeap_explanation_and_link(
         .replace2(new_addr)
 }
 
+/// Stock string: `Others will only see this group after you sent a first message.`.
+pub(crate) async fn new_group_send_first_message(context: &Context) -> String {
+    translated(context, StockMessage::NewGroupSendFirstMessage).await
+}
+
 /// Text to put in the [`Qr::Backup`] rendered SVG image.
 ///
 /// The default is "Scan to set up second device for <account name (account addr)>".  The

src/sync.rs

@@ -5,18 +5,35 @@ use lettre_email::mime::{self};
 use lettre_email::PartBuilder;
 use serde::{Deserialize, Serialize};
 
-use crate::chat::{Chat, ChatId};
+use crate::chat::{self, Chat, ChatId};
 use crate::config::Config;
 use crate::constants::Blocked;
 use crate::contact::ContactId;
 use crate::context::Context;
+use crate::log::LogExt;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
-use crate::sync::SyncData::{AddQrToken, DeleteQrToken};
+use crate::sync::SyncData::{AddQrToken, AlterChat, DeleteQrToken};
 use crate::token::Namespace;
 use crate::tools::time;
-use crate::{chat, stock_str, token};
+use crate::{stock_str, token};
+
+/// Whether to send device sync messages. Aimed for usage in the internal API.
+#[derive(Debug)]
+pub(crate) enum Sync {
+    Nosync,
+    Sync,
+}
+
+impl From<Sync> for bool {
+    fn from(sync: Sync) -> bool {
+        match sync {
+            Sync::Nosync => false,
+            Sync::Sync => true,
+        }
+    }
+}
 
 #[derive(Debug, Serialize, Deserialize)]
 pub(crate) struct QrTokenData {
@@ -29,12 +46,24 @@ pub(crate) struct QrTokenData {
 pub(crate) enum SyncData {
     AddQrToken(QrTokenData),
     DeleteQrToken(QrTokenData),
+    AlterChat {
+        id: chat::SyncId,
+        action: chat::SyncAction,
+    },
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(untagged)]
+pub(crate) enum SyncDataOrUnknown {
+    SyncData(SyncData),
+    Unknown(serde_json::Value),
 }
 
 #[derive(Debug, Serialize, Deserialize)]
 pub(crate) struct SyncItem {
     timestamp: i64,
-    data: SyncData,
+
+    data: SyncDataOrUnknown,
 }
 
 #[derive(Debug, Deserialize)]
@@ -42,8 +71,17 @@ pub(crate) struct SyncItems {
     items: Vec<SyncItem>,
 }
 
+impl From<SyncData> for SyncDataOrUnknown {
+    fn from(sync_data: SyncData) -> Self {
+        Self::SyncData(sync_data)
+    }
+}
+
 impl Context {
     /// Adds an item to the list of items that should be synchronized to other devices.
+    ///
+    /// NB: Private and `pub(crate)` functions shouldn't call this unless `Sync::Sync` is explicitly
+    /// passed to them. This way it's always clear whether the code performs synchronisation.
     pub(crate) async fn add_sync_item(&self, data: SyncData) -> Result<()> {
         self.add_sync_item_with_timestamp(data, time()).await
     }
@@ -55,7 +93,10 @@ impl Context {
             return Ok(());
         }
 
-        let item = SyncItem { timestamp, data };
+        let item = SyncItem {
+            timestamp,
+            data: data.into(),
+        };
         let item = serde_json::to_string(&item)?;
         self.sql
             .execute("INSERT INTO multi_device_sync (item) VALUES(?);", (item,))
@@ -204,58 +245,71 @@ impl Context {
         Ok(sync_items)
     }
 
-    /// Execute sync items.
+    /// Executes sync items sent by other device.
     ///
     /// CAVE: When changing the code to handle other sync items,
     /// take care that does not result in calls to `add_sync_item()`
     /// as otherwise we would add in a dead-loop between two devices
     /// sending message back and forth.
     ///
-    /// If an error is returned, the caller shall not try over.
-    /// Therefore, errors should only be returned on database errors or so.
-    /// If eg. just an item cannot be deleted,
-    /// that should not hold off the other items to be executed.
-    pub(crate) async fn execute_sync_items(&self, items: &SyncItems) -> Result<()> {
+    /// If an error is returned, the caller shall not try over because some sync items could be
+    /// already executed. Sync items are considered independent and executed in the given order but
+    /// regardless of whether executing of the previous items succeeded.
+    pub(crate) async fn execute_sync_items(&self, items: &SyncItems) {
         info!(self, "executing {} sync item(s)", items.items.len());
         for item in &items.items {
             match &item.data {
-                AddQrToken(token) => {
-                    let chat_id = if let Some(grpid) = &token.grpid {
-                        if let Some((chat_id, _, _)) =
-                            chat::get_chat_id_by_grpid(self, grpid).await?
-                        {
-                            Some(chat_id)
-                        } else {
-                            warn!(
-                                self,
-                                "Ignoring token for nonexistent/deleted group '{}'.", grpid
-                            );
-                            continue;
-                        }
-                    } else {
-                        None
-                    };
-                    token::save(self, Namespace::InviteNumber, chat_id, &token.invitenumber)
-                        .await?;
-                    token::save(self, Namespace::Auth, chat_id, &token.auth).await?;
-                }
-                DeleteQrToken(token) => {
-                    token::delete(self, Namespace::InviteNumber, &token.invitenumber).await?;
-                    token::delete(self, Namespace::Auth, &token.auth).await?;
+                SyncDataOrUnknown::SyncData(data) => match data {
+                    AddQrToken(token) => self.add_qr_token(token).await,
+                    DeleteQrToken(token) => self.delete_qr_token(token).await,
+                    AlterChat { id, action } => self.sync_alter_chat(id, action).await,
+                },
+                SyncDataOrUnknown::Unknown(data) => {
+                    warn!(self, "Ignored unknown sync item: {data}.");
+                    Ok(())
                 }
             }
+            .log_err(self)
+            .ok();
         }
+    }
+
+    async fn add_qr_token(&self, token: &QrTokenData) -> Result<()> {
+        let chat_id = if let Some(grpid) = &token.grpid {
+            if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(self, grpid).await? {
+                Some(chat_id)
+            } else {
+                warn!(
+                    self,
+                    "Ignoring token for nonexistent/deleted group '{}'.", grpid
+                );
+                return Ok(());
+            }
+        } else {
+            None
+        };
+        token::save(self, Namespace::InviteNumber, chat_id, &token.invitenumber).await?;
+        token::save(self, Namespace::Auth, chat_id, &token.auth).await?;
+        Ok(())
+    }
+
+    async fn delete_qr_token(&self, token: &QrTokenData) -> Result<()> {
+        token::delete(self, Namespace::InviteNumber, &token.invitenumber).await?;
+        token::delete(self, Namespace::Auth, &token.auth).await?;
         Ok(())
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use std::time::{Duration, SystemTime};
+
     use anyhow::bail;
 
     use super::*;
     use crate::chat::Chat;
     use crate::chatlist::Chatlist;
+    use crate::contact::{Contact, Origin};
     use crate::test_utils::TestContext;
     use crate::token::Namespace;
 
@@ -277,6 +331,20 @@ mod tests {
 
         assert!(t.build_sync_json().await?.is_none());
 
+        // Having one test on `SyncData::AlterChat` is sufficient here as
+        // `chat::SyncAction::SetMuted` introduces enums inside items and `SystemTime`. Let's avoid
+        // in-depth testing of the serialiser here which is an external crate.
+        t.add_sync_item_with_timestamp(
+            SyncData::AlterChat {
+                id: chat::SyncId::ContactAddr("bob@example.net".to_string()),
+                action: chat::SyncAction::SetMuted(chat::MuteDuration::Until(
+                    SystemTime::UNIX_EPOCH + Duration::from_millis(42999),
+                )),
+            },
+            1631781315,
+        )
+        .await?;
+
         t.add_sync_item_with_timestamp(
             SyncData::AddQrToken(QrTokenData {
                 invitenumber: "testinvite".to_string(),
@@ -300,6 +368,7 @@ mod tests {
         assert_eq!(
             serialized,
             r#"{"items":[
+{"timestamp":1631781315,"data":{"AlterChat":{"id":{"ContactAddr":"bob@example.net"},"action":{"SetMuted":{"Until":{"secs_since_epoch":42,"nanos_since_epoch":999000000}}}}}},
 {"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"testinvite","auth":"testauth","grpid":"group123"}}},
 {"timestamp":1631781317,"data":{"DeleteQrToken":{"invitenumber":"123!?\":.;{}","auth":"456","grpid":null}}}
 ]}"#
@@ -310,7 +379,7 @@ mod tests {
         assert!(t.build_sync_json().await?.is_none());
 
         let sync_items = t.parse_sync_items(serialized)?;
-        assert_eq!(sync_items.items.len(), 2);
+        assert_eq!(sync_items.items.len(), 3);
 
         Ok(())
     }
@@ -337,36 +406,44 @@ mod tests {
 
         assert!(t.parse_sync_items(r#"{"badname":[]}"#.to_string()).is_err());
 
-        assert!(t.parse_sync_items(
-                r#"{"items":[{"timestamp":1631781316,"data":{"BadItem":{"invitenumber":"in","auth":"a","grpid":null}}}]}"#
-                    .to_string(),
-            )
-            .is_err());
+        for bad_item_example in [
+            r#"{"items":[{"timestamp":1631781316,"data":{"BadItem":{"invitenumber":"in","auth":"a","grpid":null}}}]}"#,
+            r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":123}}}]}"#, // `123` is invalid for `String`
+            r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":true}}}]}"#, // `true` is invalid for `String`
+            r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":[]}}}]}"#, // `[]` is invalid for `String`
+            r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":{}}}}]}"#, // `{}` is invalid for `String`
+            r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","grpid":null}}}]}"#, // missing field
+            r#"{"items":[{"timestamp":1631781316,"data":{"AlterChat":{"id":{"ContactAddr":"bob@example.net"},"action":"Burn"}}}]}"#, // Unknown enum value
+        ] {
+            let sync_items = t.parse_sync_items(bad_item_example.to_string()).unwrap();
+            assert_eq!(sync_items.items.len(), 1);
+            assert!(matches!(sync_items.items[0].timestamp, 1631781316));
+            assert!(matches!(
+                sync_items.items[0].data,
+                SyncDataOrUnknown::Unknown(_)
+            ));
+        }
 
-        assert!(t.parse_sync_items(
-                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":123}}}]}"#.to_string(),
-            )
-            .is_err()); // `123` is invalid for `String`
-
-        assert!(t.parse_sync_items(
-                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":true}}}]}"#.to_string(),
-            )
-            .is_err()); // `true` is invalid for `String`
-
-        assert!(t.parse_sync_items(
-                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":[]}}}]}"#.to_string(),
-            )
-            .is_err()); // `[]` is invalid for `String`
-
-        assert!(t.parse_sync_items(
-                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":{}}}}]}"#.to_string(),
-            )
-            .is_err()); // `{}` is invalid for `String`
-
-        assert!(t.parse_sync_items(
-                r#"{"items":[{"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","grpid":null}}}]}"#.to_string(),
-            )
-            .is_err()); // missing field
+        // Test enums inside items and SystemTime
+        let sync_items = t.parse_sync_items(
+            r#"{"items":[{"timestamp":1631781318,"data":{"AlterChat":{"id":{"ContactAddr":"bob@example.net"},"action":{"SetMuted":{"Until":{"secs_since_epoch":42,"nanos_since_epoch":999000000}}}}}}]}"#.to_string(),
+        )?;
+        assert_eq!(sync_items.items.len(), 1);
+        let SyncDataOrUnknown::SyncData(AlterChat { id, action }) =
+            &sync_items.items.get(0).unwrap().data
+        else {
+            bail!("bad item");
+        };
+        assert_eq!(
+            *id,
+            chat::SyncId::ContactAddr("bob@example.net".to_string())
+        );
+        assert_eq!(
+            *action,
+            chat::SyncAction::SetMuted(chat::MuteDuration::Until(
+                SystemTime::UNIX_EPOCH + Duration::from_millis(42999)
+            ))
+        );
 
         // empty item list is okay
         assert_eq!(
@@ -396,7 +473,9 @@ mod tests {
         )?;
 
         assert_eq!(sync_items.items.len(), 1);
-        if let AddQrToken(token) = &sync_items.items.get(0).unwrap().data {
+        if let SyncDataOrUnknown::SyncData(AddQrToken(token)) =
+            &sync_items.items.get(0).unwrap().data
+        {
             assert_eq!(token.invitenumber, "in");
             assert_eq!(token.auth, "yip");
             assert_eq!(token.grpid, None);
@@ -423,6 +502,7 @@ mod tests {
         let sync_items = t
             .parse_sync_items(
                 r#"{"items":[
+{"timestamp":1631781315,"data":{"AlterChat":{"id":{"ContactAddr":"bob@example.net"},"action":"Block"}}},
 {"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"yip-in","auth":"a"}}},
 {"timestamp":1631781316,"data":{"DeleteQrToken":{"invitenumber":"in","auth":"delete unexistent, shall continue"}}},
 {"timestamp":1631781316,"data":{"AddQrToken":{"invitenumber":"in","auth":"yip-auth"}}},
@@ -433,8 +513,13 @@ mod tests {
                 .to_string(),
             )
             ?;
-        t.execute_sync_items(&sync_items).await?;
+        t.execute_sync_items(&sync_items).await;
 
+        assert!(
+            Contact::lookup_id_by_addr(&t, "bob@example.net", Origin::Unknown)
+                .await?
+                .is_none()
+        );
         assert!(token::exists(&t, Namespace::InviteNumber, "yip-in").await);
         assert!(token::exists(&t, Namespace::Auth, "yip-auth").await);
         assert!(!token::exists(&t, Namespace::Auth, "non-existent").await);

src/test_utils.rs

@@ -22,8 +22,8 @@ use tokio::sync::RwLock;
 use tokio::{fs, task};
 
 use crate::chat::{
-    self, add_to_chat_contacts_table, create_group_chat, Chat, ChatId, MessageListOptions,
-    ProtectionStatus,
+    self, add_to_chat_contacts_table, create_group_chat, Chat, ChatId, ChatIdBlocked,
+    MessageListOptions, ProtectionStatus,
 };
 use crate::chatlist::Chatlist;
 use crate::config::Config;
@@ -374,6 +374,11 @@ impl TestContext {
             }
         });
 
+        ctx.set_config(Config::SkipStartMessages, Some("1"))
+            .await
+            .unwrap();
+        ctx.set_config(Config::SyncMsgs, Some("0")).await.unwrap();
+
         Self {
             ctx,
             dir,
@@ -509,12 +514,7 @@ impl TestContext {
             .await
             .expect("receive_imf() seems not to have added a new message to the db");
 
-        assert_eq!(
-            received.msg_ids.len(),
-            1,
-            "recv_msg() can currently only receive messages with exactly one part"
-        );
-        let msg = Message::load_from_db(self, received.msg_ids[0])
+        let msg = Message::load_from_db(self, *received.msg_ids.last().unwrap())
             .await
             .unwrap();
 
@@ -592,14 +592,17 @@ impl TestContext {
     }
 
     /// Returns 1:1 [`Chat`] with another account. Panics if it doesn't exist.
+    /// May return a blocked chat.
     ///
     /// This first creates a contact using the configured details on the other account, then
     /// gets the 1:1 chat with this contact.
     pub async fn get_chat(&self, other: &TestContext) -> Chat {
         let contact = self.add_or_lookup_contact(other).await;
-        let chat_id = ChatId::lookup_by_contact(&self.ctx, contact.id)
+
+        let chat_id = ChatIdBlocked::lookup_by_contact(&self.ctx, contact.id)
             .await
             .unwrap()
+            .map(|chat_id_blocked| chat_id_blocked.id)
             .expect(
                 "There is no chat with this contact. \
                 Hint: Use create_chat() instead of get_chat() if this is expected.",
@@ -711,7 +714,9 @@ impl TestContext {
             })
             .collect();
 
-        let sel_chat = Chat::load_from_db(self, chat_id).await.unwrap();
+        let Ok(sel_chat) = Chat::load_from_db(self, chat_id).await else {
+            return String::from("Can't load chat\n");
+        };
         let members = chat::get_chat_contacts(self, sel_chat.id).await.unwrap();
         let subtitle = if sel_chat.is_device_talk() {
             "device-talk".to_string()

src/tests/verified_chats.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use pretty_assertions::assert_eq;
 
-use crate::chat::{Chat, ProtectionStatus};
+use crate::chat::ProtectionStatus;
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::DC_GCL_FOR_FORWARDING;
@@ -126,24 +126,22 @@ async fn test_create_verified_oneonone_chat() -> Result<()> {
         VerifiedStatus::BidirectVerified
     );
 
-    // As soon as Alice creates a chat with Fiona, it should directly be protected
+    // Alice should have a hidden protected chat with Fiona
     {
-        let chat = alice.create_chat(&fiona).await;
+        let chat = alice.get_chat(&fiona).await;
         assert!(chat.is_protected());
 
-        let msg = alice.get_last_msg().await;
+        let msg = get_chat_msg(&alice, chat.id, 0, 1).await;
         let expected_text = stock_str::chat_protection_enabled(&alice).await;
         assert_eq!(msg.text, expected_text);
     }
 
-    // Fiona should also see the chat as protected
+    // Fiona should have a hidden protected chat with Alice
     {
-        let rcvd = tcm.send_recv(&alice, &fiona, "Hi Fiona").await;
-        let alice_fiona_id = rcvd.chat_id;
-        let chat = Chat::load_from_db(&fiona, alice_fiona_id).await?;
+        let chat = fiona.get_chat(&alice).await;
         assert!(chat.is_protected());
 
-        let msg0 = get_chat_msg(&fiona, chat.id, 0, 2).await;
+        let msg0 = get_chat_msg(&fiona, chat.id, 0, 1).await;
         let expected_text = stock_str::chat_protection_enabled(&fiona).await;
         assert_eq!(msg0.text, expected_text);
     }
@@ -165,6 +163,15 @@ async fn test_create_verified_oneonone_chat() -> Result<()> {
         assert!(!chat.is_protected());
         assert!(chat.is_protection_broken());
 
+        let msg1 = get_chat_msg(&alice, chat.id, 0, 3).await;
+        assert_eq!(msg1.get_info_type(), SystemMessage::ChatProtectionEnabled);
+
+        let msg2 = get_chat_msg(&alice, chat.id, 1, 3).await;
+        assert_eq!(msg2.get_info_type(), SystemMessage::ChatProtectionDisabled);
+
+        let msg2 = get_chat_msg(&alice, chat.id, 2, 3).await;
+        assert_eq!(msg2.text, "I have a new device");
+
         // After recreating the chat, it should still be unprotected
         chat.id.delete(&alice).await?;
 
@@ -705,6 +712,39 @@ async fn test_break_protection_then_verify_again() -> Result<()> {
     Ok(())
 }
 
+/// Regression test for the following bug:
+///
+/// - Scan your chat partner's QR Code
+/// - They change devices
+/// - Scan their QR code again
+///
+/// -> The re-verification fails.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_verify_then_verify_again() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    let bob = tcm.bob().await;
+    enable_verified_oneonone_chats(&[&alice, &bob]).await;
+
+    mark_as_verified(&alice, &bob).await;
+    mark_as_verified(&bob, &alice).await;
+
+    alice.create_chat(&bob).await;
+    assert_verified(&alice, &bob, ProtectionStatus::Protected).await;
+
+    tcm.section("Bob reinstalls DC");
+    drop(bob);
+    let bob_new = tcm.unconfigured().await;
+    enable_verified_oneonone_chats(&[&bob_new]).await;
+    bob_new.configure_addr("bob@example.net").await;
+    e2ee::ensure_secret_key_exists(&bob_new).await?;
+
+    tcm.execute_securejoin(&bob_new, &alice).await;
+    assert_verified(&alice, &bob_new, ProtectionStatus::Protected).await;
+
+    Ok(())
+}
+
 /// Regression test:
 /// - Verify a contact
 /// - The contact stops using DC and sends a message from a classical MUA instead

src/tools.rs

@@ -20,6 +20,7 @@ use mailparse::headers::Headers;
 use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
 use tokio::{fs, io};
+use url::Url;
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
 use crate::constants::{DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
@@ -481,7 +482,43 @@ pub(crate) fn time() -> i64 {
         .as_secs() as i64
 }
 
-/// Very simple email address wrapper.
+/// Struct containing all mailto information
+#[derive(Debug, Default, Eq, PartialEq)]
+pub struct MailTo {
+    pub to: Vec<EmailAddress>,
+    pub subject: Option<String>,
+    pub body: Option<String>,
+}
+
+/// Parse mailto urls
+pub fn parse_mailto(mailto_url: &str) -> Option<MailTo> {
+    if let Ok(url) = Url::parse(mailto_url) {
+        if url.scheme() == "mailto" {
+            let mut mailto: MailTo = Default::default();
+            // Extract the email address
+            url.path().split(',').for_each(|email| {
+                if let Ok(email) = EmailAddress::new(email) {
+                    mailto.to.push(email);
+                }
+            });
+
+            // Extract query parameters
+            for (key, value) in url.query_pairs() {
+                if key == "subject" {
+                    mailto.subject = Some(value.to_string());
+                } else if key == "body" {
+                    mailto.body = Some(value.to_string());
+                }
+            }
+            Some(mailto)
+        } else {
+            None
+        }
+    } else {
+        None
+    }
+}
+
 ///
 /// Represents an email address, right now just the `name@domain` portion.
 ///
@@ -1283,4 +1320,55 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true";
         assert_eq!(remove_subject_prefix("Fwd: Subject"), "Subject");
         assert_eq!(remove_subject_prefix("Fw: Subject"), "Subject");
     }
+
+    #[test]
+    fn test_parse_mailto() {
+        let mailto_url = "mailto:someone@example.com";
+        let reps = parse_mailto(mailto_url);
+        assert_eq!(
+            Some(MailTo {
+                to: vec![EmailAddress {
+                    local: "someone".to_string(),
+                    domain: "example.com".to_string()
+                }],
+                subject: None,
+                body: None
+            }),
+            reps
+        );
+
+        let mailto_url = "mailto:someone@example.com?subject=Hello%20World";
+        let reps = parse_mailto(mailto_url);
+        assert_eq!(
+            Some(MailTo {
+                to: vec![EmailAddress {
+                    local: "someone".to_string(),
+                    domain: "example.com".to_string()
+                }],
+                subject: Some("Hello World".to_string()),
+                body: None
+            }),
+            reps
+        );
+
+        let mailto_url = "mailto:someone@example.com,someoneelse@example.com?subject=Hello%20World&body=This%20is%20a%20test";
+        let reps = parse_mailto(mailto_url);
+        assert_eq!(
+            Some(MailTo {
+                to: vec![
+                    EmailAddress {
+                        local: "someone".to_string(),
+                        domain: "example.com".to_string()
+                    },
+                    EmailAddress {
+                        local: "someoneelse".to_string(),
+                        domain: "example.com".to_string()
+                    }
+                ],
+                subject: Some("Hello World".to_string()),
+                body: Some("This is a test".to_string())
+            }),
+            reps
+        );
+    }
 }

src/webxdc.rs

@@ -37,7 +37,6 @@ use crate::mimefactory::wrapped_base64_encode;
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::param::Params;
-use crate::scheduler::InterruptInfo;
 use crate::tools::strip_rtlo_characters;
 use crate::tools::{create_smeared_timestamp, get_abs_path};
 
@@ -485,9 +484,7 @@ impl Context {
                  DO UPDATE SET last_serial=excluded.last_serial, descr=excluded.descr",
                 (instance.id, status_update_serial, status_update_serial, descr),
             ).await?;
-            self.scheduler
-                .interrupt_smtp(InterruptInfo::new(false))
-                .await;
+            self.scheduler.interrupt_smtp().await;
         }
         Ok(())
     }

test-data/golden/chat_test_msg_with_implicit_member_add

@@ -0,0 +1,8 @@
+Group#Chat#10: Group chat [3 member(s)] 
+--------------------------------------------------------------------------------
+Msg#10:  (Contact#Contact#11): I created a group [FRESH]
+Msg#11:  (Contact#Contact#11): Member Fiona (fiona@example.net) added by alice@example.org. [FRESH][INFO]
+Msg#12: Me (Contact#Contact#Self): You removed member Fiona (fiona@example.net). [INFO] o
+Msg#13: info (Contact#Contact#Info): Member Fiona (fiona@example.net) added. [NOTICED][INFO]
+Msg#14:  (Contact#Contact#11): Welcome, Fiona! [FRESH]
+--------------------------------------------------------------------------------

test-data/golden/chat_test_simultaneous_member_remove

@@ -0,0 +1,7 @@
+Group#Chat#10: Group chat [4 member(s)] 
+--------------------------------------------------------------------------------
+Msg#10:  (Contact#Contact#10): Hi! I created a group. [FRESH]
+Msg#11: Me (Contact#Contact#Self): You left the group. [INFO] o
+Msg#12: info (Contact#Contact#Info): Member Me (bob@example.net) added. [NOTICED][INFO]
+Msg#13:  (Contact#Contact#10): Member claire@example.net added by alice@example.org. [FRESH][INFO]
+--------------------------------------------------------------------------------

test-data/message/thunderbird_encrypted_signed.eml

@@ -0,0 +1,80 @@
+From - Thu, 02 Nov 2023 05:25:44 GMT
+X-Mozilla-Status: 0801
+X-Mozilla-Status2: 00000000
+Message-ID: <619b4cc5-1b76-2bfc-02ee-631b774bcb4a@example.org>
+Date: Thu, 2 Nov 2023 02:25:44 -0300
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101
+ Thunderbird/102.13.0
+Content-Language: en-US
+To: bob@example.net
+From: Alice <alice@example.org>
+X-Mozilla-Draft-Info: internal/draft; vcard=0; receipt=0; DSN=0; uuencode=0;
+ attachmentreminder=0; deliveryformat=0
+X-Identity-Key: id3
+Fcc: imap://alice%40example.org@in.example.org/Sent
+Subject: ...
+Content-Type: multipart/encrypted;
+ protocol="application/pgp-encrypted";
+ boundary="------------ODMSY6iRxlSk4uG4zIxjzLbw"
+
+This is an OpenPGP/MIME encrypted message (RFC 4880 and 3156)
+--------------ODMSY6iRxlSk4uG4zIxjzLbw
+Content-Type: application/pgp-encrypted
+Content-Description: PGP/MIME version identification
+
+Version: 1
+
+--------------ODMSY6iRxlSk4uG4zIxjzLbw
+Content-Type: application/octet-stream; name="encrypted.asc"
+Content-Description: OpenPGP encrypted message
+Content-Disposition: inline; filename="encrypted.asc"
+
+-----BEGIN PGP MESSAGE-----
+
+wcDMA/K57StIWPW6AQv/UTCUnHsea568d5xvOyeoaIgcwOw2F+XlP3TfHT1yzAvwk+DdIEWnEwOJ
+vwmkPBW9Yxb4bldD+I9pZFxUgQxUBzeFfl3JoP4RUiymWn1wSZ4dbH0d10+T8NzgHG35a0VY1vcy
+uw4RhfwnUZ9VJL2pW40dU40Ii34seSmdiJ82PbZuWVoAs3eCoIM3Gj+7wrDow4k1pTMHnNWfKRIG
+OLUkalQwRo3NGTFzre035Ktdc/cqt4S17EY2YXTOrJCnJ9JuabDUcHtGlu9Ir2/bbSQVryNCJN8b
+S1fnZo8yfIhaqxlatgJZy+i5+KNozb+U18lQeyx7FmBkxY53+pCcM1WqpCSnP7CNW0kgqV/oZtqj
+kOFzG0F/8vX0dwYv8UicM+iA56dwwzjEDjACmgM5+8g3yyKbO1N57HoQChNP6F+VPAHFlovmD+X7
+SNZugsXOIdmgB6RmWci2TCpzS1/C4Vjs/fwF3lRB9iSx7aRiXwRM4Eq0ORzqAeGLna5hvzev80C2
+wcBMA+PY3JvEjuMiAQf/ZoAOpxSMgOLIq6nOp1mqqo+t8U8Az6xAcF2+Aw275iRwAaXEiHrLKE2p
+cyex3ElVXigeUMgEbtetuJzhI/dr/AC+bUyz1Qqatj7TBKZWaAHsHBMfdpnGOSGQWLV8KDUsRMGH
+X5YIPqhtRt5oXLR2IpPHfTetDoqoydaCwL2okYTJUuQO0KbaAXPPWphSCochHHHbvh2zx9v+ttBQ
+XSYwQd+zOfcJvc+t/V9EWU2qDMLoI7qKkMCwgglRY7JOvxEVCIu71rCUgD5Z9ag1ZtHKw1Hiaa1l
+FsS4BbDjL+LVXHuaywR+/N8HjMamrbqfUrWf3cWe3vt0cXnYHh+28V1QstLFzQHRZ1H/KkVWIY/r
+jSBpOOPblPCTpjTp73y1t6MXXKMlwpSUzkzXQtLb9kpHOHGxM1Pu54Q5jRKsmZZSTASWpFGkPC0j
+td/+5qBtQV2S3KDn3nl6P3bzTzp6A+H9xJOBoZZ8jV97rZ85Rkc37RjIWkrNOJA8oDTVgSAOtY9U
+j2Qps4o8y4Bi79+I7tGxXI6LjCAq0LFvEylT0Zl5kbTxYKlqIQepS1PWt+96F57obka8KdvklZj2
+5z73lj8Hs5U5UE4ZSTVz3omQuarZtsI4zZ+FsZPac4iCJOTW2MWIBYvJgk/1fxvFnDk/xT3v2qLg
+Epdeqf9FXrcLaBA7qhpdF7oxHqid8F0Bj987/aamRLhVEuzBGE3rC5Lv2npIJziEjcKyNV/OzMs8
+M2aTxYBZ2EJZ0348SbVg1wzGsIgZFZ23gRUAgjvRuIWdMsS4T00yEKg8IKkzy0914lUNgos5tQ60
+FiMETsj25zGkiuM8KmC1YLHJJJDMHsAo+9a4uYYtjJde9BPrQfFNPIZcYgVxXqo37O/cKdmbjGap
+/pw8hE4prTGMeDAUq7s8V5eNVTBvSQSMarJ1XxDgVYT+xgOdSGFy2dXJmQTB4DiCIIuguu0DtiVJ
+vBSoy+cgHwxjlO0u6RGUkq02ipmRoVXs/d38LCZYztuYdsvwcBIhGiRsVkSQ7AwOFjWBxgleCakU
+jrFVAU3RUIyMjLZGpbXqZp75nu4xzdedIYElCH9cV4yx5TykdXwWfIc/mxvU7FfwIHbyW6qlcO6b
+qzauNfSUT1z7TuBtjjsFVrtTvGWPff66q7UeYF5RwFu39Du39XLpKi8tsl3o1w78R76HUnT35Tg0
+ti59CYeXPwJ/C5IsnwIMbAKHB0ekTx/XmYb4Uwl1wuQ/nNMq8z6XjioGpma+79RIQZ12nLwRcrYo
+3WsAYFnTmHwIBqICnJf1OcHW4V8ZhRb7KR2uVgksdEwFSaMY0cZkVrQC+OVIlHxLDqnfd2RaoKru
+qhewd+Fgd1a0k4ed4iGT+DlNJhCob/uBbOS9TdBWq7b1jFUCapeZtMxDQboIpFQwKweiBCMXrLWb
++cNls75x7sL7YClUTLhmRqxbfJJIFXnZyeiVQ6aTkY7L20llAKRk8TUp3+77f1vlxat8ZoSKcbzF
+SYofqKjwYVEYs63iF+cNtiD5EBH+R5Of8sye5ddii4xv07y0W8Np9C+fzFEJIgMzouOsuu4/G3C4
+T9fXchqluT93AvPlrTXEncPZQM6CBNnrUUbUlK9z6Wtyqk4LfMvnHtuJ/z9DBKZa64EtBdK+d6r1
+B+Cl3xiG5h0GwLNYcwVU8rKU9dz1VTfD2nuE20YmI+Jo2Acza0+t4I+qUzvACUMX3kVUt+Z2hQej
+VWEUmAMBGIL1951o72ZTXprvEq+W9bm0IARFg/u1Hc2WnRWrHWAIGux13bZeGXggGXFELrzLhx7H
+eb1jH0bPjpIiiTs0sc/BwEkxTYLlLW9LA0+CSThwdW4wtvaI/XYd1drcMgO5pncK3jqf4dH/DYOJ
+v71xmdNVCjd0wSKr9pTGEvlRJNfmTu/COvogpGRFuVZ4kONSWXQfBO48tjlEUUICKpbDGM1hmtfA
+Qt5DuJ40WTcq/Tqd5hQ9Xo3jO9ZGczOKvi9lG5dDnj0O8cbFGA0/8C8z1XFRyNARmm2f9km5m2oh
+EKnyZYcRSHpJzSH0ULAxhpMZl68RTgdU02JVdoXHt1jgbmb4hxX4qVR/kKYcgKHp4eVySh6hwhIt
+bKOhTtIbYOxus9o9QBsvLPOFToJ8NwRaNeA18Wg4cbvAxYe3W/qJ7L6T9atrVB8nVFC8cRNUTMZq
+niH9kpjQ7Haal966YNPhBKcTnplLSHH6kakYDNNGGxQpqyxFTAbJfDp/fhmJqvbYYfcP0syabPCL
+y6m2yxVhF9qKwb7qRz5ZM5weXNWhUC0YGCTfbsugNN7n2g8zNxZyGtAs1aWMxztjxkI8zaufoa04
+3aVPUyUcjb71UTqcqO0fIdvTQnsNldbbFxKWxcJAFf2edor4UXbrnhyA+7XWg44Cy4+ErrTxT2qu
+N+bWBAyj9dUpVTicNgXtoJwCkkWoBlFqGngc8Ho9gkOvKUDlQMZOB/74iBBrcmFc+N9TrV/GhUi+
+brxwcZA19xqSm3fU3H9TufOr6ccsthwmkXE8BEQV22P/Z94xw46SDItiYMM+PW93nuo2nvvq5cwx
+d+EkgKXHfQVfohZhSg==
+=u1yY
+-----END PGP MESSAGE-----
+
+--------------ODMSY6iRxlSk4uG4zIxjzLbw--

test-data/message/thunderbird_encrypted_signed_with_pubkey.eml

@@ -0,0 +1,165 @@
+From - Thu, 02 Nov 2023 05:20:27 GMT
+X-Mozilla-Status: 0801
+X-Mozilla-Status2: 00000000
+Message-ID: <956fad6d-206e-67af-2443-3ea5819418ff@example.org>
+Date: Thu, 2 Nov 2023 02:20:27 -0300
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101
+ Thunderbird/102.13.0
+Content-Language: en-US
+To: bob@example.net
+From: Alice <alice@example.org>
+Autocrypt: addr=alice@example.org; keydata=
+ xsDNBGOaGzQBDADCFtBNMHRDJQRkd2tNm7CJm1Yo3Y5r3qP6v0FSwP1BIHbiIf0E/jFiKZWj
+ 1uL68J2mGUuUu+Qi4ovf1l9/QQYzg/DCaLZxlbc0LKu2LXcpUL5DPu37mdw+DKs0YvNIlc+A
+ RjyFUwd3rsZN3k58inf1mYzKuKU6NpbdXULbOEYwnVEwzQsrtS2JgJ+tLSYUvNJeMJXm/cDL
+ XKJSApAyvVVdxxteG8uWcDqWV/HcXuopXLILf3yJF0De11/7G62dHNHuhmtgRLsTN4Q372Q9
+ KNdYEFLHaN91jEzyD/+aHNskATxtcGhppI8OQsU3NzNgHyd8Smzx5oTyZ/6NdhYoh0pKB8yf
+ VAyA69t5fctQRb4+bTwL+sS9KDobQOvcyOMUSccDfUhsWMghwsMCwU4Sz9hIY6dCAfroDAiL
+ vYUfdNJstAqvLf04mZtMmkI7Q2BYLETEgu4KQzQHRQekmOE/3EaSiojNa4ZTVURMdJ9U+I3E
+ q8e6TbOY7Xa4V8krAt/F2wMAEQEAAc0ZQWxpY2UgPGFsaWNlQGV4YW1wbGUub3JnPsLBDQQT
+ AQgANxYhBBSrP2X8J0u9tfp2jCXwByRZ5HriBQJjmhs1BQm7+B4AAhsDBAsJCAcFFQgJCgsF
+ FgIDAQAACgkQJfAHJFnkeuKQ2wwAgDgiCI6bz9PjqE1GoDcy/xQdy+nnYq5pOuHGUndZ7jYK
+ cOqM8LDEaG7GgrFsbs9vGhTA1fyqncM41pB7SmwQ7zBVaMdtHoulEG4RPGVboDaY9tuMOL3/
+ GVxFbovVHyU5Lr1euryNh/0JvMITY0UHaEY1k1M7izYUMyFu8I1ODZ9Iws2trUyU3Omw/sTJ
+ x15zzCsK8Aq+r3JmB+Q33SFSgWr/YWH0dQVIQ0I5iLN2q14oucmLBaKc9EXdRLiu8S8lLSQl
+ nfISJ17GBLmH1YxmPPZ3CRHC6iEKCLR6G9wzhsTPNdK7dRCYR5wTI27RVPLBcSnCKAeTopAJ
+ YskyNndtv0iaNRT7YLOfhrsBAofSjuLegR04CNiqBHtYQ3LO3WKhJ7riRcQ/Ksv0wYkmj1gJ
+ 8myMwA+ybfYrpNqO4devnCvE3Eo5gzeYbvYU2Z17n9y6HAOG9/Tm/daiGEP2ni6iwV0kqLzw
+ eC48R1D75T66PxX/jQooujrTph8+K3ckV/q+zsDNBGOaGzUBDADV+DGgKxvCpfVFuPGrSdRU
+ 06dxowdKOKavO6WGMvN3g/+CFrIsjUFy4S0Soo5ARnLh23i49ZSjacXFpgtZUNV3iGOSOcSE
+ LldLtZk5BV9w/ATqqgu4/LVdNA9rm+o197bIeSQCRTnY/QV6FdKYxVd4NBVH9abZ7t8Tm4qC
+ urZj56MjPCg3fqT+Q6sjxH+nKBrs8s8iCJkYhGBgU3q5W+wrtZ56kI9mxJec62KHpyLZ0rTE
+ xEAeVbChUJOo11vUtJfTrDhI6lhqyr72o/A6bY1OV7WzkxtiBRl35eewQ+RDLJ4yxaNj/XTS
+ UxOz60xNggEfDVtfgfjBZrBbiHXqf8iKVV1ZPGm0ycvXZGYFw2zXLI2PwevhQCm+t4Ywty1h
+ 8l019MYmGadpQgbuA4ZippuzOSzSGMQ+S4uYEzeeymR9ksxVSXn90HEzqC7LdHCcd2IO6rfu
+ g2fuRf258Adfuoh3s8YUlWyXjEHLXKo9SRgGMfGs7qgCOL/ReAwFPtKACvEAEQEAAcLA/AQY
+ AQgAJhYhBBSrP2X8J0u9tfp2jCXwByRZ5HriBQJjmhs2BQm7+B4AAhsMAAoJECXwByRZ5Hri
+ EOkMALtq4DVYX8RfoPdU0Dt6y+yDj1NALv5GefvHbgfuaVT8PaOP0gxCjWrnUDvvJEwP1W3j
+ UXYqDwKP42hiGWsnXk2hbgXbplArgP3H987x7c8bu1wIAmkJ9eLjEM++rbOD4vWbYXRwaDiH
+ LetFJ5tGHDAIfL48NYpz2o3XZ3/O7WdTZphsAcvgPxTC+zU7WkbUl2SQlj0/qwsoD+qe9RYT
+ XhVXR7q7sjcGB4TpeqzRT7YKVLoVNq+bQw2lUX4W561gAYbZvVo/XByfDCoxmkxwuMlSmajj
+ Wy7b9TuT38t1HArv4m/LyVuBHiikX0/MUNBeSSIiKDvTL6NdHTjnZM6ptZvdvW3+ou6ET0pK
+ MGDpk/1NVuMnIHJESRg/SSFV6sElgq38k9wAT2oUqLcYvYI07nHmnuciaGygkCcGt+l2PvAa
+ j4mkQQvMU0cNRDBybk5aKi820oGIJjT7e+5RnD2mYZQdOAbQhDVCHvrfS1I60bsHT1MHqyAa
+ /qMLjKwBpKEd/w==
+X-Mozilla-Draft-Info: internal/draft; vcard=0; receipt=0; DSN=0; uuencode=0;
+ attachmentreminder=0; deliveryformat=0
+X-Identity-Key: id3
+Fcc: imap://alice%40example.org@in.example.org/Sent
+Subject: ...
+Content-Type: multipart/encrypted;
+ protocol="application/pgp-encrypted";
+ boundary="------------vm4cgC4UBxQhCLFg6Vv0nrTd"
+
+This is an OpenPGP/MIME encrypted message (RFC 4880 and 3156)
+--------------vm4cgC4UBxQhCLFg6Vv0nrTd
+Content-Type: application/pgp-encrypted
+Content-Description: PGP/MIME version identification
+
+Version: 1
+
+--------------vm4cgC4UBxQhCLFg6Vv0nrTd
+Content-Type: application/octet-stream; name="encrypted.asc"
+Content-Description: OpenPGP encrypted message
+Content-Disposition: inline; filename="encrypted.asc"
+
+-----BEGIN PGP MESSAGE-----
+
+wcDMA/K57StIWPW6AQv/aDJaj22uqA0ttQ2xHqnZrkh6NLwo2DNil1MOWEw2WM2e18rKKyXtSoGB
+3l0zMMNqvlusKPIPv5aLNN8j9XB1j+/7jfg1aiHJ0l/Zo2tGgbaE0xi/V7jiP7An+oNBYHKEspv1
+gyHPT2sSdLOWq+GsJk3l0nPAw2RB6WY4uKE5uv1CQPRTPw61CzuyTrjj9o0fPPn5BmFXtAR6aRaC
+p7kXy0YEJgJ7I/OZHNP3Z6tQ8ael9lcT7W9StMfmkINooVw5U1eF/y+JqSlkNLw0qm3zweouSVvH
+G3rZYyf3wWC0/01CaZkbhk6z2CqopVZ9hiZPzm+PFEUEq4x9vbVg9pONEIjyFEmqdUZqwYZYAw5o
+Pv12Ow/kF9zY0YQA2R2KXtXitptt58lDexYo2kJPGqw2PwKGTPl2n3Bweyj+2Xdpd1jzH3GjfEf5
+dfzfJKIYF2+tI7SYvcaJhPBOuKPT0EhHN4Yd2k8Q9OrdNkAdazgiy+LpcI5JRwK+ltZjLpK/IHfQ
+wcBMA+PY3JvEjuMiAQf+LlAeycu64WiodkQOqsNbrdlp/4C/t32P4/pCKdrJXP1jBOQ3ndTBb5O+
++uMpmSeNpkJeqc/g7PaxDUlCCEOKVjuS1cHvKa7wnO3+BlIVbd0/zF2kYKppGRrv4yRhL8Sk+K9q
+Wf3Lr0egCH1/ChvSHIZVetlpDCsWI7397DLs+uIcXa2Btlrmj9O+0YZJ6L6ASqs6zpkj9VuCzWFU
+ngKNVZsjsd2IWdBXA1l7Cpsba+JqxrgyDrT7evZOmwrLJdTS/vz8MGCr3sh3O/nOnZzE497qx07e
+ARQXCblLED5AGdBBWgPA1LwTDnSzjQaCvUsCLf15Ei6AHLm4BTdqrSmu59LRRAEz/KPXzWOamjUH
+ASemp+dQMIfE5nEgyM04kfejocX1NFduhKCVASLx236TcI59Q8rI2ikSfxIZHog/7Rod2lGoWYwb
+DnL1fBNXXxZt7hQ1xtmmcuTWb1CDS+OXGk7qPqC+cEoxmnmRX2QnGp+1Qd9PCCwAojM9jOmyJ/zL
+iq0yXB5JBivp/wQUsQI3EJtUGbp//WzevyMLr7RGH7z3Yd07qkBOH9oGKXgdOECVFXl/1RvsWUe7
+/A5EuPqTqSukidP/JgPSON/IMYsgAoOoJHMU2vEBHUi3Q/gARZE0K/PGD+8F2LBxbXXcOX2Dcj4o
+EFyEi0StxD+96dhT16oEG+y7svGVSHuH2XR7EW9l5A33+EjDL/i3XCyUoVMtC8B201D4Gz44wXb7
+FBzjH0XrKChFM+5meXNeaCicyHQ/Yn42asTgLCxyKAUqNmTehVUCrQk5h1LOHLzcY9vUehBYt36j
+kjuxbMtztIcoExyPOkckmD4a/+FqknkK6tqPnR6WBMS/vfKRFRi8RxbtR136fVU+fBb9hN+veqRx
+Fc5uUmbHAaNzeZG8RukNS3iesiHokqWAtoM30QnLJugKJqL4kEkfFr+ZXzIYqbcuLRvNJoyDUjeb
+ATkiWna128bnolnBul/Zp89ueSUiMjUfDgVvlw/W8RFcnizxkifg4UrBiW9uKpqznR+OXK0Zref2
+xZ3j/BxUpVtZtz7H7DQvmk4zFRFC1P/EF7mde0swKs0ccBDykT6tdOU7bDVbzgYOfHoMhsMwiIxl
+HpQERGm64dZ/zc/YKbk2csdDsWrcJ4k9fwo0QpTTFz3YYUoRnHkmDWfR4jcWkp6WXrXicWH0ZUse
+pgb4Pn7jEtM0og++iAqOHa1f+n/qy7wpLFCnIi2vylJTzVeHAzs5chX3bobUGZ9lfRLdbUL+n9LO
+7Uuc6AQ0KflkapDX1B2bLf/sSeUIb/ZJdHVXBNIcGsQqMt8iCDMgak5PnKuumM33NAQOCmpIxn/M
+CNxmPl5kIL7jgj0LOIcPr8Od9PIFpgeoZNKlwjZvMDPM2KqD7m89NgYYnJlJIVfdylwp1kQeFPhR
+4pg4f+LA8zE2BunzlUtmWE66zAKCd4bhoHTKOPzDceKY8up0UhdeYOz8mL8ls77pJemSTzAsi5mj
+17Pon9Gb3OBnj+s/H08aaAnbg9b5Waf9lcO6h8IMznGumxYp2r2eE71pjkG39z96bgANmwSqtmAR
+sO2wXKv6tCH7dYpTwKNeAKTBaiu6Fs9YDMZheBI5V4oft2VJm/NQ4oDGHtSvsl1b9cwlUWe/cg0E
+9oGwc0jcSsYQTPps5aRj73NFodTxEBNJuy+eEaVxmMG1xT2JtxYB0dY3JIxQc8+krp41ItFcmqX1
+l4vFYNhslfpVegZWO7KPTJBI5W6dp4fE6FjbfzqZ39Kntp7cr65a7GCUxb5GrgWiT3lFAIN8pYk5
+t0tnv3YbodIS5vxnU5Al9rU6cEZCeOoqmWbNLDbjgxReyAMimuDRyGI3KppwD5q0nFfgwR7E9y3N
+8wYz+lgHceJQCYiQuOmDDGGkpud1MCW6JgtQBfnke14gJGpCH23UGS2gXyh5q6uhh2/y1j2WKJCs
+yNxecIWFXDopSFgs44Q3jMonjWnt42PWrTUkUB1ZdjK6dTE5HUwp/8sQ92yYsPIPfOP/3zUm+Rr6
+d7Akkw6l6fz96EmUf5/c5vyEmyTWSI/MKhUNu2dPgkJVZZy+85oEi9jW9T+mVlU4oIV5yDZoYn3T
+2zqjKKyhskJiYKCsplg9/DmynGNB6zHWxuKJzmDn3i65EEcl1+RiVEHZAij5zYNEKWe5mRFsLY4s
+xJlVFV3aNE4iwk/Tu5cCGlct7bbB7l8DxI7WS91fuW3bmSSD9P77PTeU1bxM9yeWYdqmA44v93lD
+uKxBQi/grUTxsM0Ukttnn5ovJtu52IKlsGfDNssE/2wbXBY9aG8fHEa5QFrERCCOoH+rGl2PIYFg
+lit8v5o+z/kmjXXI1iJdhM4OUY687yJHCckLh5VMgMqVG6BYb7CFRQojlKTvrMWOj+Sm1GxFZa9r
+BR3vvne59UzhGY9oMczU69AzbwcUsnWMM9wlwZ/xHnDtjoIr0Yu7lRM5WWtKQMJSqS943PhUHMmp
+5uJE10ZBYOtvK0tC976A1RMu54bkGSiyd888Jcw3IKzBxrTr77a+kV5QT6fZr1duvP4D6hvvmr8w
+8N1IRFrwM2/pVUYEPoP9fdDOGsHi5w1UCt0DTqgLAHgWR9tAKY33/o8jHE8Le9iRU+0qRFhuzTCi
+eaiJ4S+HFieNAco4PxWhYginoq0b6KY+0/GJCQTFnb1A2mYWkmOfBP9a8FFDsCNQSS30fXVr+K4b
+wFRtoWc0Kq5xQuiVn1Y4mtPzdVjKIt38cPJPKZ5O5yUgaXQLROXsJK+KQsm9MIeb9Qw6+7Nr5sTC
+eG6ralQWGM5si8kCTsO3P9SyMVUFga8IU7IqUXlEG7luXB6AMfYjAlzCJdikOMZWySirJZURaa98
+DoR62zNVs2DI6liuAjrqq+5/waIdrQ4nV41nOr9SmAy1anIOwIR6ieoo1po5mD2gOVPRGHFXYY97
+Ksbj9gbTzdNxnOJJWynBsH1I2H72JheGHaBGXHPx/3cCu6O5pFG5xBX9dDdSD3rxq4UK6O41o5qe
+Qh61cXjW4xxqxN9XEfz5h7ysllmcBIxW7ElC8zj44DgY/sdd3rp6qqRbzxXST62gOk+g5yic6uO0
+9B1tg/RUZzNKUZVGUAIqTalVHh5jYk8fQ/oyM4Ey8zvH5voBHq5lULZp7JMyymqYEQEvE+dKZfS1
+YdRL+JTTjL87tpK4FnCCtV+SGM/gmU5jiyf5Eb0FNSofHyLhJAbnNeEC2v+9/bgjs+jtOU/YbqFP
+Qv6zLTsdo7hqeJLJwkLjFuFHIkcnpPu1dzRk7sjvlZ6VjQGqRe5lEl7z176XaHoJnaQRIOICKxWV
+xpxzdPbxhmFvYpN+n0MBfjY9/dKX4MB1SKLa13P9DfvSMQFkiwTjag8n2r+iWaMQGH9U6+ni4uLt
+0qRWfZXbqjOgLsKjMmoTl5UAzLR+mkfgFspkYRuAlYBFubtzIsO8x9ZEjzdTXQ4h4Nilps19DFnA
+kFex6FEulzP3f+tS7lBygFYu2S20TYppNpdSPiqrVJC+QDCy6Jxhr25skSBHmXtWSe85xMsq1uvy
+U6pUUU9tkBasZcf59mDdGpdetBKngk1BYTP7iQSUa+Nr99t+Rk6o02tCdjE22DPKyMJIVS/QmbVN
+eKSVMtdt5O5KfardfMecjcXoqjePt8ICjLBXMKDx2AKbg7wpBpjiFg23UuMDUjerDoyqX+lZMkQr
+PG6tR+gezGzYAivtIPshFgIgURmfB1UtJjc7CST7R4n9cey+399zy30OhdzZT2pndYPGAEpxPU1h
+T2TIgZPsRytDf6zK1RQiAecl6peK949JFW/0XlxkVDXMCdoSV3Vi+r6p4Q5VI6UM3B+PqTm3yz4r
+8Njj0nf/lbB3hEoWrK84376yfyXRkuqz0m6SPU+uE+dxJH1DAVxJugVDyQdmF5h2lkoAF3iWeUTi
+h4tfZX06IitNMuIbvPY010vL1ZXjgPjhFdsZLG5GFWXEC1ZmCY1gpiVPIIxdupMv1W8Z8ZNxh5Rk
+97ZpjUdWZGWoaHZ0b357hjnzEgdnUSEI74LuYIAA4yEkPydyeQtNhuTiokbz+QhHNv/C8+0XRrIE
+ZrQIbDVGREb+Qm64qTHWb5aW4elk6hXpVD8F119VIkigXFYsRNTU3WaOQrWstq/qNr295WCQ96Um
+9yOyIbALt9Mb1liQaMDGLoTd5WKVmriSR1/a/GrAoiJ+NP0Wg3+HGemv+s4YbqMEOb/7OeHGP5SC
+zfSzU363KLqQ1nTwUoA845fjtz4TDw02oF9nMKhsV9lkmX250By84l61rnCIgmUcyEDJgQNYD7kQ
+Mbf+g7pm9OexPX2TkzUqzckOW0y3EJg1C9K9kkrINmKlRxenIf9LNmssOv6jUd61YaE1otvQcvaq
+mJk6pMKK9f5s6Zbe94Mn4ZLoE9JvDus593xZkDEpWhsuYBfammNpgCWVBdecyc0aP4UNz8SmJnKC
+nw5ipIumjnCjB203l0XivGjpGZDR69gmyL6oUCns5IgqgHu20MBy1RGJ/DPyeROUMVGxvPAgGVfn
+9FuEovA1r6iVLWhUYnXGeF9LDOdO9cOOXeFFjVrznF+6PAClR9z3QExAvWogKMK8bhvtKvz6JV6s
+c8hMCWK8M9MoVty4XeqOH2xKenUcRrao9/t+Zoa/nhqn6cF7LGGuLaThY0xWUpPzZdjszIG0iJU7
+jBslXNQfAei+spBdKFPQNr/SLvG06hzpKqj6iOdY9HCJBdiQ7BvG0/bNOMiZHAcHSukvXO0HyXXN
+ozC3hYGqObm8mDAtF+4Hl+1vFS2ckNO9RzhbDHMTeRg3FKBzyyazMI7iu/5gMqojhykb2qFoV1DN
+Aq5ToHsyP7l8y8u2Bp8WkRUbIbIxnyquqQjH7+7zDrdXHa5vE/8rgr3gmWvZdMdJAPpWlwc09Zdb
+FppO2cuuarGyyf8Ko1JV5uy2FrZsOKcKGK3Fn2JtEvbgPcnXhmfk9mUF5eiYrXxyqzKPrxw4aG+J
+bRjIC/8qG/ozswn1YuVUxAWGSewfRZCGsPT3GThG7lthpcZIVqwCojoYtfWELo/tRIl+wMXFHvD6
+FRNig1sYHVEYOqM2UArfo6hCedgIPXIwgfb5q0B9XMn3RSA2bCAaADBzFWqLd1P/GiYxcE7HzjBw
+OFCquuUrAeY3BQXgynJQhCMmL95rFujn0qzcBT6pxgPvJ8zULETe0ct8BClHJmzjwYNdr6fxYTre
+9lT4ZMA8KS7UGHoYfXcRSWrBfKIoOwD2DsFV77mWvUDAKsPr8kIS1uokk7x7VLGCNPkdPUONkhTw
+medxzvA8W5QSKDU1jIPLXIItJ4muQOY2ewyM3EeRREnUs+tCUi+tQ0o5mvwPnfULOp5mtP0cq3F2
+/SBG7hQiuexlhpJ33elflLZ81VdDkB0cMeZwh5oWXeqBkFIdmc/2O0aQyy/EN+U4f/tj/DBk1wUC
+OeAeM3cEXgXg0XVBNc84qR/csAwyLOkeV2RgwzUBchbrPdw+gsku9mi/g0bDN81G96zjKfUXZT67
+FHTESXYfrD/Tvpm4HSd1STKEUD7GjW8AoF+8MUJU/ZXUO9KVsHvv0ygs3KxcTBj2FeDv0cnVgpaR
+Jwgb6Ss2zmAr8REOjCmHtLXJB3OGqukgBDiEpMZzZEDoPh5BpOkd1AeKxyhKPMdDxVq+pGC22bh3
+S01PCLYabwU2mDlbM3NnkA0ZBK7TnEAE53C5+9DpAfym5iWLWDrR57fNeMrUYpufti26uAbkYesb
++Hr2VZGaIKLwZVIgSESmywNVonUGSqWBhbHUQJXc9ndUq8BlvKXDDE4d0bCrp4WgmfA+IR99V5B/
+EVFoTc+xNgsLEGJLbFLXjZteH3DwJM5lbghikznzhIu32CgejCwYXjV3Kgpc8+VyzPQWYe5xqXsS
+iOFYdNOuMa8rKC7tjVG5l/072+NwFS5Lgi3OGsDkKUVjM6Lw8jAQQ3Thy9Kf3CoiN1zr49K6do8x
+mfB3k0qucBhe52O60x/G04P+hcRza4rHFLRzrqPKD8QhULPC8Wd9VU1kbj2WfZi/PBRCx9zQcB5s
+KBFXNMJokoFTggTpOiga/zOa21PTYXuUi4rvl4qZX5zWAvapKXWOixsNmJxwv4winMl2bSeCj+gb
+w0Zoo4d4LrjuVqRJGbjTb0V214kLNayN1E8vsnGyzUBZAzzpXS+c/jawW5BPsFuEO1aEhH335aMk
+R9kEglm7f3YSpDk9+7TMf8LkExU2Reo4wHfc8TTK6UB40YS2zTzCCLMwA8PC6CXmwaLGhisvNPgS
+NICOCezsYyVbneaj+Oh877wazzsQF8UHJhvq6+GPYAAG25IVaYer/Ehi8eDftzrzdXsgE/4+Cx/k
+D7raLsceHzqlPSwRxzFEZctpvvN1J3WA5/3yvTF2ZlliI/tFso7Kh9lTJVu2d3yhHqnfDG+OKU6f
+xvNU/DotCyfOkhNwEmbCuoCxCU8eRjumP9xkUeQswMPDKHUCwZFhP98=
+=UKyr
+-----END PGP MESSAGE-----
+
+--------------vm4cgC4UBxQhCLFg6Vv0nrTd--