chore(release): prepare for 2.27.0

Add the ability to withdraw broadcast invite codes

After merging:
- [x] Create issues in iOS, Desktop and UT repositories

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ permissions: {}
 
 env:
   RUSTFLAGS: -Dwarnings
-  RUST_VERSION: 1.90.0
+  RUST_VERSION: 1.91.0
 
   # Minimum Supported Rust Version
   MSRV: 1.85.0
@@ -168,7 +168,7 @@ jobs:
         run: cargo build -p deltachat_ffi
 
       - name: Upload C library
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug/libdeltachat.a
@@ -193,7 +193,7 @@ jobs:
         run: cargo build -p deltachat-rpc-server
 
       - name: Upload deltachat-rpc-server
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: ${{ matrix.os == 'windows-latest' && 'target/debug/deltachat-rpc-server.exe' || 'target/debug/deltachat-rpc-server' }}
@@ -252,7 +252,7 @@ jobs:
           persist-credentials: false
 
       - name: Download libdeltachat.a
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug
@@ -313,7 +313,7 @@ jobs:
         run: pip install tox
 
       - name: Download deltachat-rpc-server
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: target/debug

.github/workflows/deltachat-rpc-server.yml

@@ -34,13 +34,13 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
 
       - name: Build deltachat-rpc-server binaries
         run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-linux
 
       - name: Upload binary
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}-linux
           path: result/bin/deltachat-rpc-server
@@ -58,13 +58,13 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
 
       - name: Build deltachat-rpc-server binaries
         run: nix build .#deltachat-rpc-server-${{ matrix.arch }}
 
       - name: Upload binary
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}
           path: result/bin/deltachat-rpc-server.exe
@@ -91,7 +91,7 @@ jobs:
         run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.arch }}-apple-darwin --features vendored
 
       - name: Upload binary
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}-macos
           path: target/${{ matrix.arch }}-apple-darwin/release/deltachat-rpc-server
@@ -109,13 +109,13 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
 
       - name: Build deltachat-rpc-server binaries
         run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-android
 
       - name: Upload binary
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}-android
           path: result/bin/deltachat-rpc-server
@@ -136,70 +136,70 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
 
       - name: Download Linux aarch64 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-aarch64-linux
           path: deltachat-rpc-server-aarch64-linux.d
 
       - name: Download Linux armv7l binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-armv7l-linux
           path: deltachat-rpc-server-armv7l-linux.d
 
       - name: Download Linux armv6l binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-armv6l-linux
           path: deltachat-rpc-server-armv6l-linux.d
 
       - name: Download Linux i686 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-i686-linux
           path: deltachat-rpc-server-i686-linux.d
 
       - name: Download Linux x86_64 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-x86_64-linux
           path: deltachat-rpc-server-x86_64-linux.d
 
       - name: Download Win32 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-win32
           path: deltachat-rpc-server-win32.d
 
       - name: Download Win64 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-win64
           path: deltachat-rpc-server-win64.d
 
       - name: Download macOS binary for x86_64
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-x86_64-macos
           path: deltachat-rpc-server-x86_64-macos.d
 
       - name: Download macOS binary for aarch64
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-aarch64-macos
           path: deltachat-rpc-server-aarch64-macos.d
 
       - name: Download Android binary for arm64-v8a
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-arm64-v8a-android
           path: deltachat-rpc-server-arm64-v8a-android.d
 
       - name: Download Android binary for armeabi-v7a
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-armeabi-v7a-android
           path: deltachat-rpc-server-armeabi-v7a-android.d
@@ -294,67 +294,67 @@ jobs:
           python-version: "3.11"
 
       - name: Download Linux aarch64 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-aarch64-linux
           path: deltachat-rpc-server-aarch64-linux.d
 
       - name: Download Linux armv7l binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-armv7l-linux
           path: deltachat-rpc-server-armv7l-linux.d
 
       - name: Download Linux armv6l binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-armv6l-linux
           path: deltachat-rpc-server-armv6l-linux.d
 
       - name: Download Linux i686 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-i686-linux
           path: deltachat-rpc-server-i686-linux.d
 
       - name: Download Linux x86_64 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-x86_64-linux
           path: deltachat-rpc-server-x86_64-linux.d
 
       - name: Download Win32 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-win32
           path: deltachat-rpc-server-win32.d
 
       - name: Download Win64 binary
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-win64
           path: deltachat-rpc-server-win64.d
 
       - name: Download macOS binary for x86_64
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-x86_64-macos
           path: deltachat-rpc-server-x86_64-macos.d
 
       - name: Download macOS binary for aarch64
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-aarch64-macos
           path: deltachat-rpc-server-aarch64-macos.d
 
       - name: Download Android binary for arm64-v8a
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-arm64-v8a-android
           path: deltachat-rpc-server-arm64-v8a-android.d
 
       - name: Download Android binary for armeabi-v7a
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: deltachat-rpc-server-armeabi-v7a-android
           path: deltachat-rpc-server-armeabi-v7a-android.d
@@ -384,7 +384,7 @@ jobs:
           ls -lah
 
       - name: Upload to artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: deltachat-rpc-server-npm-package
           path: deltachat-rpc-server/npm-package/*.tgz
@@ -401,7 +401,7 @@ jobs:
             deltachat-rpc-server/npm-package/*.tgz
 
       # Configure Node.js for publishing.
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v6
         with:
           node-version: 20
           registry-url: "https://registry.npmjs.org"

.github/workflows/jsonrpc-client-npm-package.yml

@@ -19,7 +19,7 @@ jobs:
           show-progress: false
           persist-credentials: false
 
-      - uses: actions/setup-node@v5
+      - uses: actions/setup-node@v6
         with:
           node-version: 20
           registry-url: "https://registry.npmjs.org"

.github/workflows/jsonrpc.yml

@@ -21,7 +21,7 @@ jobs:
           show-progress: false
           persist-credentials: false
       - name: Use Node.js 18.x
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v6
         with:
           node-version: 18.x
       - name: Add Rust cache

.github/workflows/nix.yml

@@ -25,7 +25,7 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
       - run: nix fmt flake.nix -- --check
 
   build:
@@ -84,7 +84,7 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
       - run: nix build .#${{ matrix.installable }}
 
   build-macos:
@@ -105,5 +105,5 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
       - run: nix build .#${{ matrix.installable }}

.github/workflows/publish-deltachat-rpc-client-pypi.yml

@@ -23,7 +23,7 @@ jobs:
         working-directory: deltachat-rpc-client
         run: python3 -m build
       - name: Store the distribution packages
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: python-package-distributions
           path: deltachat-rpc-client/dist/
@@ -42,7 +42,7 @@ jobs:
 
     steps:
       - name: Download all the dists
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: python-package-distributions
           path: dist/

.github/workflows/repl.yml

@@ -18,11 +18,11 @@ jobs:
         with:
           show-progress: false
           persist-credentials: false
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
       - name: Build
         run: nix build .#deltachat-repl-win64
       - name: Upload binary
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: repl.exe
           path: "result/bin/deltachat-repl.exe"

.github/workflows/upload-docs.yml

@@ -36,7 +36,7 @@ jobs:
           show-progress: false
           persist-credentials: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
       - name: Build Python documentation
         run: nix build .#python-docs
       - name: Upload to py.delta.chat
@@ -55,7 +55,7 @@ jobs:
           show-progress: false
           persist-credentials: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - uses: cachix/install-nix-action@7ab6e7fd29da88e74b1e314a4ae9ac6b5cda3801 # v31
+      - uses: cachix/install-nix-action@fd24c48048070c1be9acd18c9d369a83f0fe94d7 # v31
       - name: Build C documentation
         run: nix build .#docs
       - name: Upload to c.delta.chat
@@ -78,7 +78,7 @@ jobs:
           persist-credentials: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - name: Use Node.js
-        uses: actions/setup-node@v5
+        uses: actions/setup-node@v6
         with:
           node-version: '18'
       - name: npm install

.github/workflows/zizmor-scan.yml

@@ -19,7 +19,7 @@ jobs:
           persist-credentials: false
 
       - name: Install the latest version of uv
-        uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d
+        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41
 
       - name: Run zizmor
         run: uvx zizmor --format sarif . > results.sarif

CHANGELOG.md

@@ -1,5 +1,217 @@
 # Changelog
 
+## [2.27.0] - 2025-11-16
+
+### API-Changes
+
+- Add APIs to stop background fetch.
+- [**breaking**]: rename JSON-RPC method accounts_background_fetch() into background_fetch()
+- rpc-client: Add APIs for background fetch.
+- rpc-client: Add Account.wait_for_msg().
+- Deprecate deletion timer string for '1 Minute'.
+
+### Features / Changes
+
+- Implement RFC 9788 (Header Protection for Cryptographically Protected Email) ([#7130](https://github.com/chatmail/core/pull/7130)).
+- Tweak initial info-message for unencrypted chats ([#7427](https://github.com/chatmail/core/pull/7427)).
+- Add Contact::get_or_gen_color. Use it in CFFI and JSON-RPC to avoid gray self-color ([#7374](https://github.com/chatmail/core/pull/7374)).
+- [**breaking**] Withdraw broadcast invites. Add Qr::WithdrawJoinBroadcast and Qr::ReviveJoinBroadcast QR code types. ([#7439](https://github.com/chatmail/core/pull/7439)).
+
+### Fixes
+
+- Set `get_max_smtp_rcpt_to` for chatmail to the actual limit of 1000 instead of unlimited. ([#7432](https://github.com/chatmail/core/pull/7432)).
+- Always set bcc_self on backup import/export.
+- Escape connectivity HTML.
+- Send webm as file, it is not supported by all UI.
+
+### Build system
+
+- nix: Exclude CONTRIBUTING.md from the source files.
+
+### Refactor
+
+- Use wait_for_incoming_msg() in more tests.
+
+### Tests
+
+- Fix flaky test_send_receive_locations.
+- Port folder-related CFFI tests to JSON-RPC.
+- HP-Outer headers are added to messages with standard Header Protection ([#7130](https://github.com/chatmail/core/pull/7130)).
+- rpc-client: Test_qr_securejoin_broadcast: Wait for incoming message before getting chatlist ([#7442](https://github.com/chatmail/core/pull/7442)).
+- Add pytest fixture for account manager.
+- Test background_fetch() and stop_background_fetch().
+
+## [2.26.0] - 2025-11-11
+
+### API-Changes
+
+- [**breaking**] JSON-RPC: `chat_type` now contains a variant of a string enum/union. Affected places: `FullChat.chat_type`, `BasicChat.chat_type`, `ChatListItemFetchResult::ChatListItem.chat_type`, `Event:: SecurejoinInviterProgress.chat_type` and `MessageSearchResult.chat_type` ([#7285](https://github.com/chatmail/core/pull/7285))
+
+### Features / Changes
+
+- Error toast for "Not creating securejoin QR for old broadcast".
+
+### Fixes
+
+- `is_encrypted()` should be true for Saved Messages chat so messages there are editable.
+- Do not return an error from `receive_imf` if we fail to add a member because we are not in chat.
+- Do not add QR inviter to groups immediately.
+- Do not ignore I/O errors in `BlobObject::store_from_base64`.
+
+### Miscellaneous Tasks
+
+- Rustfmt.
+
+### Refactor
+
+- imap: Move resync request from Context to Imap.
+- Replace imap:: calls in migration 73 with SQL queries.
+- Remove unused imports.
+
+### Documentation
+
+- Readme: update language binding section to avoid usage of cffi in new projects ([#7380](https://github.com/chatmail/core/pull/7380)).
+- Fix Context::set_stock_translation reference.
+
+### Tests
+
+- Test editing saved messages.
+- Remove ThreadPoolExecutor from test_wait_next_messages.
+- Move test_two_group_securejoins from receive_imf to securejoin module.
+- At the end of securejoin Bob has two members in a group chat.
+- Bob has 0 members in the chat until securejoin finishes.
+- Do not add QR inviter to groups right after scanning the code.
+
+## [2.25.0] - 2025-11-05
+
+### Features / Changes
+
+- Put self-name into group invite codes ([#7398](https://github.com/chatmail/core/pull/7398)).
+- Slightly nicer and shorter QR and invite codes ([#7390](https://github.com/chatmail/core/pull/7390))
+
+### Fixes
+
+- Add device message instead of partial message when receive_imf fails. This fixes a rare bug where the IMAP loop got stuck.
+- Add info message if user tries to create a QR code for deprecated channel ([#7399](https://github.com/chatmail/core/pull/7399)).
+
+### Miscellaneous Tasks
+
+- deps: Bump actions/upload-artifact from 4 to 5.
+- deps: Bump actions/download-artifact from 5 to 6.
+- deps: Bump astral-sh/setup-uv from 7.1.0 to 7.1.2.
+
+### Refactor
+
+- sql: Do not expose rusqlite Error type in query_map methods.
+
+## [2.24.0] - 2025-11-03
+
+***Note that in v2.24.0, the IMAP loop can get stuck in rare circumstances;
+use v2.23.0 or v2.25.0 instead.***
+
+### Documentation
+
+- Comment why spaced en dash is used to separate message Subject from text.
+
+### Features / Changes
+
+- [**breaking**] QR codes and symmetric encryption for broadcast channels ([#7268](https://github.com/chatmail/core/pull/7268)).
+  - A new QR type AskJoinBroadcast; cloning a broadcast
+    channel is no longer possible; manually adding a member to a broadcast
+    channel is no longer possible (the only way to join a channel is scanning a QR code or clicking a link)
+
+### Refactor
+
+- Split "transport" module out of "login_param".
+
+## [2.23.0] - 2025-11-01
+
+### API-Changes
+
+- Make `dc_chat_is_protected` always return 0.
+- [**breaking**] Remove public APIs to check if the chat is protected.
+- [**breaking**] Remove APIs to create protected chats.
+- [**breaking**] Remove Chat.is_protected().
+- deltachat-rpc-client: Add Account.add_transport_from_qr() API.
+- JSON-RPC: add `get_push_state` to check push notification state ([#7356](https://github.com/chatmail/core/pull/7356)).
+- JSON-RPC: remove unused TypeScript constants ([#7355](https://github.com/chatmail/core/pull/7355)).
+- Remove `Config::SentboxWatch` ([#7178](https://github.com/chatmail/core/pull/7178)).
+- Remove `Config::ConfiguredSentboxFolder` and everything related.
+
+### Build system
+
+- Ignore configuration for the zed editor ([#7322](https://github.com/chatmail/core/pull/7322)).
+- nix: Fix build of deltachat-rpc-server-x86_64-darwin.
+- Update rand to 0.9.
+- Do not install `pdbpp` in the test environment for CFFI Python bindings.
+- Migrate from tokio-tar to astral-tokio-tar.
+- deps: Bump actions/setup-node from 5 to 6.
+- deps: Bump cachix/install-nix-action from 31.8.0 to 31.8.1.
+- Fix Rust 1.91.0 lint for derivable Default.
+
+### CI
+
+- Pin GitHub action `astral-sh/setup-uv`.
+- Set 7 days cooldown on Dependabot updates.
+- Update Rust to 1.91.0.
+
+### Documentation
+
+- Document Autocrypt-Gossip `_verified` attribute.
+
+### Features/Changes
+
+Metadata reduction:
+- Protect Autocrypt header.
+- Anonymize OpenPGP recipients (temorarily disabled due to interoperability problems, see <https://github.com/chatmail/core/issues/7384>).
+- Protect the `Date` header.
+
+Onboarding improvements:
+- Allow plain domain in `dcaccount:` scheme.
+- Do not resolve MX records during configuration.
+
+Preparation for multi-transport:
+- Move the messages only from INBOX and Spam folders.
+- deltachat-rpc-client: Support multiple transports in resetup_account().
+
+Various other changes:
+- Opt-in weekly sending of statistics ([#6851](https://github.com/chatmail/core/pull/6851))
+- Synchronize encrypted groups creation across devices ([#7001](https://github.com/chatmail/core/pull/7001)).
+- Do not send Autocrypt in MDNs.
+- Do not run SecureJoin if we are already in the group.
+- Show if proxy is enabled in connectivity view ([#7359](https://github.com/chatmail/core/pull/7359)).
+
+### Fixes
+
+- Don't ignore QR token timestamp from sync messages.
+- Do not allow sync item timestamps to be in the future.
+- jsonrpc: Fix `ChatListItem::is_self_in_group`.
+- Delete obsolete "configured*" keys from `config` table ([#7171](https://github.com/chatmail/core/pull/7171)).
+- Fix flaky tests::verified_chats::test_verified_chat_editor_reordering and receive_imf::receive_imf_tests::test_two_group_securejoins.
+- Stop using `leftgrps` table.
+- Stop notifying about messages in contact request chats.
+
+### Refactor
+
+- Remove invalid Gmail OAuth2 tokens.
+- Remove ProtectionStatus.
+- Rename chat::create_group_chat() to create_group().
+- Remove error stock strings that are rarely used these days ([#7327](https://github.com/chatmail/core/pull/7327)).
+- Jsonrpc rename change casing in names of jsonrpc structs/enums to comply with rust naming conventions. ([#7324](https://github.com/chatmail/core/pull/7324)).
+- Stop using deprecated Account.configure().
+- add_transport_from_qr: Do not set deprecated config values.
+- sql: Change second query_map function from FnMut to FnOnce.
+- sql: Add query_map_vec().
+- sql: Add query_map_collect().
+- Use rand::fill() instead of rand::rng().fill().
+- Use SampleString.
+- Remove unused call to get_credentials().
+
+### Tests
+
+- rpc-client: VCard color is the same as the contact color ([#7294](https://github.com/chatmail/core/pull/7294)).
+- Add unique offsets to ids generated by `TestContext` to increase test correctness ([#7297](https://github.com/chatmail/core/pull/7297)).
+
 ## [2.22.0] - 2025-10-17
 
 ### Fixes
@@ -6968,3 +7180,8 @@ https://github.com/chatmail/core/pulls?q=is%3Apr+is%3Aclosed
 [2.20.0]: https://github.com/chatmail/core/compare/v2.19.0..v2.20.0
 [2.21.0]: https://github.com/chatmail/core/compare/v2.20.0..v2.21.0
 [2.22.0]: https://github.com/chatmail/core/compare/v2.21.0..v2.22.0
+[2.23.0]: https://github.com/chatmail/core/compare/v2.22.0..v2.23.0
+[2.24.0]: https://github.com/chatmail/core/compare/v2.23.0..v2.24.0
+[2.25.0]: https://github.com/chatmail/core/compare/v2.24.0..v2.25.0
+[2.26.0]: https://github.com/chatmail/core/compare/v2.25.0..v2.26.0
+[2.27.0]: https://github.com/chatmail/core/compare/v2.26.0..v2.27.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "2.22.0"
+version = "2.27.0"
 edition = "2024"
 license = "MPL-2.0"
 rust-version = "1.85"
@@ -66,8 +66,8 @@ humansize = "2"
 hyper = "1"
 hyper-util = "0.1.16"
 image = { version = "0.25.6", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh-gossip = { version = "0.94", default-features = false, features = ["net"] }
-iroh = { version = "0.94", default-features = false }
+iroh-gossip = { version = "0.35", default-features = false, features = ["net"] }
+iroh = { version = "0.35", default-features = false }
 kamadak-exif = "0.6.1"
 libc = { workspace = true }
 mail-builder = { version = "0.4.4", default-features = false }
@@ -156,6 +156,11 @@ name = "receive_emails"
 required-features = ["internals"]
 harness = false
 
+[[bench]]
+name = "decrypting"
+required-features = ["internals"]
+harness = false
+
 [[bench]]
 name = "get_chat_msgs"
 harness = false

README.md

@@ -197,12 +197,10 @@ and then run the script.
 Language bindings are available for:
 
 - **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
+   - -> libdeltachat is going to be deprecated and only exists because Android, iOS and Ubuntu Touch are still using it. If you build a new project, then please use the jsonrpc api instead.
 - **JS**: \[[📂 source](./deltachat-rpc-client) | [📦 npm](https://www.npmjs.com/package/@deltachat/jsonrpc-client) | [📚 docs](https://js.jsonrpc.delta.chat/)\]
 - **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go**
-  - over jsonrpc: \[[📂 source](https://github.com/deltachat/deltachat-rpc-client-go/)\]
-  - over cffi[^1]: \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
-- **Free Pascal**[^1] \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
+- **Go** \[[📂 source](https://github.com/deltachat/deltachat-rpc-client-go/)\]
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -215,5 +213,3 @@ or its language bindings:
 - [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - [Ubuntu Touch](https://codeberg.org/lk108/deltatouch)
 - several **Bots**
-
-[^1]: Out of date / unmaintained, if you like those languages feel free to start maintaining them. If you have questions we'll help you, please ask in the issues.

benches/decrypting.rs

@@ -0,0 +1,200 @@
+//! Benchmarks for message decryption,
+//! comparing decryption of symmetrically-encrypted messages
+//! to decryption of asymmetrically-encrypted messages.
+//!
+//! Call with
+//!
+//! ```text
+//! cargo bench --bench decrypting --features="internals"
+//! ```
+//!
+//! or, if you want to only run e.g. the 'Decrypt a symmetrically encrypted message' benchmark:
+//!
+//! ```text
+//! cargo bench --bench decrypting --features="internals" -- 'Decrypt a symmetrically encrypted message'
+//! ```
+//!
+//! You can also pass a substring.
+//! So, you can run all 'Decrypt and parse' benchmarks with:
+//!
+//! ```text
+//! cargo bench --bench decrypting --features="internals" -- 'Decrypt and parse'
+//! ```
+//!
+//! Symmetric decryption has to try out all known secrets,
+//! You can benchmark this by adapting the `NUM_SECRETS` variable.
+
+use std::hint::black_box;
+
+use criterion::{Criterion, criterion_group, criterion_main};
+use deltachat::internals_for_benches::create_broadcast_secret;
+use deltachat::internals_for_benches::create_dummy_keypair;
+use deltachat::internals_for_benches::save_broadcast_secret;
+use deltachat::{
+    Events,
+    chat::ChatId,
+    config::Config,
+    context::Context,
+    internals_for_benches::key_from_asc,
+    internals_for_benches::parse_and_get_text,
+    internals_for_benches::store_self_keypair,
+    pgp::{KeyPair, decrypt, pk_encrypt, symm_encrypt_message},
+    stock_str::StockStrings,
+};
+use rand::{Rng, rng};
+use tempfile::tempdir;
+
+const NUM_SECRETS: usize = 500;
+
+async fn create_context() -> Context {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let context = Context::new(dbfile.as_path(), 100, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
+
+    context
+        .set_config(Config::ConfiguredAddr, Some("bob@example.net"))
+        .await
+        .unwrap();
+    let secret = key_from_asc(include_str!("../test-data/key/bob-secret.asc")).unwrap();
+    let public = secret.signed_public_key();
+    let key_pair = KeyPair { public, secret };
+    store_self_keypair(&context, &key_pair)
+        .await
+        .expect("Failed to save key");
+
+    context
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Decrypt");
+
+    // ===========================================================================================
+    // Benchmarks for decryption only, without any other parsing
+    // ===========================================================================================
+
+    group.sample_size(10);
+
+    group.bench_function("Decrypt a symmetrically encrypted message", |b| {
+        let plain = generate_plaintext();
+        let secrets = generate_secrets();
+        let encrypted = tokio::runtime::Runtime::new().unwrap().block_on(async {
+            let secret = secrets[NUM_SECRETS / 2].clone();
+            symm_encrypt_message(
+                plain.clone(),
+                create_dummy_keypair("alice@example.org").unwrap().secret,
+                black_box(&secret),
+                true,
+            )
+            .await
+            .unwrap()
+        });
+
+        b.iter(|| {
+            let mut msg =
+                decrypt(encrypted.clone().into_bytes(), &[], black_box(&secrets)).unwrap();
+            let decrypted = msg.as_data_vec().unwrap();
+
+            assert_eq!(black_box(decrypted), plain);
+        });
+    });
+
+    group.bench_function("Decrypt a public-key encrypted message", |b| {
+        let plain = generate_plaintext();
+        let key_pair = create_dummy_keypair("alice@example.org").unwrap();
+        let secrets = generate_secrets();
+        let encrypted = tokio::runtime::Runtime::new().unwrap().block_on(async {
+            pk_encrypt(
+                plain.clone(),
+                vec![black_box(key_pair.public.clone())],
+                Some(key_pair.secret.clone()),
+                true,
+                true,
+            )
+            .await
+            .unwrap()
+        });
+
+        b.iter(|| {
+            let mut msg = decrypt(
+                encrypted.clone().into_bytes(),
+                std::slice::from_ref(&key_pair.secret),
+                black_box(&secrets),
+            )
+            .unwrap();
+            let decrypted = msg.as_data_vec().unwrap();
+
+            assert_eq!(black_box(decrypted), plain);
+        });
+    });
+
+    // ===========================================================================================
+    // Benchmarks for the whole parsing pipeline, incl. decryption (but excl. receive_imf())
+    // ===========================================================================================
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let mut secrets = generate_secrets();
+
+    // "secret" is the shared secret that was used to encrypt text_symmetrically_encrypted.eml.
+    // Put it into the middle of our secrets:
+    secrets[NUM_SECRETS / 2] = "secret".to_string();
+
+    let context = rt.block_on(async {
+        let context = create_context().await;
+        for (i, secret) in secrets.iter().enumerate() {
+            save_broadcast_secret(&context, ChatId::new(10 + i as u32), secret)
+                .await
+                .unwrap();
+        }
+        context
+    });
+
+    group.bench_function("Decrypt and parse a symmetrically encrypted message", |b| {
+        b.to_async(&rt).iter(|| {
+            let ctx = context.clone();
+            async move {
+                let text = parse_and_get_text(
+                    &ctx,
+                    include_bytes!("../test-data/message/text_symmetrically_encrypted.eml"),
+                )
+                .await
+                .unwrap();
+                assert_eq!(text, "Symmetrically encrypted message");
+            }
+        });
+    });
+
+    group.bench_function("Decrypt and parse a public-key encrypted message", |b| {
+        b.to_async(&rt).iter(|| {
+            let ctx = context.clone();
+            async move {
+                let text = parse_and_get_text(
+                    &ctx,
+                    include_bytes!("../test-data/message/text_from_alice_encrypted.eml"),
+                )
+                .await
+                .unwrap();
+                assert_eq!(text, "hi");
+            }
+        });
+    });
+
+    group.finish();
+}
+
+fn generate_secrets() -> Vec<String> {
+    let secrets: Vec<String> = (0..NUM_SECRETS)
+        .map(|_| create_broadcast_secret())
+        .collect();
+    secrets
+}
+
+fn generate_plaintext() -> Vec<u8> {
+    let mut plain: Vec<u8> = vec![0; 500];
+    rng().fill(&mut plain[..]);
+    plain
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "2.22.0"
+version = "2.27.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -2563,6 +2563,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 
 #define         DC_QR_ASK_VERIFYCONTACT      200 // id=contact
 #define         DC_QR_ASK_VERIFYGROUP        202 // text1=groupname
+#define         DC_QR_ASK_VERIFYBROADCAST    204 // text1=broadcast name
 #define         DC_QR_FPR_OK                 210 // id=contact
 #define         DC_QR_FPR_MISMATCH           220 // id=contact
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
@@ -2577,8 +2578,10 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_ERROR                  400 // text1=error string
 #define         DC_QR_WITHDRAW_VERIFYCONTACT 500
 #define         DC_QR_WITHDRAW_VERIFYGROUP   502 // text1=groupname
+#define         DC_QR_WITHDRAW_JOINBROADCAST 504 // text1=broadcast name
 #define         DC_QR_REVIVE_VERIFYCONTACT   510
 #define         DC_QR_REVIVE_VERIFYGROUP     512 // text1=groupname
+#define         DC_QR_REVIVE_JOINBROADCAST   514 // text1=broadcast name
 #define         DC_QR_LOGIN                  520 // text1=email_address
 
 /**
@@ -2595,8 +2598,9 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   ask whether to verify the contact;
  *   if so, start the protocol with dc_join_securejoin().
  *
- * - DC_QR_ASK_VERIFYGROUP with dc_lot_t::text1=Group name:
- *   ask whether to join the group;
+ * - DC_QR_ASK_VERIFYGROUP or DC_QR_ASK_VERIFYBROADCAST
+ *   with dc_lot_t::text1=Group name:
+ *   ask whether to join the chat;
  *   if so, start the protocol with dc_join_securejoin().
  *
  * - DC_QR_FPR_OK with dc_lot_t::id=Contact ID:
@@ -2679,7 +2683,8 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  * Get QR code text that will offer an Setup-Contact or Verified-Group invitation.
  *
  * The scanning device will pass the scanned content to dc_check_qr() then;
- * if dc_check_qr() returns DC_QR_ASK_VERIFYCONTACT or DC_QR_ASK_VERIFYGROUP
+ * if dc_check_qr() returns
+ * DC_QR_ASK_VERIFYCONTACT, DC_QR_ASK_VERIFYGROUP or DC_QR_ASK_VERIFYBROADCAST
  * an out-of-band-verification can be joined using dc_join_securejoin()
  *
  * The returned text will also work as a normal https:-link,
@@ -2720,7 +2725,7 @@ char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_
  * Continue a Setup-Contact or Verified-Group-Invite protocol
  * started on another device with dc_get_securejoin_qr().
  * This function is typically called when dc_check_qr() returns
- * lot.state=DC_QR_ASK_VERIFYCONTACT or lot.state=DC_QR_ASK_VERIFYGROUP.
+ * lot.state=DC_QR_ASK_VERIFYCONTACT, lot.state=DC_QR_ASK_VERIFYGROUP or lot.state=DC_QR_ASK_VERIFYBROADCAST
  *
  * The function returns immediately and the handshake runs in background,
  * sending and receiving several messages.
@@ -3293,12 +3298,30 @@ void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
  * without forgetting to create notifications caused by timing race conditions.
  *
  * @memberof dc_accounts_t
+ * @param accounts The account manager as created by dc_accounts_new().
  * @param timeout The timeout in seconds
  * @return Return 1 if DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE was emitted and 0 otherwise.
  */
 int            dc_accounts_background_fetch    (dc_accounts_t* accounts, uint64_t timeout);
 
 
+/**
+ * Stop ongoing background fetch.
+ *
+ * Calling this function allows to stop dc_accounts_background_fetch() early.
+ * dc_accounts_background_fetch() will then return immediately
+ * and emit DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE unless
+ * if it has failed and returned 0.
+ *
+ * If there is no ongoing dc_accounts_background_fetch() call,
+ * calling this function does nothing.
+ *
+ * @memberof dc_accounts_t
+ * @param accounts The account manager as created by dc_accounts_new().
+ */
+void           dc_accounts_stop_background_fetch (dc_accounts_t *accounts);
+
+
 /**
  * Sets device token for Apple Push Notification service.
  * Returns immediately.
@@ -7515,14 +7538,13 @@ void dc_event_unref(dc_event_t* event);
 
 /// "You set message deletion timer to 1 minute."
 ///
-/// Used in status messages.
+/// @deprecated 2025-11-14, this string is no longer needed
 #define DC_STR_EPHEMERAL_TIMER_1_MINUTE_BY_YOU 142
 
 /// "Message deletion timer is set to 1 minute by %1$s."
 ///
 /// `%1$s` will be replaced by name and address of the contact.
-///
-/// Used in status messages.
+/// @deprecated 2025-11-14, this string is no longer needed
 #define DC_STR_EPHEMERAL_TIMER_1_MINUTE_BY_OTHER 143
 
 /// "You set message deletion timer to 1 hour."
@@ -7736,6 +7758,21 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced with the channel name.
 #define DC_STR_SECURE_JOIN_CHANNEL_QR_DESC  201
 
+/// "Proxy Enabled"
+///
+/// Title for proxy section in connectivity view.
+#define DC_STR_PROXY_ENABLED  220
+
+/// "You are using a proxy. If you're having trouble connecting, try a different proxy."
+///
+/// Description in connectivity view when proxy is enabled.
+#define DC_STR_PROXY_ENABLED_DESCRIPTION  221
+
+/// "Messages in this chat use classic email and are not encrypted."
+///
+/// Used as the first info messages in newly created classic email threads.
+#define DC_STR_CHAT_UNENCRYPTED_EXPLANATON 230
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -4240,7 +4240,17 @@ pub unsafe extern "C" fn dc_contact_get_color(contact: *mut dc_contact_t) -> u32
         return 0;
     }
     let ffi_contact = &*contact;
-    ffi_contact.contact.get_color()
+    let ctx = &*ffi_contact.context;
+    block_on(async move {
+        ffi_contact
+            .contact
+            // We don't want any UIs displaying gray self-color.
+            .get_or_gen_color(ctx)
+            .await
+            .context("Contact::get_color()")
+            .log_err(ctx)
+            .unwrap_or(0)
+    })
 }
 
 #[no_mangle]
@@ -5017,6 +5027,17 @@ pub unsafe extern "C" fn dc_accounts_background_fetch(
     1
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_stop_background_fetch(accounts: *mut dc_accounts_t) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_stop_background_fetch()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    block_on(accounts.read()).stop_background_fetch();
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_set_push_device_token(
     accounts: *mut dc_accounts_t,

deltachat-ffi/src/lot.rs

@@ -45,6 +45,7 @@ impl Lot {
             Self::Qr(qr) => match qr {
                 Qr::AskVerifyContact { .. } => None,
                 Qr::AskVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
+                Qr::AskJoinBroadcast { name, .. } => Some(Cow::Borrowed(name)),
                 Qr::FprOk { .. } => None,
                 Qr::FprMismatch { .. } => None,
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(Cow::Borrowed(fingerprint)),
@@ -57,8 +58,10 @@ impl Lot {
                 Qr::Text { text } => Some(Cow::Borrowed(text)),
                 Qr::WithdrawVerifyContact { .. } => None,
                 Qr::WithdrawVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
+                Qr::WithdrawJoinBroadcast { name, .. } => Some(Cow::Borrowed(name)),
                 Qr::ReviveVerifyContact { .. } => None,
                 Qr::ReviveVerifyGroup { grpname, .. } => Some(Cow::Borrowed(grpname)),
+                Qr::ReviveJoinBroadcast { name, .. } => Some(Cow::Borrowed(name)),
                 Qr::Login { address, .. } => Some(Cow::Borrowed(address)),
             },
             Self::Error(err) => Some(Cow::Borrowed(err)),
@@ -98,6 +101,7 @@ impl Lot {
             Self::Qr(qr) => match qr {
                 Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
                 Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
+                Qr::AskJoinBroadcast { .. } => LotState::QrAskJoinBroadcast,
                 Qr::FprOk { .. } => LotState::QrFprOk,
                 Qr::FprMismatch { .. } => LotState::QrFprMismatch,
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
@@ -110,8 +114,10 @@ impl Lot {
                 Qr::Text { .. } => LotState::QrText,
                 Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
                 Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
+                Qr::WithdrawJoinBroadcast { .. } => LotState::QrWithdrawJoinBroadcast,
                 Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
                 Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
+                Qr::ReviveJoinBroadcast { .. } => LotState::QrReviveJoinBroadcast,
                 Qr::Login { .. } => LotState::QrLogin,
             },
             Self::Error(_err) => LotState::QrError,
@@ -124,6 +130,7 @@ impl Lot {
             Self::Qr(qr) => match qr {
                 Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::AskVerifyGroup { .. } => Default::default(),
+                Qr::AskJoinBroadcast { .. } => Default::default(),
                 Qr::FprOk { contact_id } => contact_id.to_u32(),
                 Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
@@ -135,9 +142,11 @@ impl Lot {
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
                 Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::WithdrawVerifyGroup { .. } => Default::default(),
+                Qr::WithdrawVerifyGroup { .. } | Qr::WithdrawJoinBroadcast { .. } => {
+                    Default::default()
+                }
                 Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::ReviveVerifyGroup { .. } => Default::default(),
+                Qr::ReviveVerifyGroup { .. } | Qr::ReviveJoinBroadcast { .. } => Default::default(),
                 Qr::Login { .. } => Default::default(),
             },
             Self::Error(_) => Default::default(),
@@ -166,6 +175,9 @@ pub enum LotState {
     /// text1=groupname
     QrAskVerifyGroup = 202,
 
+    /// text1=broadcast_name
+    QrAskJoinBroadcast = 204,
+
     /// id=contact
     QrFprOk = 210,
 
@@ -201,11 +213,15 @@ pub enum LotState {
 
     /// text1=groupname
     QrWithdrawVerifyGroup = 502,
+    /// text1=broadcast channel name
+    QrWithdrawJoinBroadcast = 504,
 
     QrReviveVerifyContact = 510,
 
     /// text1=groupname
     QrReviveVerifyGroup = 512,
+    /// text1=groupname
+    QrReviveJoinBroadcast = 514,
 
     /// text1=email_address
     QrLogin = 520,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "2.22.0"
+version = "2.27.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 license = "MPL-2.0"

deltachat-jsonrpc/src/api.rs

@@ -53,6 +53,7 @@ use types::contact::{ContactObject, VcardContact};
 use types::events::Event;
 use types::http::HttpResponse;
 use types::message::{MessageData, MessageObject, MessageReadReceipt};
+use types::notify_state::JsonrpcNotifyState;
 use types::provider_info::ProviderInfo;
 use types::reactions::JsonrpcReactions;
 use types::webxdc::WebxdcMessageInfo;
@@ -272,7 +273,7 @@ impl CommandApi {
     /// The `AccountsBackgroundFetchDone` event is emitted at the end even in case of timeout.
     /// Process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
-    async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
+    async fn background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
         let future = {
             let lock = self.accounts.read().await;
             lock.background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
@@ -282,6 +283,11 @@ impl CommandApi {
         Ok(())
     }
 
+    async fn stop_background_fetch(&self) -> Result<()> {
+        self.accounts.read().await.stop_background_fetch();
+        Ok(())
+    }
+
     // ---------------------------------------------
     // Methods that work on individual accounts
     // ---------------------------------------------
@@ -312,6 +318,12 @@ impl CommandApi {
         }
     }
 
+    /// Get the current push notification state.
+    async fn get_push_state(&self, account_id: u32) -> Result<JsonrpcNotifyState> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx.push_state().await.into())
+    }
+
     /// Get the combined filesize of an account in bytes
     async fn get_account_file_size(&self, account_id: u32) -> Result<u64> {
         let ctx = self.get_context(account_id).await?;
@@ -1023,7 +1035,7 @@ impl CommandApi {
             .await
     }
 
-    /// Create a new **broadcast channel**
+    /// Create a new, outgoing **broadcast channel**
     /// (called "Channel" in the UI).
     ///
     /// Broadcast channels are similar to groups on the sending device,

deltachat-jsonrpc/src/api/types/account.rs

@@ -32,7 +32,10 @@ impl Account {
             let addr = ctx.get_config(Config::Addr).await?;
             let profile_image = ctx.get_config(Config::Selfavatar).await?;
             let color = color_int_to_hex_string(
-                Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
+                Contact::get_by_id(ctx, ContactId::SELF)
+                    .await?
+                    .get_or_gen_color(ctx)
+                    .await?,
             );
             let private_tag = ctx.get_config(Config::PrivateTag).await?;
             Ok(Account::Configured {

deltachat-jsonrpc/src/api/types/chat.rs

@@ -6,7 +6,6 @@ use deltachat::chat::{Chat, ChatId};
 use deltachat::constants::Chattype;
 use deltachat::contact::{Contact, ContactId};
 use deltachat::context::Context;
-use num_traits::cast::ToPrimitive;
 use serde::{Deserialize, Serialize};
 use typescript_type_def::TypeDef;
 
@@ -46,7 +45,7 @@ pub struct FullChat {
     archived: bool,
     pinned: bool,
     // subtitle  - will be moved to frontend because it uses translation functions
-    chat_type: u32,
+    chat_type: JsonrpcChatType,
     is_unpromoted: bool,
     is_self_talk: bool,
     contacts: Vec<ContactObject>,
@@ -130,7 +129,7 @@ impl FullChat {
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
             pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
-            chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
+            chat_type: chat.get_type().into(),
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),
             contacts,
@@ -192,7 +191,7 @@ pub struct BasicChat {
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
     pinned: bool,
-    chat_type: u32,
+    chat_type: JsonrpcChatType,
     is_unpromoted: bool,
     is_self_talk: bool,
     color: String,
@@ -220,7 +219,7 @@ impl BasicChat {
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
             pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
-            chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
+            chat_type: chat.get_type().into(),
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),
             color,
@@ -274,3 +273,37 @@ impl JsonrpcChatVisibility {
         }
     }
 }
+
+#[derive(Clone, Serialize, Deserialize, PartialEq, TypeDef, schemars::JsonSchema)]
+#[serde(rename = "ChatType")]
+pub enum JsonrpcChatType {
+    Single,
+    Group,
+    Mailinglist,
+    OutBroadcast,
+    InBroadcast,
+}
+
+impl From<Chattype> for JsonrpcChatType {
+    fn from(chattype: Chattype) -> Self {
+        match chattype {
+            Chattype::Single => JsonrpcChatType::Single,
+            Chattype::Group => JsonrpcChatType::Group,
+            Chattype::Mailinglist => JsonrpcChatType::Mailinglist,
+            Chattype::OutBroadcast => JsonrpcChatType::OutBroadcast,
+            Chattype::InBroadcast => JsonrpcChatType::InBroadcast,
+        }
+    }
+}
+
+impl From<JsonrpcChatType> for Chattype {
+    fn from(chattype: JsonrpcChatType) -> Self {
+        match chattype {
+            JsonrpcChatType::Single => Chattype::Single,
+            JsonrpcChatType::Group => Chattype::Group,
+            JsonrpcChatType::Mailinglist => Chattype::Mailinglist,
+            JsonrpcChatType::OutBroadcast => Chattype::OutBroadcast,
+            JsonrpcChatType::InBroadcast => Chattype::InBroadcast,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -11,6 +11,7 @@ use num_traits::cast::ToPrimitive;
 use serde::Serialize;
 use typescript_type_def::TypeDef;
 
+use super::chat::JsonrpcChatType;
 use super::color_int_to_hex_string;
 use super::message::MessageViewtype;
 
@@ -23,7 +24,7 @@ pub enum ChatListItemFetchResult {
         name: String,
         avatar_path: Option<String>,
         color: String,
-        chat_type: u32,
+        chat_type: JsonrpcChatType,
         last_updated: Option<i64>,
         summary_text1: String,
         summary_text2: String,
@@ -151,7 +152,7 @@ pub(crate) async fn get_chat_list_item_by_id(
         name: chat.get_name().to_owned(),
         avatar_path,
         color,
-        chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
+        chat_type: chat.get_type().into(),
         last_updated,
         summary_text1,
         summary_text2,

deltachat-jsonrpc/src/api/types/events.rs

@@ -1,8 +1,9 @@
 use deltachat::{Event as CoreEvent, EventType as CoreEventType};
-use num_traits::ToPrimitive;
 use serde::Serialize;
 use typescript_type_def::TypeDef;
 
+use super::chat::JsonrpcChatType;
+
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct Event {
@@ -307,7 +308,7 @@ pub enum EventType {
         /// The type of the joined chat.
         /// This can take the same values
         /// as `BasicChat.chatType` ([`crate::api::types::chat::BasicChat::chat_type`]).
-        chat_type: u32,
+        chat_type: JsonrpcChatType,
         /// ID of the chat in case of success.
         chat_id: u32,
 
@@ -570,7 +571,7 @@ impl From<CoreEventType> for EventType {
                 progress,
             } => SecurejoinInviterProgress {
                 contact_id: contact_id.to_u32(),
-                chat_type: chat_type.to_u32().unwrap_or(0),
+                chat_type: chat_type.into(),
                 chat_id: chat_id.to_u32(),
                 progress,
             },

deltachat-jsonrpc/src/api/types/message.rs

@@ -16,6 +16,7 @@ use num_traits::cast::ToPrimitive;
 use serde::{Deserialize, Serialize};
 use typescript_type_def::TypeDef;
 
+use super::chat::JsonrpcChatType;
 use super::color_int_to_hex_string;
 use super::contact::ContactObject;
 use super::reactions::JsonrpcReactions;
@@ -531,7 +532,7 @@ pub struct MessageSearchResult {
     chat_profile_image: Option<String>,
     chat_color: String,
     chat_name: String,
-    chat_type: u32,
+    chat_type: JsonrpcChatType,
     is_chat_contact_request: bool,
     is_chat_archived: bool,
     message: String,
@@ -569,7 +570,7 @@ impl MessageSearchResult {
             chat_id: chat.id.to_u32(),
             chat_name: chat.get_name().to_owned(),
             chat_color,
-            chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
+            chat_type: chat.get_type().into(),
             chat_profile_image,
             is_chat_contact_request: chat.is_contact_request(),
             is_chat_archived: chat.get_visibility() == ChatVisibility::Archived,

deltachat-jsonrpc/src/api/types/mod.rs

@@ -8,6 +8,7 @@ pub mod http;
 pub mod location;
 pub mod login_param;
 pub mod message;
+pub mod notify_state;
 pub mod provider_info;
 pub mod qr;
 pub mod reactions;

deltachat-jsonrpc/src/api/types/notify_state.rs

@@ -0,0 +1,26 @@
+use deltachat::push::NotifyState;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef, schemars::JsonSchema)]
+#[serde(rename = "NotifyState")]
+pub enum JsonrpcNotifyState {
+    /// Not subscribed to push notifications.
+    NotConnected,
+
+    /// Subscribed to heartbeat push notifications.
+    Heartbeat,
+
+    /// Subscribed to push notifications for new messages.
+    Connected,
+}
+
+impl From<NotifyState> for JsonrpcNotifyState {
+    fn from(state: NotifyState) -> Self {
+        match state {
+            NotifyState::NotConnected => Self::NotConnected,
+            NotifyState::Heartbeat => Self::Heartbeat,
+            NotifyState::Connected => Self::Connected,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/qr.rs

@@ -35,6 +35,26 @@ pub enum QrObject {
         /// Authentication code.
         authcode: String,
     },
+    /// Ask the user whether to join the broadcast channel.
+    AskJoinBroadcast {
+        /// The user-visible name of this broadcast channel
+        name: String,
+        /// A string of random characters,
+        /// uniquely identifying this broadcast channel across all databases/clients.
+        /// Called `grpid` for historic reasons:
+        /// The id of multi-user chats is always called `grpid` in the database
+        /// because groups were once the only multi-user chats.
+        grpid: String,
+        /// ID of the contact who owns the broadcast channel and created the QR code.
+        contact_id: u32,
+        /// Fingerprint of the broadcast channel owner's key as scanned from the QR code.
+        fingerprint: String,
+
+        /// Invite number.
+        invitenumber: String,
+        /// Authentication code.
+        authcode: String,
+    },
     /// Contact fingerprint is verified.
     ///
     /// Ask the user if they want to start chatting.
@@ -137,6 +157,21 @@ pub enum QrObject {
         /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to withdraw their own broadcast channel invite QR code.
+    WithdrawJoinBroadcast {
+        /// Broadcast name.
+        name: String,
+        /// ID, uniquely identifying this chat. Called grpid for historic reasons.
+        grpid: String,
+        /// Contact ID. Always `ContactId::SELF`.
+        contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
+        fingerprint: String,
+        /// Invite number.
+        invitenumber: String,
+        /// Authentication code.
+        authcode: String,
+    },
     /// Ask the user if they want to revive their own QR code.
     ReviveVerifyContact {
         /// Contact ID.
@@ -163,6 +198,21 @@ pub enum QrObject {
         /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own broadcast channel invite QR code.
+    ReviveJoinBroadcast {
+        /// Broadcast name.
+        name: String,
+        /// Globally unique chat ID. Called grpid for historic reasons.
+        grpid: String,
+        /// Contact ID. Always `ContactId::SELF`.
+        contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
+        fingerprint: String,
+        /// Invite number.
+        invitenumber: String,
+        /// Authentication code.
+        authcode: String,
+    },
     /// `dclogin:` scheme parameters.
     ///
     /// Ask the user if they want to login with the email address.
@@ -208,6 +258,25 @@ impl From<Qr> for QrObject {
                     authcode,
                 }
             }
+            Qr::AskJoinBroadcast {
+                name,
+                grpid,
+                contact_id,
+                fingerprint,
+                authcode,
+                invitenumber,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::AskJoinBroadcast {
+                    name,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    authcode,
+                    invitenumber,
+                }
+            }
             Qr::FprOk { contact_id } => {
                 let contact_id = contact_id.to_u32();
                 QrObject::FprOk { contact_id }
@@ -267,6 +336,25 @@ impl From<Qr> for QrObject {
                     authcode,
                 }
             }
+            Qr::WithdrawJoinBroadcast {
+                name,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::WithdrawJoinBroadcast {
+                    name,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
             Qr::ReviveVerifyContact {
                 contact_id,
                 fingerprint,
@@ -301,6 +389,25 @@ impl From<Qr> for QrObject {
                     authcode,
                 }
             }
+            Qr::ReviveJoinBroadcast {
+                name,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::ReviveJoinBroadcast {
+                    name,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
             Qr::Login { address, .. } => QrObject::Login { address },
         }
     }

deltachat-jsonrpc/typescript/package.json

@@ -54,5 +54,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "2.22.0"
+  "version": "2.27.0"
 }

deltachat-jsonrpc/typescript/scripts/generate-constants.js

@@ -40,15 +40,35 @@ const constants = data
       key.startsWith("DC_DOWNLOAD") ||
       key.startsWith("DC_INFO_") ||
       (key.startsWith("DC_MSG") && !key.startsWith("DC_MSG_ID")) ||
-      key.startsWith("DC_QR_")
+      key.startsWith("DC_QR_") ||
+      key.startsWith("DC_CERTCK_") ||
+      key.startsWith("DC_SOCKET_") ||
+      key.startsWith("DC_LP_AUTH_") ||
+      key.startsWith("DC_PUSH_") ||
+      key.startsWith("DC_TEXT1_") ||
+      key.startsWith("DC_CHAT_TYPE")
     );
   })
   .map((row) => {
-    return `  ${row.key}: ${row.value}`;
+    return `  export const ${row.key} = ${row.value};`;
   })
-  .join(",\n");
+  .join("\n");
 
 writeFileSync(
   resolve(__dirname, "../generated/constants.ts"),
-  `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, " =")},\n}\n`,
+  `// Generated!
+
+export namespace C {
+${constants}
+  /** @deprecated 10-8-2025 compare string directly with \`== "Group"\` */
+  export const DC_CHAT_TYPE_GROUP = "Group";
+  /** @deprecated 10-8-2025 compare string directly with \`== "InBroadcast"\`*/
+  export const DC_CHAT_TYPE_IN_BROADCAST = "InBroadcast";
+  /** @deprecated 10-8-2025 compare string directly with \`== "Mailinglist"\` */
+  export const DC_CHAT_TYPE_MAILINGLIST = "Mailinglist";
+  /** @deprecated 10-8-2025 compare string directly with \`== "OutBroadcast"\` */
+  export const DC_CHAT_TYPE_OUT_BROADCAST = "OutBroadcast";
+  /** @deprecated 10-8-2025 compare string directly with \`== "Single"\` */
+  export const DC_CHAT_TYPE_SINGLE = "Single";
+}\n`,
 );

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "2.22.0"
+version = "2.27.0"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/chatmail/core"

deltachat-repl/src/cmdline.rs

@@ -70,11 +70,6 @@ async fn reset_tables(context: &Context, bits: i32) {
             .await
             .unwrap();
         context.sql().config_cache().write().await.clear();
-        context
-            .sql()
-            .execute("DELETE FROM leftgrps;", ())
-            .await
-            .unwrap();
         println!("(8) Rest but server config reset.");
     }
 

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "2.22.0"
+version = "2.27.0"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -125,6 +125,11 @@ class Account:
         """Add a new transport."""
         yield self._rpc.add_or_update_transport.future(self.id, params)
 
+    @futuremethod
+    def add_transport_from_qr(self, qr: str):
+        """Add a new transport using a QR code."""
+        yield self._rpc.add_transport_from_qr.future(self.id, qr)
+
     @futuremethod
     def list_transports(self):
         """Return the list of all email accounts that are used as a transport in the current profile."""
@@ -321,7 +326,7 @@ class Account:
         return Chat(self, self._rpc.create_group_chat(self.id, name, False))
 
     def create_broadcast(self, name: str) -> Chat:
-        """Create a new **broadcast channel**
+        """Create a new, outgoing **broadcast channel**
         (called "Channel" in the UI).
 
         Broadcast channels are similar to groups on the sending device,
@@ -394,9 +399,10 @@ class Account:
         next_msg_ids = self._rpc.get_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
+    @futuremethod
     def wait_next_messages(self) -> list[Message]:
         """Wait for new messages and return a list of them."""
-        next_msg_ids = self._rpc.wait_next_msgs(self.id)
+        next_msg_ids = yield self._rpc.wait_next_msgs.future(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
     def wait_for_incoming_msg_event(self):
@@ -411,12 +417,21 @@ class Account:
         """Wait for messages noticed event and return it."""
         return self.wait_for_event(EventType.MSGS_NOTICED)
 
+    def wait_for_msg(self, event_type) -> Message:
+        """Wait for an event about the message.
+
+        Consumes all events before the matching event.
+        Returns a message corresponding to the msg_id field of the event.
+        """
+        event = self.wait_for_event(event_type)
+        return self.get_message_by_id(event.msg_id)
+
     def wait_for_incoming_msg(self):
         """Wait for incoming message and return it.
 
         Consumes all events before the next incoming message event.
         """
-        return self.get_message_by_id(self.wait_for_incoming_msg_event().msg_id)
+        return self.wait_for_msg(EventType.INCOMING_MSG)
 
     def wait_for_securejoin_inviter_success(self):
         """Wait until SecureJoin process finishes successfully on the inviter side."""

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -91,19 +91,17 @@ class ChatId(IntEnum):
     LAST_SPECIAL = 9
 
 
-class ChatType(IntEnum):
+class ChatType(str, Enum):
     """Chat type."""
 
-    UNDEFINED = 0
-
-    SINGLE = 100
+    SINGLE = "Single"
     """1:1 chat, i.e. a direct chat with a single contact"""
 
-    GROUP = 120
+    GROUP = "Group"
 
-    MAILINGLIST = 140
+    MAILINGLIST = "Mailinglist"
 
-    OUT_BROADCAST = 160
+    OUT_BROADCAST = "OutBroadcast"
     """Outgoing broadcast channel, called "Channel" in the UI.
 
     The user can send into this channel,
@@ -115,7 +113,7 @@ class ChatType(IntEnum):
     which would make it hard to grep for it.
     """
 
-    IN_BROADCAST = 165
+    IN_BROADCAST = "InBroadcast"
     """Incoming broadcast channel, called "Channel" in the UI.
 
     This channel is read-only,

deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from ._utils import AttrDict
+from ._utils import AttrDict, futuremethod
 from .account import Account
 
 if TYPE_CHECKING:
@@ -39,6 +39,15 @@ class DeltaChat:
         """Stop the I/O of all accounts."""
         self.rpc.stop_io_for_all_accounts()
 
+    @futuremethod
+    def background_fetch(self, timeout_in_seconds: int) -> None:
+        """Run background fetch for all accounts."""
+        yield self.rpc.background_fetch.future(timeout_in_seconds)
+
+    def stop_background_fetch(self) -> None:
+        """Stop ongoing background fetch."""
+        self.rpc.stop_background_fetch()
+
     def maybe_network(self) -> None:
         """Indicate that the network conditions might have changed."""
         self.rpc.maybe_network()

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -93,6 +93,17 @@ class Message:
             if event.kind == EventType.MSG_DELIVERED and event.msg_id == self.id:
                 break
 
+    def resend(self) -> None:
+        """Resend messages and make information available for newly added chat members.
+        Resending sends out the original message, however, recipients and webxdc-status may differ.
+        Clients that already have the original message can still ignore the resent message as
+        they have tracked the state by dedicated updates.
+
+        Some messages cannot be resent, eg. info-messages, drafts, already pending messages,
+        or messages that are not sent by SELF.
+        """
+        self._rpc.resend_messages(self.account.id, [self.id])
+
     @futuremethod
     def send_webxdc_realtime_advertisement(self):
         """Send an advertisement to join the realtime channel."""

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -43,10 +43,9 @@ class ACFactory:
     @futuremethod
     def new_configured_account(self):
         """Create a new configured account."""
-        addr, password = self.get_credentials()
         account = self.get_unconfigured_account()
-        params = {"addr": addr, "password": password}
-        yield account.add_or_update_transport.future(params)
+        domain = os.getenv("CHATMAIL_DOMAIN")
+        yield account.add_transport_from_qr.future(f"dcaccount:{domain}")
 
         assert account.is_configured()
         return account
@@ -136,9 +135,15 @@ def rpc(tmp_path) -> AsyncGenerator:
 
 
 @pytest.fixture
-def acfactory(rpc) -> AsyncGenerator:
+def dc(rpc) -> DeltaChat:
+    """Return account manager."""
+    return DeltaChat(rpc)
+
+
+@pytest.fixture
+def acfactory(dc) -> AsyncGenerator:
     """Return account factory fixture."""
-    return ACFactory(DeltaChat(rpc))
+    return ACFactory(dc)
 
 
 @pytest.fixture

deltachat-rpc-client/tests/conftest.py

@@ -85,11 +85,11 @@ class DirectImap:
 
     def get_all_messages(self) -> list[MailMessage]:
         assert not self._idling
-        return list(self.conn.fetch())
+        return list(self.conn.fetch(mark_seen=False))
 
     def get_unread_messages(self) -> list[str]:
         assert not self._idling
-        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
+        return [msg.uid for msg in self.conn.fetch(AND(seen=False), mark_seen=False)]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
@@ -173,7 +173,6 @@ class DirectImap:
 class IdleManager:
     def __init__(self, direct_imap) -> None:
         self.direct_imap = direct_imap
-        self.log = direct_imap.account.log
         # fetch latest messages before starting idle so that it only
         # returns messages that arrive anew
         self.direct_imap.conn.fetch("1:*")
@@ -181,14 +180,11 @@ class IdleManager:
 
     def check(self, timeout=None) -> list[bytes]:
         """(blocking) wait for next idle message from server."""
-        self.log("imap-direct: calling idle_check")
-        res = self.direct_imap.conn.idle.poll(timeout=timeout)
-        self.log(f"imap-direct: idle_check returned {res!r}")
-        return res
+        return self.direct_imap.conn.idle.poll(timeout=timeout)
 
-    def wait_for_new_message(self, timeout=None) -> bytes:
+    def wait_for_new_message(self) -> bytes:
         while True:
-            for item in self.check(timeout=timeout):
+            for item in self.check():
                 if b"EXISTS" in item or b"RECENT" in item:
                     return item
 
@@ -196,10 +192,8 @@ class IdleManager:
         """Return first message with SEEN flag from a running idle-stream."""
         while True:
             for item in self.check(timeout=timeout):
-                if FETCH in item:
-                    self.log(str(item))
-                    if FLAGS in item and rb"\Seen" in item:
-                        return int(item.split(b" ")[1])
+                if FETCH in item and FLAGS in item and rb"\Seen" in item:
+                    return int(item.split(b" ")[1])
 
     def done(self):
         """send idle-done to server if we are currently in idle mode."""

deltachat-rpc-client/tests/test_calls.py

@@ -103,7 +103,7 @@ def test_no_contact_request_call(acfactory) -> None:
         # There should be no incoming call notification.
         assert event.kind != EventType.INCOMING_CALL
 
-        if event.kind == EventType.INCOMING_MSG:
+        if event.kind == EventType.MSGS_CHANGED:
             msg = bob.get_message_by_id(event.msg_id)
-            assert msg.get_snapshot().text == "Hello!"
-            break
+            if msg.get_snapshot().text == "Hello!":
+                break

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -169,6 +169,8 @@ def test_imap_sync_seen_msgs(acfactory: ACFactory) -> None:
     """
     alice, alice_second_device, bob, alice_chat_bob = get_multi_account_test_setup(acfactory)
 
+    bob.create_chat(alice)
+
     alice_chat_bob.send_text("hello")
 
     msg = bob.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_folders.py

@@ -0,0 +1,538 @@
+import logging
+import re
+import time
+
+import pytest
+from imap_tools import AND, U
+
+from deltachat_rpc_client import Contact, EventType, Message
+
+
+def test_move_works(acfactory):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac2.set_config("mvbox_move", "1")
+    ac2.bring_online()
+
+    chat = ac1.create_chat(ac2)
+    chat.send_text("message1")
+
+    # Message is moved to the movebox
+    ac2.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
+
+    # Message is downloaded
+    msg = ac2.wait_for_incoming_msg().get_snapshot()
+    assert msg.text == "message1"
+
+
+def test_move_avoids_loop(acfactory, direct_imap):
+    """Test that the message is only moved from INBOX to DeltaChat.
+
+    This is to avoid busy loop if moved message reappears in the Inbox
+    or some scanned folder later.
+    For example, this happens on servers that alias `INBOX.DeltaChat` to `DeltaChat` folder,
+    so the message moved to `DeltaChat` appears as a new message in the `INBOX.DeltaChat` folder.
+    We do not want to move this message from `INBOX.DeltaChat` to `DeltaChat` again.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac2.set_config("mvbox_move", "1")
+    ac2.set_config("delete_server_after", "0")
+    ac2.bring_online()
+
+    # Create INBOX.DeltaChat folder and make sure
+    # it is detected by full folder scan.
+    ac2_direct_imap = direct_imap(ac2)
+    ac2_direct_imap.create_folder("INBOX.DeltaChat")
+    ac2.stop_io()
+    ac2.start_io()
+
+    while True:
+        event = ac2.wait_for_event()
+        # Wait until the end of folder scan.
+        if event.kind == EventType.INFO and "Found folders:" in event.msg:
+            break
+
+    ac1_chat = acfactory.get_accepted_chat(ac1, ac2)
+    ac1_chat.send_text("Message 1")
+
+    # Message is moved to the DeltaChat folder and downloaded.
+    ac2_msg1 = ac2.wait_for_incoming_msg().get_snapshot()
+    assert ac2_msg1.text == "Message 1"
+
+    # Move the message to the INBOX.DeltaChat again.
+    # We assume that test server uses "." as the delimiter.
+    ac2_direct_imap.select_folder("DeltaChat")
+    ac2_direct_imap.conn.move(["*"], "INBOX.DeltaChat")
+
+    ac1_chat.send_text("Message 2")
+    ac2_msg2 = ac2.wait_for_incoming_msg().get_snapshot()
+    assert ac2_msg2.text == "Message 2"
+
+    # Stop and start I/O to trigger folder scan.
+    ac2.stop_io()
+    ac2.start_io()
+    while True:
+        event = ac2.wait_for_event()
+        # Wait until the end of folder scan.
+        if event.kind == EventType.INFO and "Found folders:" in event.msg:
+            break
+
+    # Check that Message 1 is still in the INBOX.DeltaChat folder
+    # and Message 2 is in the DeltaChat folder.
+    ac2_direct_imap.select_folder("INBOX")
+    assert len(ac2_direct_imap.get_all_messages()) == 0
+    ac2_direct_imap.select_folder("DeltaChat")
+    assert len(ac2_direct_imap.get_all_messages()) == 1
+    ac2_direct_imap.select_folder("INBOX.DeltaChat")
+    assert len(ac2_direct_imap.get_all_messages()) == 1
+
+
+def test_reactions_for_a_reordering_move(acfactory, direct_imap):
+    """When a batch of messages is moved from Inbox to DeltaChat folder with a single MOVE command,
+    their UIDs may be reordered (e.g. Gmail is known for that) which led to that messages were
+    processed by receive_imf in the wrong order, and, particularly, reactions were processed before
+    messages they refer to and thus dropped.
+    """
+    (ac1,) = acfactory.get_online_accounts(1)
+
+    addr, password = acfactory.get_credentials()
+    ac2 = acfactory.get_unconfigured_account()
+    ac2.add_or_update_transport({"addr": addr, "password": password})
+    ac2.set_config("mvbox_move", "1")
+    assert ac2.is_configured()
+
+    ac2.bring_online()
+    chat1 = acfactory.get_accepted_chat(ac1, ac2)
+    ac2.stop_io()
+
+    logging.info("sending message + reaction from ac1 to ac2")
+    msg1 = chat1.send_text("hi")
+    msg1.wait_until_delivered()
+    # It's is sad, but messages must differ in their INTERNALDATEs to be processed in the correct
+    # order by DC, and most (if not all) mail servers provide only seconds precision.
+    time.sleep(1.1)
+    react_str = "\N{THUMBS UP SIGN}"
+    msg1.send_reaction(react_str).wait_until_delivered()
+
+    logging.info("moving messages to ac2's DeltaChat folder in the reverse order")
+    ac2_direct_imap = direct_imap(ac2)
+    ac2_direct_imap.connect()
+    for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()], reverse=True):
+        ac2_direct_imap.conn.move(uid, "DeltaChat")
+
+    logging.info("receiving messages by ac2")
+    ac2.start_io()
+    msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
+    assert msg2.get_snapshot().text == msg1.get_snapshot().text
+    reactions = msg2.get_reactions()
+    contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
+    assert len(contacts) == 1
+    assert contacts[0].get_snapshot().address == ac1.get_config("addr")
+    assert list(reactions.reactions_by_contact.values())[0] == [react_str]
+
+
+def test_delete_deltachat_folder(acfactory, direct_imap):
+    """Test that DeltaChat folder is recreated if user deletes it manually."""
+    ac1 = acfactory.new_configured_account()
+    ac1.set_config("mvbox_move", "1")
+    ac1.bring_online()
+
+    ac1_direct_imap = direct_imap(ac1)
+    ac1_direct_imap.conn.folder.delete("DeltaChat")
+    assert "DeltaChat" not in ac1_direct_imap.list_folders()
+
+    # Wait until new folder is created and UIDVALIDITY is updated.
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == EventType.INFO and "uid/validity change folder DeltaChat" in event.msg:
+            break
+
+    ac2 = acfactory.get_online_account()
+    ac2.create_chat(ac1).send_text("hello")
+    msg = ac1.wait_for_incoming_msg().get_snapshot()
+    assert msg.text == "hello"
+
+    assert "DeltaChat" in ac1_direct_imap.list_folders()
+
+
+def test_dont_show_emails(acfactory, direct_imap, log):
+    """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
+    So: If it's outgoing AND there is no Received header, then ignore the email.
+
+    If the draft email is sent out and received later (i.e. it's in "Inbox"), it must be shown.
+
+    Also, test that unknown emails in the Spam folder are not shown."""
+    ac1 = acfactory.new_configured_account()
+    ac1.stop_io()
+    ac1.set_config("show_emails", "2")
+
+    ac1.create_contact("alice@example.org").create_chat()
+
+    ac1_direct_imap = direct_imap(ac1)
+    ac1_direct_imap.create_folder("Drafts")
+    ac1_direct_imap.create_folder("Spam")
+    ac1_direct_imap.create_folder("Junk")
+
+    # Learn UID validity for all folders.
+    ac1.set_config("scan_all_folders_debounce_secs", "0")
+    ac1.start_io()
+    ac1.wait_for_event(EventType.IMAP_INBOX_IDLE)
+    ac1.stop_io()
+
+    ac1_direct_imap.append(
+        "Drafts",
+        """
+        From: ac1 <{}>
+        Subject: subj
+        To: alice@example.org
+        Message-ID: <aepiors@example.org>
+        Content-Type: text/plain; charset=utf-8
+
+        message in Drafts received later
+    """.format(
+            ac1.get_config("configured_addr"),
+        ),
+    )
+    ac1_direct_imap.append(
+        "Spam",
+        """
+        From: unknown.address@junk.org
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Unknown message in Spam
+    """.format(
+            ac1.get_config("configured_addr"),
+        ),
+    )
+    ac1_direct_imap.append(
+        "Spam",
+        """
+        From: unknown.address@junk.org, unkwnown.add@junk.org
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message2@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Unknown & malformed message in Spam
+    """.format(
+            ac1.get_config("configured_addr"),
+        ),
+    )
+    ac1_direct_imap.append(
+        "Spam",
+        """
+        From: delta<address: inbox@nhroy.com>
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message99@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Unknown & malformed message in Spam
+    """.format(
+            ac1.get_config("configured_addr"),
+        ),
+    )
+    ac1_direct_imap.append(
+        "Spam",
+        """
+        From: alice@example.org
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message3@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Actually interesting message in Spam
+    """.format(
+            ac1.get_config("configured_addr"),
+        ),
+    )
+    ac1_direct_imap.append(
+        "Junk",
+        """
+        From: unknown.address@junk.org
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Unknown message in Junk
+    """.format(
+            ac1.get_config("configured_addr"),
+        ),
+    )
+
+    ac1.set_config("scan_all_folders_debounce_secs", "0")
+    log.section("All prepared, now let DC find the message")
+    ac1.start_io()
+
+    # Wait until each folder was scanned, this is necessary for this test to test what it should test:
+    ac1.wait_for_event(EventType.IMAP_INBOX_IDLE)
+
+    fresh_msgs = list(ac1.get_fresh_messages())
+    msg = fresh_msgs[0].get_snapshot()
+    chat_msgs = msg.chat.get_messages()
+    assert len(chat_msgs) == 1
+    assert msg.text == "subj – Actually interesting message in Spam"
+
+    assert not any("unknown.address" in c.get_full_snapshot().name for c in ac1.get_chatlist())
+    ac1_direct_imap.select_folder("Spam")
+    assert ac1_direct_imap.get_uid_by_message_id("spam.message@junk.org")
+
+    ac1.stop_io()
+    log.section("'Send out' the draft by moving it to Inbox, and wait for DC to display it this time")
+    ac1_direct_imap.select_folder("Drafts")
+    uid = ac1_direct_imap.get_uid_by_message_id("aepiors@example.org")
+    ac1_direct_imap.conn.move(uid, "Inbox")
+
+    ac1.start_io()
+    event = ac1.wait_for_event(EventType.MSGS_CHANGED)
+    msg2 = Message(ac1, event.msg_id).get_snapshot()
+
+    assert msg2.text == "subj – message in Drafts received later"
+    assert len(msg.chat.get_messages()) == 2
+
+
+def test_move_works_on_self_sent(acfactory):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    # Enable movebox and wait until it is created.
+    ac1.set_config("mvbox_move", "1")
+    ac1.set_config("bcc_self", "1")
+    ac1.bring_online()
+
+    chat = ac1.create_chat(ac2)
+    chat.send_text("message1")
+    ac1.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
+    chat.send_text("message2")
+    ac1.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
+    chat.send_text("message3")
+    ac1.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
+
+
+def test_moved_markseen(acfactory, direct_imap):
+    """Test that message already moved to DeltaChat folder is marked as seen."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac2.set_config("mvbox_move", "1")
+    ac2.set_config("delete_server_after", "0")
+    ac2.set_config("sync_msgs", "0")  # Do not send a sync message when accepting a contact request.
+    ac2.bring_online()
+
+    ac2.stop_io()
+    ac2_direct_imap = direct_imap(ac2)
+    with ac2_direct_imap.idle() as idle2:
+        ac1.create_chat(ac2).send_text("Hello!")
+        idle2.wait_for_new_message()
+
+    # Emulate moving of the message to DeltaChat folder by Sieve rule.
+    ac2_direct_imap.conn.move(["*"], "DeltaChat")
+    ac2_direct_imap.select_folder("DeltaChat")
+    assert len(list(ac2_direct_imap.conn.fetch("*", mark_seen=False))) == 1
+
+    with ac2_direct_imap.idle() as idle2:
+        ac2.start_io()
+
+        ev = ac2.wait_for_event(EventType.MSGS_CHANGED)
+        msg = ac2.get_message_by_id(ev.msg_id)
+        assert msg.get_snapshot().text == "Messages are end-to-end encrypted."
+
+        ev = ac2.wait_for_event(EventType.INCOMING_MSG)
+        msg = ac2.get_message_by_id(ev.msg_id)
+        chat = ac2.get_chat_by_id(ev.chat_id)
+
+        # Accept the contact request.
+        chat.accept()
+        msg.mark_seen()
+        idle2.wait_for_seen()
+
+    assert len(list(ac2_direct_imap.conn.fetch(AND(seen=True, uid=U(1, "*")), mark_seen=False))) == 1
+
+
+@pytest.mark.parametrize("mvbox_move", [True, False])
+def test_markseen_message_and_mdn(acfactory, direct_imap, mvbox_move):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    for ac in ac1, ac2:
+        ac.set_config("delete_server_after", "0")
+        if mvbox_move:
+            ac.set_config("mvbox_move", "1")
+            ac.bring_online()
+
+    # Do not send BCC to self, we only want to test MDN on ac1.
+    ac1.set_config("bcc_self", "0")
+
+    acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+    msg = ac2.wait_for_incoming_msg()
+    msg.mark_seen()
+
+    if mvbox_move:
+        rex = re.compile("Marked messages [0-9]+ in folder DeltaChat as seen.")
+    else:
+        rex = re.compile("Marked messages [0-9]+ in folder INBOX as seen.")
+
+    for ac in ac1, ac2:
+        while True:
+            event = ac.wait_for_event()
+            if event.kind == EventType.INFO and rex.search(event.msg):
+                break
+
+    folder = "mvbox" if mvbox_move else "inbox"
+    ac1_direct_imap = direct_imap(ac1)
+    ac2_direct_imap = direct_imap(ac2)
+
+    ac1_direct_imap.select_config_folder(folder)
+    ac2_direct_imap.select_config_folder(folder)
+
+    # Check that the mdn is marked as seen
+    assert len(list(ac1_direct_imap.conn.fetch(AND(seen=True), mark_seen=False))) == 1
+    # Check original message is marked as seen
+    assert len(list(ac2_direct_imap.conn.fetch(AND(seen=True), mark_seen=False))) == 1
+
+
+def test_mvbox_and_trash(acfactory, direct_imap, log):
+    log.section("ac1: start with mvbox")
+    ac1 = acfactory.get_online_account()
+    ac1.set_config("mvbox_move", "1")
+    ac1.bring_online()
+
+    log.section("ac2: start without a mvbox")
+    ac2 = acfactory.get_online_account()
+
+    log.section("ac1: create trash")
+    ac1_direct_imap = direct_imap(ac1)
+    ac1_direct_imap.create_folder("Trash")
+    ac1.set_config("scan_all_folders_debounce_secs", "0")
+    ac1.stop_io()
+    ac1.start_io()
+
+    log.section("ac1: send message and wait for ac2 to receive it")
+    acfactory.get_accepted_chat(ac1, ac2).send_text("message1")
+    assert ac2.wait_for_incoming_msg().get_snapshot().text == "message1"
+
+    assert ac1.get_config("configured_mvbox_folder") == "DeltaChat"
+    while ac1.get_config("configured_trash_folder") != "Trash":
+        ac1.wait_for_event(EventType.CONNECTIVITY_CHANGED)
+
+
+@pytest.mark.parametrize(
+    ("folder", "move", "expected_destination"),
+    [
+        (
+            "xyz",
+            False,
+            "xyz",
+        ),  # Test that emails aren't found in a random folder
+        (
+            "xyz",
+            True,
+            "xyz",
+        ),  # ...emails are found in a random folder and downloaded without moving
+        (
+            "Spam",
+            False,
+            "INBOX",
+        ),  # ...emails are moved from the spam folder to the Inbox
+    ],
+)
+# Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
+# the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
+def test_scan_folders(acfactory, log, direct_imap, folder, move, expected_destination):
+    """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
+    variant = folder + "-" + str(move) + "-" + expected_destination
+    log.section("Testing variant " + variant)
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("delete_server_after", "0")
+    if move:
+        ac1.set_config("mvbox_move", "1")
+        ac1.bring_online()
+
+    ac1.stop_io()
+    ac1_direct_imap = direct_imap(ac1)
+    ac1_direct_imap.create_folder(folder)
+
+    # Wait until each folder was selected once and we are IDLEing:
+    ac1.start_io()
+    ac1.bring_online()
+
+    ac1.stop_io()
+    assert folder in ac1_direct_imap.list_folders()
+
+    log.section("Send a message from ac2 to ac1 and manually move it to `folder`")
+    ac1_direct_imap.select_config_folder("inbox")
+    with ac1_direct_imap.idle() as idle1:
+        acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
+        idle1.wait_for_new_message()
+    ac1_direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
+
+    log.section("start_io() and see if DeltaChat finds the message (" + variant + ")")
+    ac1.set_config("scan_all_folders_debounce_secs", "0")
+    ac1.start_io()
+    chat = ac1.create_chat(ac2)
+    n_msgs = 1  # "Messages are end-to-end encrypted."
+    if folder == "Spam":
+        msg = ac1.wait_for_incoming_msg().get_snapshot()
+        assert msg.text == "hello"
+        n_msgs += 1
+    else:
+        ac1.wait_for_event(EventType.IMAP_INBOX_IDLE)
+    assert len(chat.get_messages()) == n_msgs
+
+    # The message has reached its destination.
+    ac1_direct_imap.select_folder(expected_destination)
+    assert len(ac1_direct_imap.get_all_messages()) == 1
+    if folder != expected_destination:
+        ac1_direct_imap.select_folder(folder)
+        assert len(ac1_direct_imap.get_all_messages()) == 0
+
+
+def test_trash_multiple_messages(acfactory, direct_imap, log):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac2.stop_io()
+
+    # Create the Trash folder on IMAP server and configure deletion to it. There was a bug that if
+    # Trash wasn't configured initially, it can't be configured later, let's check this.
+    log.section("Creating trash folder")
+    ac2_direct_imap = direct_imap(ac2)
+    ac2_direct_imap.create_folder("Trash")
+    ac2.set_config("delete_server_after", "0")
+    ac2.set_config("sync_msgs", "0")
+    ac2.set_config("delete_to_trash", "1")
+
+    log.section("Check that Trash can be configured initially as well")
+    ac3 = ac2.clone()
+    ac3.bring_online()
+    assert ac3.get_config("configured_trash_folder")
+    ac3.stop_io()
+
+    ac2.start_io()
+    chat12 = acfactory.get_accepted_chat(ac1, ac2)
+
+    log.section("ac1: sending 3 messages")
+    texts = ["first", "second", "third"]
+    for text in texts:
+        chat12.send_text(text)
+
+    log.section("ac2: waiting for all messages on the other side")
+    to_delete = []
+    for text in texts:
+        msg = ac2.wait_for_incoming_msg().get_snapshot()
+        assert msg.text in texts
+        if text != "second":
+            to_delete.append(msg)
+    # ac2 has received some messages, this is impossible w/o the trash folder configured, let's
+    # check the configuration.
+    assert ac2.get_config("configured_trash_folder") == "Trash"
+
+    log.section("ac2: deleting all messages except second")
+    assert len(to_delete) == len(texts) - 1
+    ac2.delete_messages(to_delete)
+
+    log.section("ac2: test that only one message is left")
+    while 1:
+        ac2.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
+        ac2_direct_imap.select_config_folder("inbox")
+        nr_msgs = len(ac2_direct_imap.get_all_messages())
+        assert nr_msgs > 0
+        if nr_msgs == 1:
+            break

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -84,7 +84,7 @@ def test_realtime_sequentially(acfactory, path_to_webxdc):
 
     # share a webxdc app between ac1 and ac2
     ac1_webxdc_msg = acfactory.send_message(from_account=ac1, to_account=ac2, text="play", file=path_to_webxdc)
-    ac2_webxdc_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
     snapshot = ac2_webxdc_msg.get_snapshot()
     assert snapshot.text == "play"
 
@@ -94,7 +94,7 @@ def test_realtime_sequentially(acfactory, path_to_webxdc):
     acfactory.send_message(from_account=ac1, to_account=ac2, text="ping1")
 
     log("waiting for incoming message on ac2")
-    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac2.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "ping1"
 
     log("sending ac2 -> ac1 realtime advertisement and additional message")
@@ -102,7 +102,7 @@ def test_realtime_sequentially(acfactory, path_to_webxdc):
     acfactory.send_message(from_account=ac2, to_account=ac1, text="ping2")
 
     log("waiting for incoming message on ac1")
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac1.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "ping2"
 
     log("sending realtime data ac1 -> ac2")
@@ -214,7 +214,9 @@ def test_advertisement_after_chatting(acfactory, path_to_webxdc):
     ac1_ac2_chat = ac1.create_chat(ac2)
     ac1_webxdc_msg = ac1_ac2_chat.send_message(text="WebXDC", file=path_to_webxdc)
     ac2_webxdc_msg = ac2.wait_for_incoming_msg()
-    assert ac2_webxdc_msg.get_snapshot().text == "WebXDC"
+    ac2_webxdc_msg_snapshot = ac2_webxdc_msg.get_snapshot()
+    assert ac2_webxdc_msg_snapshot.text == "WebXDC"
+    ac2_webxdc_msg_snapshot.chat.accept()
 
     ac1_ac2_chat.send_text("Hello!")
     ac2_hello_msg = ac2.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_multidevice.py

@@ -4,6 +4,41 @@ from deltachat_rpc_client import EventType
 from deltachat_rpc_client.const import MessageState
 
 
+def test_bcc_self_delete_server_after_defaults(acfactory):
+    """Test default values for bcc_self and delete_server_after."""
+    ac = acfactory.get_online_account()
+
+    # Initially after getting online
+    # the setting bcc_self is set to 0 because there is only one device
+    # and delete_server_after is "1", meaning immediate deletion.
+    assert ac.get_config("bcc_self") == "0"
+    assert ac.get_config("delete_server_after") == "1"
+
+    # Setup a second device.
+    ac_clone = ac.clone()
+    ac_clone.bring_online()
+
+    # Second device setup
+    # enables bcc_self and changes default delete_server_after.
+    assert ac.get_config("bcc_self") == "1"
+    assert ac.get_config("delete_server_after") == "0"
+
+    assert ac_clone.get_config("bcc_self") == "1"
+    assert ac_clone.get_config("delete_server_after") == "0"
+
+    # Manually disabling bcc_self
+    # also restores the default for delete_server_after.
+    ac.set_config("bcc_self", "0")
+    assert ac.get_config("bcc_self") == "0"
+    assert ac.get_config("delete_server_after") == "1"
+
+    # Cloning the account again enables bcc_self
+    # even though it was manually disabled.
+    ac_clone = ac.clone()
+    assert ac.get_config("bcc_self") == "1"
+    assert ac.get_config("delete_server_after") == "0"
+
+
 def test_one_account_send_bcc_setting(acfactory, log, direct_imap):
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_clone = ac1.clone()

deltachat-rpc-client/tests/test_securejoin.py

@@ -3,6 +3,7 @@ import logging
 import pytest
 
 from deltachat_rpc_client import Chat, EventType, SpecialContactId
+from deltachat_rpc_client.const import ChatType
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
@@ -85,7 +86,7 @@ def test_qr_securejoin(acfactory):
     alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
     assert alice_contact_bob_snapshot.is_verified
 
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Member Me added by {}.".format(alice.get_config("addr"))
 
     # Test that Bob verified Alice's profile.
@@ -109,6 +110,143 @@ def test_qr_securejoin(acfactory):
     fiona.wait_for_securejoin_joiner_success()
 
 
+@pytest.mark.parametrize("all_devices_online", [True, False])
+def test_qr_securejoin_broadcast(acfactory, all_devices_online):
+    alice, bob, fiona = acfactory.get_online_accounts(3)
+
+    alice2 = alice.clone()
+    bob2 = bob.clone()
+
+    if all_devices_online:
+        alice2.start_io()
+        bob2.start_io()
+
+    logging.info("===================== Alice creates a broadcast =====================")
+    alice_chat = alice.create_broadcast("Broadcast channel!")
+    snapshot = alice_chat.get_basic_snapshot()
+    assert not snapshot.is_unpromoted  # Broadcast channels are never unpromoted
+
+    logging.info("===================== Bob joins the broadcast =====================")
+
+    qr_code = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    alice.wait_for_securejoin_inviter_success()
+    bob.wait_for_securejoin_joiner_success()
+    alice_chat.send_text("Hello everyone!")
+
+    def get_broadcast(ac):
+        chat = ac.get_chatlist(query="Broadcast channel!")[0]
+        assert chat.get_basic_snapshot().name == "Broadcast channel!"
+        return chat
+
+    def wait_for_broadcast_messages(ac):
+        snapshot1 = ac.wait_for_incoming_msg().get_snapshot()
+        assert snapshot1.text == "You joined the channel."
+
+        snapshot2 = ac.wait_for_incoming_msg().get_snapshot()
+        assert snapshot2.text == "Hello everyone!"
+
+        chat = get_broadcast(ac)
+        assert snapshot1.chat_id == chat.id
+        assert snapshot2.chat_id == chat.id
+
+    def check_account(ac, contact, inviter_side, please_wait_info_msg=False):
+        # Check that the chat partner is verified.
+        contact_snapshot = contact.get_snapshot()
+        assert contact_snapshot.is_verified
+
+        chat = get_broadcast(ac)
+        chat_msgs = chat.get_messages()
+
+        if please_wait_info_msg:
+            first_msg = chat_msgs.pop(0).get_snapshot()
+            assert first_msg.text == "Establishing guaranteed end-to-end encryption, please wait…"
+            assert first_msg.is_info
+
+        encrypted_msg = chat_msgs[0].get_snapshot()
+        assert encrypted_msg.text == "Messages are end-to-end encrypted."
+        assert encrypted_msg.is_info
+
+        member_added_msg = chat_msgs[1].get_snapshot()
+        if inviter_side:
+            assert member_added_msg.text == f"Member {contact_snapshot.display_name} added."
+        else:
+            assert member_added_msg.text == "You joined the channel."
+        assert member_added_msg.is_info
+
+        hello_msg = chat_msgs[2].get_snapshot()
+        assert hello_msg.text == "Hello everyone!"
+        assert not hello_msg.is_info
+        assert hello_msg.show_padlock
+        assert hello_msg.error is None
+
+        assert len(chat_msgs) == 3
+
+        chat_snapshot = chat.get_full_snapshot()
+        assert chat_snapshot.is_encrypted
+        assert chat_snapshot.name == "Broadcast channel!"
+        if inviter_side:
+            assert chat_snapshot.chat_type == ChatType.OUT_BROADCAST
+        else:
+            assert chat_snapshot.chat_type == ChatType.IN_BROADCAST
+        assert chat_snapshot.can_send == inviter_side
+
+        chat_contacts = chat_snapshot.contact_ids
+        assert contact.id in chat_contacts
+        if inviter_side:
+            assert len(chat_contacts) == 1
+        else:
+            assert len(chat_contacts) == 2
+            assert SpecialContactId.SELF in chat_contacts
+            assert chat_snapshot.self_in_group
+
+    wait_for_broadcast_messages(bob)
+
+    check_account(alice, alice.create_contact(bob), inviter_side=True)
+    check_account(bob, bob.create_contact(alice), inviter_side=False, please_wait_info_msg=True)
+
+    logging.info("===================== Test Alice's second device =====================")
+
+    # Start second Alice device, if it wasn't started already.
+    alice2.start_io()
+
+    while True:
+        msg_id = alice2.wait_for_msgs_changed_event().msg_id
+        if msg_id:
+            snapshot = alice2.get_message_by_id(msg_id).get_snapshot()
+            if snapshot.text == "Hello everyone!":
+                break
+
+    check_account(alice2, alice2.create_contact(bob), inviter_side=True)
+
+    logging.info("===================== Test Bob's second device =====================")
+
+    # Start second Bob device, if it wasn't started already.
+    bob2.start_io()
+    bob2.wait_for_securejoin_joiner_success()
+    wait_for_broadcast_messages(bob2)
+    check_account(bob2, bob2.create_contact(alice), inviter_side=False)
+
+    # The QR code token is synced, so alice2 must be able to handle join requests.
+    logging.info("===================== Fiona joins the group via alice2 =====================")
+    alice.stop_io()
+    fiona.secure_join(qr_code)
+    alice2.wait_for_securejoin_inviter_success()
+    fiona.wait_for_securejoin_joiner_success()
+
+    snapshot = fiona.wait_for_incoming_msg().get_snapshot()
+    assert snapshot.text == "You joined the channel."
+
+    get_broadcast(alice2).get_messages()[2].resend()
+    snapshot = fiona.wait_for_incoming_msg().get_snapshot()
+    assert snapshot.text == "Hello everyone!"
+
+    check_account(fiona, fiona.create_contact(alice), inviter_side=False, please_wait_info_msg=True)
+
+    # For Bob, the channel must not have changed:
+    check_account(bob, bob.create_contact(alice), inviter_side=False, please_wait_info_msg=True)
+
+
 def test_qr_securejoin_contact_request(acfactory) -> None:
     """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
     alice, bob = acfactory.get_online_accounts(2)
@@ -117,7 +255,7 @@ def test_qr_securejoin_contact_request(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Hello!"
     bob_chat_alice = snapshot.chat
     assert bob_chat_alice.get_basic_snapshot().is_contact_request
@@ -161,8 +299,7 @@ def test_qr_readreceipt(acfactory) -> None:
 
     logging.info("Bob and Charlie receive a group")
 
-    bob_msg_id = bob.wait_for_incoming_msg_event().msg_id
-    bob_message = bob.get_message_by_id(bob_msg_id)
+    bob_message = bob.wait_for_incoming_msg()
     bob_snapshot = bob_message.get_snapshot()
     assert bob_snapshot.text == "Hello"
 
@@ -173,8 +310,7 @@ def test_qr_readreceipt(acfactory) -> None:
 
     bob_out_message = bob_snapshot.chat.send_message(text="Hi from Bob!")
 
-    charlie_msg_id = charlie.wait_for_incoming_msg_event().msg_id
-    charlie_message = charlie.get_message_by_id(charlie_msg_id)
+    charlie_message = charlie.wait_for_incoming_msg()
     charlie_snapshot = charlie_message.get_snapshot()
     assert charlie_snapshot.text == "Hi from Bob!"
 
@@ -249,7 +385,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     ac3_contact_ac2 = ac3.create_contact(ac2)
     ac3_chat.remove_contact(ac3_contact_ac2_old)
 
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac1.wait_for_incoming_msg().get_snapshot()
     assert "removed" in snapshot.text
 
     ac3_chat.add_contact(ac3_contact_ac2)
@@ -262,25 +398,26 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     logging.info("ac2 got event message: %s", snapshot.text)
     assert "added" in snapshot.text
 
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac1.wait_for_incoming_msg().get_snapshot()
     assert "added" in snapshot.text
 
     chat = Chat(ac2, chat_id)
     chat.send_text("Works again!")
 
-    msg_id = ac3.wait_for_incoming_msg_event().msg_id
-    message = ac3.get_message_by_id(msg_id)
+    message = ac3.wait_for_incoming_msg()
     snapshot = message.get_snapshot()
     assert snapshot.text == "Works again!"
 
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac1.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Works again!"
 
     ac1_contact_ac2 = ac1.create_contact(ac2)
     ac1_contact_ac3 = ac1.create_contact(ac3)
     ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
-    assert ac1_contact_ac2_snapshot.is_verified
-    assert ac1_contact_ac2_snapshot.verifier_id == ac1_contact_ac3.id
+    # Until we reset verifications and then send the _verified header,
+    # verification is not gossiped here:
+    assert not ac1_contact_ac2_snapshot.is_verified
+    assert ac1_contact_ac2_snapshot.verifier_id != ac1_contact_ac3.id
 
 
 def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
@@ -307,7 +444,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
     # ensure ac1 can write and ac2 receives messages in verified chat
     ch1.send_text("ac1 says hello")
     while 1:
-        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        snapshot = ac2.wait_for_incoming_msg().get_snapshot()
         if snapshot.text == "ac1 says hello":
             break
 
@@ -328,7 +465,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
     # ensure ac2 receives message in VG
     vg.send_text("hello")
     while 1:
-        msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        msg = ac2.wait_for_incoming_msg().get_snapshot()
         if msg.text == "hello":
             break
 
@@ -365,7 +502,7 @@ def test_qr_new_group_unblocked(acfactory):
     ac2.wait_for_incoming_msg_event()
 
     ac1_new_chat.send_text("Hello!")
-    ac2_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    ac2_msg = ac2.wait_for_incoming_msg().get_snapshot()
     assert ac2_msg.text == "Hello!"
     assert ac2_msg.chat.get_basic_snapshot().is_contact_request
 
@@ -390,7 +527,7 @@ def test_aeap_flow_verified(acfactory):
 
     logging.info("receiving first message")
     ac2.wait_for_incoming_msg_event()  # member added message
-    msg_in_1 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    msg_in_1 = ac2.wait_for_incoming_msg().get_snapshot()
     assert msg_in_1.text == msg_out.text
 
     logging.info("changing email account")
@@ -404,7 +541,7 @@ def test_aeap_flow_verified(acfactory):
     msg_out = chat.send_text("changed address").get_snapshot()
 
     logging.info("receiving second message")
-    msg_in_2 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    msg_in_2 = ac2.wait_for_incoming_msg()
     msg_in_2_snapshot = msg_in_2.get_snapshot()
     assert msg_in_2_snapshot.text == msg_out.text
     assert msg_in_2_snapshot.chat.id == msg_in_1.chat.id
@@ -436,13 +573,15 @@ def test_gossip_verification(acfactory) -> None:
     bob_group_chat.add_contact(bob_contact_carol)
     bob_group_chat.send_message(text="Hello Autocrypt group")
 
-    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = carol.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Hello Autocrypt group"
     assert snapshot.show_padlock
 
     # Group propagates verification using Autocrypt-Gossip header.
     carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
-    assert carol_contact_alice_snapshot.is_verified
+    # Until we reset verifications and then send the _verified header,
+    # verification is not gossiped here:
+    assert not carol_contact_alice_snapshot.is_verified
 
     logging.info("Bob creates a Securejoin group")
     bob_group_chat = bob.create_group("Securejoin Group")
@@ -450,13 +589,15 @@ def test_gossip_verification(acfactory) -> None:
     bob_group_chat.add_contact(bob_contact_carol)
     bob_group_chat.send_message(text="Hello Securejoin group")
 
-    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = carol.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Hello Securejoin group"
     assert snapshot.show_padlock
 
     # Securejoin propagates verification.
     carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
-    assert carol_contact_alice_snapshot.is_verified
+    # Until we reset verifications and then send the _verified header,
+    # verification is not gossiped here:
+    assert not carol_contact_alice_snapshot.is_verified
 
 
 def test_securejoin_after_contact_resetup(acfactory) -> None:
@@ -476,7 +617,7 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     ac1.wait_for_securejoin_joiner_success()
 
     # ac1 waits for member added message and creates a QR code.
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac1.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Member Me added by {}.".format(ac3.get_config("addr"))
     ac1_qr_code = snapshot.chat.get_qr_code()
 
@@ -513,7 +654,7 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
 
     # Wait for member added.
     logging.info("ac2 waits for member added message")
-    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = ac2.wait_for_incoming_msg().get_snapshot()
     assert snapshot.is_info
     ac2_chat = snapshot.chat
     assert len(ac2_chat.get_contacts()) == 3
@@ -535,7 +676,7 @@ def test_withdraw_securejoin_qr(acfactory):
 
     alice.clear_all_events()
 
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Member Me added by {}.".format(alice.get_config("addr"))
     bob_chat.leave()
 

deltachat-rpc-client/tests/test_something.py

@@ -11,7 +11,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from deltachat_rpc_client import Contact, EventType, Message, events
-from deltachat_rpc_client.const import ChatType, DownloadState, MessageState
+from deltachat_rpc_client.const import DownloadState, MessageState
 from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS
 from deltachat_rpc_client.rpc import JsonRpcError
 
@@ -338,44 +338,26 @@ def test_receive_imf_failure(acfactory) -> None:
 
     bob.set_config("fail_on_receiving_full_msg", "1")
     alice_chat_bob.send_text("Hello!")
-    event = bob.wait_for_incoming_msg_event()
-    chat_id = event.chat_id
+    event = bob.wait_for_event(EventType.MSGS_CHANGED)
+    assert event.chat_id == bob.get_device_chat().id
     msg_id = event.msg_id
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
-    assert snapshot.chat_id == chat_id
-    assert snapshot.download_state == DownloadState.AVAILABLE
-    assert snapshot.error is not None
-    assert snapshot.show_padlock
+    assert (
+        snapshot.text == "❌ Failed to receive a message:"
+        " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`."
+        " Please report this bug to delta@merlinux.eu or https://support.delta.chat/."
+    )
 
     # The failed message doesn't break the IMAP loop.
     bob.set_config("fail_on_receiving_full_msg", "0")
     alice_chat_bob.send_text("Hello again!")
-    event = bob.wait_for_incoming_msg_event()
-    assert event.chat_id == chat_id
-    msg_id = event.msg_id
-    message1 = bob.get_message_by_id(msg_id)
-    snapshot = message1.get_snapshot()
-    assert snapshot.chat_id == chat_id
+    message = bob.wait_for_incoming_msg()
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Hello again!"
     assert snapshot.download_state == DownloadState.DONE
     assert snapshot.error is None
 
-    # The failed message can be re-downloaded later.
-    bob._rpc.download_full_message(bob.id, message.id)
-    event = bob.wait_for_event(EventType.MSGS_CHANGED)
-    message = bob.get_message_by_id(event.msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.download_state == DownloadState.IN_PROGRESS
-    event = bob.wait_for_event(EventType.MSGS_CHANGED)
-    assert event.chat_id == chat_id
-    msg_id = event.msg_id
-    message = bob.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.chat_id == chat_id
-    assert snapshot.download_state == DownloadState.DONE
-    assert snapshot.error is None
-    assert snapshot.text == "Hello!"
-
 
 def test_selfavatar_sync(acfactory, data, log) -> None:
     alice = acfactory.get_online_account()
@@ -439,10 +421,7 @@ def test_is_bot(acfactory) -> None:
     alice.set_config("bot", "1")
     alice_chat_bob.send_text("Hello!")
 
-    event = bob.wait_for_incoming_msg_event()
-    message = bob.get_message_by_id(event.msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.chat_id == event.chat_id
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Hello!"
     assert snapshot.is_bot
 
@@ -500,22 +479,21 @@ def test_wait_next_messages(acfactory) -> None:
     # There are no old messages and the call returns immediately.
     assert not bot.wait_next_messages()
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
-        # Bot starts waiting for messages.
-        next_messages_task = executor.submit(bot.wait_next_messages)
+    # Bot starts waiting for messages.
+    next_messages_task = bot.wait_next_messages.future()
 
-        alice_contact_bot = alice.create_contact(bot, "Bot")
-        alice_chat_bot = alice_contact_bot.create_chat()
-        alice_chat_bot.send_text("Hello!")
+    alice_contact_bot = alice.create_contact(bot, "Bot")
+    alice_chat_bot = alice_contact_bot.create_chat()
+    alice_chat_bot.send_text("Hello!")
 
-        next_messages = next_messages_task.result()
+    next_messages = next_messages_task()
 
-        if len(next_messages) == E2EE_INFO_MSGS:
-            next_messages += bot.wait_next_messages()
+    if len(next_messages) == E2EE_INFO_MSGS:
+        next_messages += bot.wait_next_messages()
 
-        assert len(next_messages) == 1 + E2EE_INFO_MSGS
-        snapshot = next_messages[0 + E2EE_INFO_MSGS].get_snapshot()
-        assert snapshot.text == "Hello!"
+    assert len(next_messages) == 1 + E2EE_INFO_MSGS
+    snapshot = next_messages[0 + E2EE_INFO_MSGS].get_snapshot()
+    assert snapshot.text == "Hello!"
 
 
 def test_import_export_backup(acfactory, tmp_path) -> None:
@@ -535,7 +513,7 @@ def test_import_export_keys(acfactory, tmp_path) -> None:
     alice_chat_bob = alice.create_chat(bob)
     alice_chat_bob.send_text("Hello Bob!")
 
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Hello Bob!"
 
     # Alice resetups account, but keeps the key.
@@ -547,7 +525,7 @@ def test_import_export_keys(acfactory, tmp_path) -> None:
 
     snapshot.chat.accept()
     snapshot.chat.send_text("Hello Alice!")
-    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = alice.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "Hello Alice!"
     assert snapshot.show_padlock
 
@@ -592,18 +570,13 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
 
     # Alice sends a message to Bob.
     alice_chat_bob.send_text("Hello Bob!")
-    event = bob.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    message = bob.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
 
     # Bob sends a message to Alice.
     bob_chat_alice = snapshot.chat
     bob_chat_alice.accept()
     bob_chat_alice.send_text("Hello Alice!")
-    event = alice.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    message = alice.get_message_by_id(msg_id)
+    message = alice.wait_for_incoming_msg()
     snapshot = message.get_snapshot()
     assert snapshot.show_padlock
 
@@ -613,10 +586,7 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
 
     # Bob sends a message to Alice, it should also be encrypted.
     bob_chat_alice.send_text("Hi Alice!")
-    event = alice.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    message = alice.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
+    snapshot = alice.wait_for_incoming_msg().get_snapshot()
     assert snapshot.show_padlock
 
 
@@ -674,50 +644,6 @@ def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
     assert list(reactions.reactions_by_contact.values())[0] == [react_str]
 
 
-def test_reactions_for_a_reordering_move(acfactory, direct_imap):
-    """When a batch of messages is moved from Inbox to DeltaChat folder with a single MOVE command,
-    their UIDs may be reordered (e.g. Gmail is known for that) which led to that messages were
-    processed by receive_imf in the wrong order, and, particularly, reactions were processed before
-    messages they refer to and thus dropped.
-    """
-    (ac1,) = acfactory.get_online_accounts(1)
-
-    addr, password = acfactory.get_credentials()
-    ac2 = acfactory.get_unconfigured_account()
-    ac2.add_or_update_transport({"addr": addr, "password": password})
-    ac2.set_config("mvbox_move", "1")
-    assert ac2.is_configured()
-
-    ac2.bring_online()
-    chat1 = acfactory.get_accepted_chat(ac1, ac2)
-    ac2.stop_io()
-
-    logging.info("sending message + reaction from ac1 to ac2")
-    msg1 = chat1.send_text("hi")
-    msg1.wait_until_delivered()
-    # It's is sad, but messages must differ in their INTERNALDATEs to be processed in the correct
-    # order by DC, and most (if not all) mail servers provide only seconds precision.
-    time.sleep(1.1)
-    react_str = "\N{THUMBS UP SIGN}"
-    msg1.send_reaction(react_str).wait_until_delivered()
-
-    logging.info("moving messages to ac2's DeltaChat folder in the reverse order")
-    ac2_direct_imap = direct_imap(ac2)
-    ac2_direct_imap.connect()
-    for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()], reverse=True):
-        ac2_direct_imap.conn.move(uid, "DeltaChat")
-
-    logging.info("receiving messages by ac2")
-    ac2.start_io()
-    msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
-    assert msg2.get_snapshot().text == msg1.get_snapshot().text
-    reactions = msg2.get_reactions()
-    contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
-    assert len(contacts) == 1
-    assert contacts[0].get_snapshot().address == ac1.get_config("addr")
-    assert list(reactions.reactions_by_contact.values())[0] == [react_str]
-
-
 @pytest.mark.parametrize("n_accounts", [3, 2])
 def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
     download_limit = 300000
@@ -729,7 +655,7 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
     for account in others:
         chat = account.create_chat(alice)
         chat.send_text("Hello Alice!")
-        assert alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot().text == "Hello Alice!"
+        assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!"
 
         contact = alice.create_contact(account)
         alice_group.add_contact(contact)
@@ -739,7 +665,7 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
     bob.set_config("download_limit", str(download_limit))
 
     alice_group.send_text("hi")
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = bob.wait_for_incoming_msg().get_snapshot()
     assert snapshot.text == "hi"
     bob_group = snapshot.chat
 
@@ -749,7 +675,7 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
     for i in range(10):
         logging.info("Sending message %s", i)
         alice_group.send_file(str(path))
-        snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        snapshot = bob.wait_for_incoming_msg().get_snapshot()
         assert snapshot.download_state == DownloadState.AVAILABLE
         if n_accounts > 2:
             assert snapshot.chat == bob_group
@@ -776,8 +702,8 @@ def test_markseen_contact_request(acfactory):
     alice_chat_bob = alice.create_chat(bob)
     alice_chat_bob.send_text("Hello Bob!")
 
-    message = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id)
-    message2 = bob2.get_message_by_id(bob2.wait_for_incoming_msg_event().msg_id)
+    message = bob.wait_for_incoming_msg()
+    message2 = bob2.wait_for_incoming_msg()
     assert message2.get_snapshot().state == MessageState.IN_FRESH
 
     message.mark_seen()
@@ -799,7 +725,7 @@ def test_read_receipt(acfactory):
     msg = bob.wait_for_incoming_msg()
     msg.mark_seen()
 
-    read_msg = alice.get_message_by_id(alice.wait_for_event(EventType.MSG_READ).msg_id)
+    read_msg = alice.wait_for_msg(EventType.MSG_READ)
     read_receipts = read_msg.get_read_receipts()
     assert len(read_receipts) == 1
     assert read_receipts[0].contact_id == alice_contact_bob.id
@@ -816,7 +742,7 @@ def test_configured_imap_certificate_checks(acfactory):
     alice = acfactory.new_configured_account()
 
     # Certificate checks should be configured (not None)
-    assert "cert_automatic" in alice.get_info().used_account_settings
+    assert "cert_strict" in alice.get_info().used_account_settings
 
     # "cert_old_automatic" is the value old Delta Chat core versions used
     # to mean user entered "imap_certificate_checks=0" (Automatic)
@@ -888,10 +814,12 @@ def test_rename_group(acfactory):
     bob_msg = bob.wait_for_incoming_msg()
     bob_chat = bob_msg.get_snapshot().chat
     assert bob_chat.get_basic_snapshot().name == "Test group"
+    bob.wait_for_event(EventType.CHATLIST_ITEM_CHANGED)
 
     for name in ["Baz", "Foo bar", "Xyzzy"]:
         alice_group.set_name(name)
-        bob.wait_for_incoming_msg_event()
+        bob.wait_for_event(EventType.CHATLIST_ITEM_CHANGED)
+        bob.wait_for_event(EventType.CHATLIST_ITEM_CHANGED)
         assert bob_chat.get_basic_snapshot().name == name
 
 
@@ -903,58 +831,174 @@ def test_get_all_accounts_deadlock(rpc):
         all_accounts()
 
 
-def test_delete_deltachat_folder(acfactory, direct_imap):
-    """Test that DeltaChat folder is recreated if user deletes it manually."""
-    ac1 = acfactory.new_configured_account()
-    ac1.set_config("mvbox_move", "1")
-    ac1.bring_online()
-
-    ac1_direct_imap = direct_imap(ac1)
-    ac1_direct_imap.conn.folder.delete("DeltaChat")
-    assert "DeltaChat" not in ac1_direct_imap.list_folders()
-
-    # Wait until new folder is created and UIDVALIDITY is updated.
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == EventType.INFO and "uid/validity change folder DeltaChat" in event.msg:
-            break
-
-    ac2 = acfactory.get_online_account()
-    ac2.create_chat(ac1).send_text("hello")
-    msg = ac1.wait_for_incoming_msg().get_snapshot()
-    assert msg.text == "hello"
-
-    assert "DeltaChat" in ac1_direct_imap.list_folders()
-
-
-def test_broadcast(acfactory):
+@pytest.mark.parametrize("all_devices_online", [True, False])
+def test_leave_broadcast(acfactory, all_devices_online):
     alice, bob = acfactory.get_online_accounts(2)
 
-    alice_chat = alice.create_broadcast("My great channel")
-    snapshot = alice_chat.get_basic_snapshot()
-    assert snapshot.name == "My great channel"
-    assert snapshot.is_unpromoted
-    assert snapshot.is_encrypted
-    assert snapshot.chat_type == ChatType.OUT_BROADCAST
+    bob2 = bob.clone()
 
-    alice_contact_bob = alice.create_contact(bob, "Bob")
-    alice_chat.add_contact(alice_contact_bob)
+    if all_devices_online:
+        bob2.start_io()
 
-    alice_msg = alice_chat.send_message(text="hello").get_snapshot()
-    assert alice_msg.text == "hello"
-    assert alice_msg.show_padlock
+    logging.info("===================== Alice creates a broadcast =====================")
+    alice_chat = alice.create_broadcast("Broadcast channel!")
 
-    bob_msg = bob.wait_for_incoming_msg().get_snapshot()
-    assert bob_msg.text == "hello"
-    assert bob_msg.show_padlock
-    assert bob_msg.error is None
+    logging.info("===================== Bob joins the broadcast =====================")
+    qr_code = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    alice.wait_for_securejoin_inviter_success()
+    bob.wait_for_securejoin_joiner_success()
 
-    bob_chat = bob.get_chat_by_id(bob_msg.chat_id)
-    bob_chat_snapshot = bob_chat.get_basic_snapshot()
-    assert bob_chat_snapshot.name == "My great channel"
-    assert not bob_chat_snapshot.is_unpromoted
-    assert bob_chat_snapshot.is_encrypted
-    assert bob_chat_snapshot.chat_type == ChatType.IN_BROADCAST
-    assert bob_chat_snapshot.is_contact_request
+    alice_bob_contact = alice.create_contact(bob)
+    alice_contacts = alice_chat.get_contacts()
+    assert len(alice_contacts) == 1  # 1 recipient
+    assert alice_contacts[0].id == alice_bob_contact.id
 
-    assert not bob_chat.can_send()
+    member_added_msg = bob.wait_for_incoming_msg()
+    assert member_added_msg.get_snapshot().text == "You joined the channel."
+
+    def get_broadcast(ac):
+        chat = ac.get_chatlist(query="Broadcast channel!")[0]
+        assert chat.get_basic_snapshot().name == "Broadcast channel!"
+        return chat
+
+    def check_account(ac, contact, inviter_side, please_wait_info_msg=False):
+        chat = get_broadcast(ac)
+        contact_snapshot = contact.get_snapshot()
+        chat_msgs = chat.get_messages()
+
+        if please_wait_info_msg:
+            first_msg = chat_msgs.pop(0).get_snapshot()
+            assert first_msg.text == "Establishing guaranteed end-to-end encryption, please wait…"
+            assert first_msg.is_info
+
+        encrypted_msg = chat_msgs.pop(0).get_snapshot()
+        assert encrypted_msg.text == "Messages are end-to-end encrypted."
+        assert encrypted_msg.is_info
+
+        member_added_msg = chat_msgs.pop(0).get_snapshot()
+        if inviter_side:
+            assert member_added_msg.text == f"Member {contact_snapshot.display_name} added."
+        else:
+            assert member_added_msg.text == "You joined the channel."
+        assert member_added_msg.is_info
+
+        if not inviter_side:
+            leave_msg = chat_msgs.pop(0).get_snapshot()
+            assert leave_msg.text == "You left the channel."
+
+        assert len(chat_msgs) == 0
+
+        chat_snapshot = chat.get_full_snapshot()
+
+        # On Alice's side, SELF is not in the list of contact ids
+        # because OutBroadcast chats never contain SELF in the list.
+        # On Bob's side, SELF is not in the list because he left.
+        if inviter_side:
+            assert len(chat_snapshot.contact_ids) == 0
+        else:
+            assert chat_snapshot.contact_ids == [contact.id]
+
+    logging.info("===================== Bob leaves the broadcast =====================")
+    bob_chat = get_broadcast(bob)
+    assert bob_chat.get_full_snapshot().self_in_group
+    assert len(bob_chat.get_contacts()) == 2  # Alice and Bob
+
+    bob_chat.leave()
+    assert not bob_chat.get_full_snapshot().self_in_group
+    # After Bob left, only Alice will be left in Bob's memberlist
+    assert len(bob_chat.get_contacts()) == 1
+
+    check_account(bob, bob.create_contact(alice), inviter_side=False, please_wait_info_msg=True)
+
+    logging.info("===================== Test Alice's device =====================")
+    while len(alice_chat.get_contacts()) != 0:  # After Bob left, there will be 0 recipients
+        alice.wait_for_event(EventType.CHAT_MODIFIED)
+
+    check_account(alice, alice.create_contact(bob), inviter_side=True)
+
+    logging.info("===================== Test Bob's second device =====================")
+    # Start second Bob device, if it wasn't started already.
+    bob2.start_io()
+
+    member_added_msg = bob2.wait_for_incoming_msg()
+    assert member_added_msg.get_snapshot().text == "You joined the channel."
+
+    bob2_chat = get_broadcast(bob2)
+
+    # After Bob left, only Alice will be left in Bob's memberlist
+    while len(bob2_chat.get_contacts()) != 1:
+        bob2.wait_for_event(EventType.CHAT_MODIFIED)
+
+    check_account(bob2, bob2.create_contact(alice), inviter_side=False)
+
+
+def test_immediate_autodelete(acfactory, direct_imap, log):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    # "1" means delete immediately, while "0" means do not delete
+    ac2.set_config("delete_server_after", "1")
+
+    log.section("ac1: create chat with ac2")
+    chat1 = ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+
+    log.section("ac1: send message to ac2")
+    sent_msg = chat1.send_text("hello")
+
+    msg = ac2.wait_for_incoming_msg()
+    assert msg.get_snapshot().text == "hello"
+
+    log.section("ac2: wait for close/expunge on autodelete")
+    ac2.wait_for_event(EventType.IMAP_MESSAGE_DELETED)
+    while True:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:
+            break
+
+    log.section("ac2: check that message was autodeleted on server")
+    ac2_direct_imap = direct_imap(ac2)
+    assert len(ac2_direct_imap.get_all_messages()) == 0
+
+    log.section("ac2: Mark deleted message as seen and check that read receipt arrives")
+    msg.mark_seen()
+    ev = ac1.wait_for_event(EventType.MSG_READ)
+    assert ev.chat_id == chat1.id
+    assert ev.msg_id == sent_msg.id
+
+
+def test_background_fetch(acfactory, dc):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.stop_io()
+
+    ac1_chat = ac1.create_chat(ac2)
+
+    ac2_chat = ac2.create_chat(ac1)
+    ac2_chat.send_text("Hello!")
+
+    while True:
+        dc.background_fetch(300)
+        messages = ac1_chat.get_messages()
+        snapshot = messages[-1].get_snapshot()
+        if snapshot.text == "Hello!":
+            break
+
+    # Stopping background fetch immediately after starting
+    # does not result in any errors.
+    background_fetch_future = dc.background_fetch.future(300)
+    dc.stop_background_fetch()
+    background_fetch_future()
+
+    # Starting background fetch with zero timeout is ok,
+    # it should terminate immediately.
+    dc.background_fetch(0)
+
+    # Background fetch can still be used to send and receive messages.
+    ac2_chat.send_text("Hello again!")
+
+    while True:
+        dc.background_fetch(300)
+        messages = ac1_chat.get_messages()
+        snapshot = messages[-1].get_snapshot()
+        if snapshot.text == "Hello again!":
+            break

deltachat-rpc-client/tests/test_vcard.py

@@ -1,6 +1,7 @@
 def test_vcard(acfactory) -> None:
     alice, bob, fiona = acfactory.get_online_accounts(3)
 
+    bob.create_chat(alice)
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_contact_charlie = alice.create_contact("charlie@example.org", "Charlie")
     alice_contact_charlie_snapshot = alice_contact_charlie.get_snapshot()

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "2.22.0"
+version = "2.27.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "2.22.0"
+  "version": "2.27.0"
 }

deny.toml

@@ -7,9 +7,6 @@ ignore = [
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
 
-    # Archived repository
-    "RUSTSEC-2023-0089",
-
     # Unmaintained instant
     "RUSTSEC-2024-0384",
 
@@ -29,20 +26,21 @@ skip = [
      { name = "derive_more", version = "1.0.0" },
      { name = "event-listener", version = "2.5.3" },
      { name = "getrandom", version = "0.2.12" },
+     { name = "hashbrown", version = "0.14.5" },
      { name = "heck", version = "0.4.1" },
      { name = "http", version = "0.2.12" },
      { name = "linux-raw-sys", version = "0.4.14" },
-     { name = "netdev", version = "0.36.0" },
-     { name = "netlink-packet-route", version = "0.22.0" },
+     { name = "lru", version = "0.12.3" },
+     { name = "netlink-packet-route", version = "0.17.1" },
      { name = "nom", version = "7.1.3" },
      { name = "rand_chacha", version = "0.3.1" },
      { name = "rand_core", version = "0.6.4" },
      { name = "rand", version = "0.8.5" },
      { name = "rustix", version = "0.38.44" },
-     { name = "rustls-webpki", version = "0.102.8" },
      { name = "serdect", version = "0.2.0" },
-     { name = "socket2", version = "0.5.9" },
      { name = "spin", version = "0.9.8" },
+     { name = "strum_macros", version = "0.26.2" },
+     { name = "strum", version = "0.26.2" },
      { name = "syn", version = "1.0.109" },
      { name = "thiserror-impl", version = "1.0.69" },
      { name = "thiserror", version = "1.0.69" },
@@ -57,7 +55,6 @@ skip = [
      { name = "windows_i686_msvc" },
      { name = "windows-implement" },
      { name = "windows-interface" },
-     { name = "windows-link" },
      { name = "windows-result" },
      { name = "windows-strings" },
      { name = "windows-sys" },

flake.nix

@@ -34,7 +34,6 @@
             ./Cargo.lock
             ./Cargo.toml
             ./CMakeLists.txt
-            ./CONTRIBUTING.md
             ./deltachat_derive
             ./deltachat-contact-tools
             ./deltachat-ffi

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "2.22.0"
+version = "2.27.0"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.8"

python/tests/test_0_complex_or_slow.py

@@ -116,7 +116,6 @@ class TestGroupStressTests:
 
 def test_qr_verified_group_and_chatting(acfactory, lp):
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-    ac1_addr = ac1.get_self_contact().addr
     lp.sec("ac1: create verified-group QR, ac2 scans and joins")
     chat1 = ac1.create_group_chat("hello")
     qr = chat1.get_join_qr()
@@ -174,7 +173,9 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
     for ac2_contact in chat2.get_contacts():
         if ac2_contact == ac2_ac1_contact or ac2_contact.id == dc.const.DC_CONTACT_ID_SELF:
             continue
-        assert ac2.get_self_contact().get_verifier(ac2_contact).addr == ac1_addr
+        # Until we reset verifications and then send the _verified header,
+        # verification is not gossiped here:
+        assert ac2.get_self_contact().get_verifier(ac2_contact) is None
 
     lp.sec("ac2: send message and let ac3 read it")
     chat2.send_text("hi")
@@ -409,7 +410,9 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     assert msg_in.text == "hi2"
     assert msg_in.chat == chat2_offl
     assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
-    assert ac2_offl_ac1_contact.is_verified()
+    # Until we reset verifications and then send the _verified header,
+    # verification is not gossiped here:
+    assert not ac2_offl_ac1_contact.is_verified()
 
 
 def test_deleted_msgs_dont_reappear(acfactory):

python/tests/test_1_online.py

@@ -5,7 +5,7 @@ import base64
 from datetime import datetime, timezone
 
 import pytest
-from imap_tools import AND, U
+from imap_tools import AND
 
 import deltachat as dc
 from deltachat import account_hookimpl, Message
@@ -269,112 +269,6 @@ def test_enable_mvbox_move(acfactory, lp):
     assert ac2._evtracker.wait_next_incoming_message().text == "message1"
 
 
-def test_mvbox_thread_and_trash(acfactory, lp):
-    lp.sec("ac1: start with mvbox thread")
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=True)
-
-    lp.sec("ac2: start without a mvbox thread")
-    ac2 = acfactory.new_online_configuring_account(mvbox_move=False)
-
-    lp.sec("ac2 and ac1: waiting for configuration")
-    acfactory.bring_accounts_online()
-
-    lp.sec("ac1: create trash")
-    ac1.direct_imap.create_folder("Trash")
-    ac1.set_config("scan_all_folders_debounce_secs", "0")
-    ac1.stop_io()
-    ac1.start_io()
-
-    lp.sec("ac1: send message and wait for ac2 to receive it")
-    acfactory.get_accepted_chat(ac1, ac2).send_text("message1")
-    assert ac2._evtracker.wait_next_incoming_message().text == "message1"
-
-    assert ac1.get_config("configured_mvbox_folder") == "DeltaChat"
-    while ac1.get_config("configured_trash_folder") != "Trash":
-        ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-
-def test_move_works(acfactory):
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(mvbox_move=True)
-    acfactory.bring_accounts_online()
-    chat = acfactory.get_accepted_chat(ac1, ac2)
-    chat.send_text("message1")
-
-    # Message is moved to the movebox
-    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-
-    # Message is downloaded
-    ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG")
-    assert ev.data2 > dc.const.DC_CHAT_ID_LAST_SPECIAL
-
-
-def test_move_avoids_loop(acfactory):
-    """Test that the message is only moved from INBOX to DeltaChat.
-
-    This is to avoid busy loop if moved message reappears in the Inbox
-    or some scanned folder later.
-    For example, this happens on servers that alias `INBOX.DeltaChat` to `DeltaChat` folder,
-    so the message moved to `DeltaChat` appears as a new message in the `INBOX.DeltaChat` folder.
-    We do not want to move this message from `INBOX.DeltaChat` to `DeltaChat` again.
-    """
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(mvbox_move=True)
-    acfactory.bring_accounts_online()
-
-    # Create INBOX.DeltaChat folder and make sure
-    # it is detected by full folder scan.
-    ac2.direct_imap.create_folder("INBOX.DeltaChat")
-    ac2.stop_io()
-    ac2.start_io()
-    ac2._evtracker.get_info_contains("Found folders:")  #  Wait until the end of folder scan.
-
-    ac1_chat = acfactory.get_accepted_chat(ac1, ac2)
-    ac1_chat.send_text("Message 1")
-
-    # Message is moved to the DeltaChat folder and downloaded.
-    ac2_msg1 = ac2._evtracker.wait_next_incoming_message()
-    assert ac2_msg1.text == "Message 1"
-
-    # Move the message to the INBOX.DeltaChat again.
-    # We assume that test server uses "." as the delimiter.
-    ac2.direct_imap.select_folder("DeltaChat")
-    ac2.direct_imap.conn.move(["*"], "INBOX.DeltaChat")
-
-    ac1_chat.send_text("Message 2")
-    ac2_msg2 = ac2._evtracker.wait_next_incoming_message()
-    assert ac2_msg2.text == "Message 2"
-
-    # Stop and start I/O to trigger folder scan.
-    ac2.stop_io()
-    ac2.start_io()
-    ac2._evtracker.get_info_contains("Found folders:")  #  Wait until the end of folder scan.
-
-    # Check that Message 1 is still in the INBOX.DeltaChat folder
-    # and Message 2 is in the DeltaChat folder.
-    ac2.direct_imap.select_folder("INBOX")
-    assert len(ac2.direct_imap.get_all_messages()) == 0
-    ac2.direct_imap.select_folder("DeltaChat")
-    assert len(ac2.direct_imap.get_all_messages()) == 1
-    ac2.direct_imap.select_folder("INBOX.DeltaChat")
-    assert len(ac2.direct_imap.get_all_messages()) == 1
-
-
-def test_move_works_on_self_sent(acfactory):
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=True)
-    ac2 = acfactory.new_online_configuring_account()
-    acfactory.bring_accounts_online()
-    ac1.set_config("bcc_self", "1")
-
-    chat = acfactory.get_accepted_chat(ac1, ac2)
-    chat.send_text("message1")
-    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-    chat.send_text("message2")
-    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-    chat.send_text("message3")
-    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-
-
 def test_move_sync_msgs(acfactory):
     ac1 = acfactory.new_online_configuring_account(bcc_self=True, sync_msgs=True, fix_is_chatmail=True)
     acfactory.bring_accounts_online()
@@ -460,7 +354,7 @@ def test_forward_own_message(acfactory, lp):
 
 def test_resend_message(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
-    chat1 = ac1.create_chat(ac2)
+    chat1 = acfactory.get_accepted_chat(ac1, ac2)
 
     lp.sec("ac1: send message to ac2")
     chat1.send_text("message")
@@ -607,39 +501,6 @@ def test_send_and_receive_message_markseen(acfactory, lp):
         pass  # mark_seen_messages() has generated events before it returns
 
 
-def test_moved_markseen(acfactory):
-    """Test that message already moved to DeltaChat folder is marked as seen."""
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(mvbox_move=True)
-    acfactory.bring_accounts_online()
-
-    ac2.stop_io()
-    with ac2.direct_imap.idle() as idle2:
-        ac1.create_chat(ac2).send_text("Hello!")
-        idle2.wait_for_new_message()
-
-    # Emulate moving of the message to DeltaChat folder by Sieve rule.
-    ac2.direct_imap.conn.move(["*"], "DeltaChat")
-    ac2.direct_imap.select_folder("DeltaChat")
-
-    with ac2.direct_imap.idle() as idle2:
-        ac2.start_io()
-
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-        msg = ac2.get_message_by_id(ev.data2)
-        assert msg.text == "Messages are end-to-end encrypted."
-
-        ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-        msg = ac2.get_message_by_id(ev.data2)
-
-        # Accept the contact request.
-        msg.chat.accept()
-        ac2.mark_seen_messages([msg])
-        uid = idle2.wait_for_seen()
-
-    assert len(list(ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*"))))) == 1
-
-
 def test_message_override_sender_name(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1.set_config("displayname", "ac1-default-displayname")
@@ -674,36 +535,6 @@ def test_message_override_sender_name(acfactory, lp):
     assert not msg2.override_sender_name
 
 
-@pytest.mark.parametrize("mvbox_move", [True, False])
-def test_markseen_message_and_mdn(acfactory, mvbox_move):
-    # Please only change this test if you are very sure that it will still catch the issues it catches now.
-    # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=mvbox_move)
-    ac2 = acfactory.new_online_configuring_account(mvbox_move=mvbox_move)
-    acfactory.bring_accounts_online()
-    # Do not send BCC to self, we only want to test MDN on ac1.
-    ac1.set_config("bcc_self", "0")
-
-    acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
-    msg = ac2._evtracker.wait_next_incoming_message()
-
-    ac2.mark_seen_messages([msg])
-
-    folder = "mvbox" if mvbox_move else "inbox"
-    for ac in [ac1, ac2]:
-        if mvbox_move:
-            ac._evtracker.get_info_contains("Marked messages [0-9]+ in folder DeltaChat as seen.")
-        else:
-            ac._evtracker.get_info_contains("Marked messages [0-9]+ in folder INBOX as seen.")
-    ac1.direct_imap.select_config_folder(folder)
-    ac2.direct_imap.select_config_folder(folder)
-
-    # Check that the mdn is marked as seen
-    assert len(list(ac1.direct_imap.conn.fetch(AND(seen=True)))) == 1
-    # Check original message is marked as seen
-    assert len(list(ac2.direct_imap.conn.fetch(AND(seen=True)))) == 1
-
-
 def test_reply_privately(acfactory):
     ac1, ac2 = acfactory.get_online_accounts(2)
 
@@ -853,140 +684,6 @@ def test_no_draft_if_cant_send(acfactory):
     assert device_chat.get_draft() is None
 
 
-def test_dont_show_emails(acfactory, lp):
-    """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
-    So: If it's outgoing AND there is no Received header, then ignore the email.
-
-    If the draft email is sent out and received later (i.e. it's in "Inbox"), it must be shown.
-
-    Also, test that unknown emails in the Spam folder are not shown."""
-    ac1 = acfactory.new_online_configuring_account()
-    ac1.set_config("show_emails", "2")
-    ac1.create_contact("alice@example.org").create_chat()
-
-    acfactory.wait_configured(ac1)
-    ac1.direct_imap.create_folder("Drafts")
-    ac1.direct_imap.create_folder("Spam")
-    ac1.direct_imap.create_folder("Junk")
-
-    acfactory.bring_accounts_online()
-    ac1.stop_io()
-
-    ac1.direct_imap.append(
-        "Drafts",
-        """
-        From: ac1 <{}>
-        Subject: subj
-        To: alice@example.org
-        Message-ID: <aepiors@example.org>
-        Content-Type: text/plain; charset=utf-8
-
-        message in Drafts received later
-    """.format(
-            ac1.get_config("configured_addr"),
-        ),
-    )
-    ac1.direct_imap.append(
-        "Spam",
-        """
-        From: unknown.address@junk.org
-        Subject: subj
-        To: {}
-        Message-ID: <spam.message@junk.org>
-        Content-Type: text/plain; charset=utf-8
-
-        Unknown message in Spam
-    """.format(
-            ac1.get_config("configured_addr"),
-        ),
-    )
-    ac1.direct_imap.append(
-        "Spam",
-        """
-        From: unknown.address@junk.org, unkwnown.add@junk.org
-        Subject: subj
-        To: {}
-        Message-ID: <spam.message2@junk.org>
-        Content-Type: text/plain; charset=utf-8
-
-        Unknown & malformed message in Spam
-    """.format(
-            ac1.get_config("configured_addr"),
-        ),
-    )
-    ac1.direct_imap.append(
-        "Spam",
-        """
-        From: delta<address: inbox@nhroy.com>
-        Subject: subj
-        To: {}
-        Message-ID: <spam.message99@junk.org>
-        Content-Type: text/plain; charset=utf-8
-
-        Unknown & malformed message in Spam
-    """.format(
-            ac1.get_config("configured_addr"),
-        ),
-    )
-    ac1.direct_imap.append(
-        "Spam",
-        """
-        From: alice@example.org
-        Subject: subj
-        To: {}
-        Message-ID: <spam.message3@junk.org>
-        Content-Type: text/plain; charset=utf-8
-
-        Actually interesting message in Spam
-    """.format(
-            ac1.get_config("configured_addr"),
-        ),
-    )
-    ac1.direct_imap.append(
-        "Junk",
-        """
-        From: unknown.address@junk.org
-        Subject: subj
-        To: {}
-        Message-ID: <spam.message@junk.org>
-        Content-Type: text/plain; charset=utf-8
-
-        Unknown message in Junk
-    """.format(
-            ac1.get_config("configured_addr"),
-        ),
-    )
-
-    ac1.set_config("scan_all_folders_debounce_secs", "0")
-    lp.sec("All prepared, now let DC find the message")
-    ac1.start_io()
-
-    # Wait until each folder was scanned, this is necessary for this test to test what it should test:
-    ac1._evtracker.wait_idle_inbox_ready()
-
-    fresh_msgs = list(ac1.get_fresh_messages())
-    msg = fresh_msgs[0]
-    chat_msgs = msg.chat.get_messages()
-    assert len(chat_msgs) == 1
-    assert any(msg.text == "subj – Actually interesting message in Spam" for msg in chat_msgs)
-
-    assert not any("unknown.address" in c.get_name() for c in ac1.get_chats())
-    ac1.direct_imap.select_folder("Spam")
-    assert ac1.direct_imap.get_uid_by_message_id("spam.message@junk.org")
-
-    ac1.stop_io()
-    lp.sec("'Send out' the draft by moving it to Inbox, and wait for DC to display it this time")
-    ac1.direct_imap.select_folder("Drafts")
-    uid = ac1.direct_imap.get_uid_by_message_id("aepiors@example.org")
-    ac1.direct_imap.conn.move(uid, "Inbox")
-
-    ac1.start_io()
-    msg2 = ac1._evtracker.wait_next_messages_changed()
-
-    assert msg2.text == "subj – message in Drafts received later"
-    assert len(msg.chat.get_messages()) == 2
-
-
 def test_bot(acfactory, lp):
     """Test that bot messages can be identified as such"""
     ac1, ac2 = acfactory.get_online_accounts(2)
@@ -1238,8 +935,11 @@ def test_qr_email_capitalization(acfactory, lp):
 
     # ac1 should see both ac3 and ac2 as verified.
     assert len(ac1_chat.get_contacts()) == 3
+    # Until we reset verifications and then send the _verified header,
+    # the verification of ac2 is not gossiped here:
     for contact in ac1_chat.get_contacts():
-        assert contact.is_verified()
+        is_ac2 = contact.addr == ac2.get_config("addr")
+        assert contact.is_verified() != is_ac2
 
 
 def test_set_get_contact_avatar(acfactory, data, lp):
@@ -1506,9 +1206,15 @@ def test_send_receive_locations(acfactory, lp):
     assert locations[0].latitude == 2.0
     assert locations[0].longitude == 3.0
     assert locations[0].accuracy == 0.5
-    assert locations[0].timestamp > now
     assert locations[0].marker is None
 
+    # Make sure the timestamp is not in the past.
+    # Note that location timestamp has only 1 second precision,
+    # while `now` has a fractional part, so we have to truncate it
+    # first, otherwise `now` may appear to be in the future
+    # even though it is the same second.
+    assert int(locations[0].timestamp.timestamp()) >= int(now.timestamp())
+
     contact = ac2.create_contact(ac1)
     locations2 = chat2.get_locations(contact=contact)
     assert len(locations2) == 1
@@ -1519,38 +1225,6 @@ def test_send_receive_locations(acfactory, lp):
     assert not locations3
 
 
-def test_immediate_autodelete(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account()
-    acfactory.bring_accounts_online()
-
-    # "1" means delete immediately, while "0" means do not delete
-    ac2.set_config("delete_server_after", "1")
-
-    lp.sec("ac1: create chat with ac2")
-    chat1 = ac1.create_chat(ac2)
-    ac2.create_chat(ac1)
-
-    lp.sec("ac1: send message to ac2")
-    sent_msg = chat1.send_text("hello")
-
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "hello"
-
-    lp.sec("ac2: wait for close/expunge on autodelete")
-    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
-    ac2._evtracker.get_info_contains("Close/expunge succeeded.")
-
-    lp.sec("ac2: check that message was autodeleted on server")
-    assert len(ac2.direct_imap.get_all_messages()) == 0
-
-    lp.sec("ac2: Mark deleted message as seen and check that read receipt arrives")
-    msg.mark_seen()
-    ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ")
-    assert ev.data1 == chat1.id
-    assert ev.data2 == sent_msg.id
-
-
 def test_delete_multiple_messages(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     chat12 = acfactory.get_accepted_chat(ac1, ac2)
@@ -1583,55 +1257,6 @@ def test_delete_multiple_messages(acfactory, lp):
             break
 
 
-def test_trash_multiple_messages(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac2.stop_io()
-
-    # Create the Trash folder on IMAP server and configure deletion to it. There was a bug that if
-    # Trash wasn't configured initially, it can't be configured later, let's check this.
-    lp.sec("Creating trash folder")
-    ac2.direct_imap.create_folder("Trash")
-    ac2.set_config("delete_to_trash", "1")
-
-    lp.sec("Check that Trash can be configured initially as well")
-    ac3 = acfactory.new_online_configuring_account(cloned_from=ac2)
-    acfactory.bring_accounts_online()
-    assert ac3.get_config("configured_trash_folder")
-    ac3.stop_io()
-
-    ac2.start_io()
-    chat12 = acfactory.get_accepted_chat(ac1, ac2)
-
-    lp.sec("ac1: sending 3 messages")
-    texts = ["first", "second", "third"]
-    for text in texts:
-        chat12.send_text(text)
-
-    lp.sec("ac2: waiting for all messages on the other side")
-    to_delete = []
-    for text in texts:
-        msg = ac2._evtracker.wait_next_incoming_message()
-        assert msg.text in texts
-        if text != "second":
-            to_delete.append(msg)
-    # ac2 has received some messages, this is impossible w/o the trash folder configured, let's
-    # check the configuration.
-    assert ac2.get_config("configured_trash_folder") == "Trash"
-
-    lp.sec("ac2: deleting all messages except second")
-    assert len(to_delete) == len(texts) - 1
-    ac2.delete_messages(to_delete)
-
-    lp.sec("ac2: test that only one message is left")
-    while 1:
-        ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac2.direct_imap.select_config_folder("inbox")
-        nr_msgs = len(ac2.direct_imap.get_all_messages())
-        assert nr_msgs > 0
-        if nr_msgs == 1:
-            break
-
-
 def test_configure_error_msgs_wrong_pw(acfactory):
     (ac1,) = acfactory.get_online_accounts(1)
 
@@ -1755,71 +1380,6 @@ def test_group_quote(acfactory, lp):
     assert received_reply.quote.id == out_msg.id
 
 
-@pytest.mark.parametrize(
-    ("folder", "move", "expected_destination"),
-    [
-        (
-            "xyz",
-            False,
-            "xyz",
-        ),  # Test that emails aren't found in a random folder
-        (
-            "xyz",
-            True,
-            "xyz",
-        ),  # ...emails are found in a random folder and downloaded without moving
-        (
-            "Spam",
-            False,
-            "INBOX",
-        ),  # ...emails are moved from the spam folder to the Inbox
-    ],
-)
-# Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
-# the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
-def test_scan_folders(acfactory, lp, folder, move, expected_destination):
-    """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
-    variant = folder + "-" + str(move) + "-" + expected_destination
-    lp.sec("Testing variant " + variant)
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=move)
-    ac2 = acfactory.new_online_configuring_account()
-
-    acfactory.wait_configured(ac1)
-    ac1.direct_imap.create_folder(folder)
-
-    # Wait until each folder was selected once and we are IDLEing:
-    acfactory.bring_accounts_online()
-    ac1.stop_io()
-    assert folder in ac1.direct_imap.list_folders()
-
-    lp.sec("Send a message to from ac2 to ac1 and manually move it to `folder`")
-    ac1.direct_imap.select_config_folder("inbox")
-    with ac1.direct_imap.idle() as idle1:
-        acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
-        idle1.wait_for_new_message()
-    ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
-
-    lp.sec("start_io() and see if DeltaChat finds the message (" + variant + ")")
-    ac1.set_config("scan_all_folders_debounce_secs", "0")
-    ac1.start_io()
-    chat = ac1.create_chat(ac2)
-    n_msgs = 1  # "Messages are end-to-end encrypted."
-    if folder == "Spam":
-        msg = ac1._evtracker.wait_next_incoming_message()
-        assert msg.text == "hello"
-        n_msgs += 1
-    else:
-        ac1._evtracker.wait_idle_inbox_ready()
-    assert len(chat.get_messages()) == n_msgs
-
-    # The message has reached its destination.
-    ac1.direct_imap.select_folder(expected_destination)
-    assert len(ac1.direct_imap.get_all_messages()) == 1
-    if folder != expected_destination:
-        ac1.direct_imap.select_folder(folder)
-        assert len(ac1.direct_imap.get_all_messages()) == 0
-
-
 def test_archived_muted_chat(acfactory, lp):
     """If an archived and muted chat receives a new message, DC_EVENT_MSGS_CHANGED for
     DC_CHAT_ID_ARCHIVED_LINK must be generated if the chat had only seen messages previously.

python/tox.ini

@@ -23,7 +23,6 @@ deps =
     pytest
     pytest-timeout
     pytest-xdist
-    pdbpp
     requests
 # urllib3 2.0 does not work in manylinux2014 containers.
 # https://github.com/deltachat/deltachat-core-rust/issues/4788

release-date.in

@@ -1 +1 @@
-2025-10-17
+2025-11-16

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.90.0
+RUST_VERSION=1.91.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

src/accounts.rs

@@ -3,8 +3,12 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::future::Future;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use anyhow::{Context as _, Result, bail, ensure};
+use async_channel::{self, Receiver, Sender};
+use futures::FutureExt as _;
+use futures_lite::FutureExt as _;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
@@ -18,7 +22,7 @@ use tokio::time::{Duration, sleep};
 
 use crate::context::{Context, ContextBuilder};
 use crate::events::{Event, EventEmitter, EventType, Events};
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::push::PushSubscriber;
 use crate::stock_str::StockStrings;
 
@@ -41,6 +45,13 @@ pub struct Accounts {
 
     /// Push notification subscriber shared between accounts.
     push_subscriber: PushSubscriber,
+
+    /// Channel sender to cancel ongoing background_fetch().
+    ///
+    /// If background_fetch() is not running, this is `None`.
+    /// New background_fetch() should not be started if this
+    /// contains `Some`.
+    background_fetch_interrupt_sender: Arc<parking_lot::Mutex<Option<Sender<()>>>>,
 }
 
 impl Accounts {
@@ -96,6 +107,7 @@ impl Accounts {
             events,
             stockstrings,
             push_subscriber,
+            background_fetch_interrupt_sender: Default::default(),
         })
     }
 
@@ -352,6 +364,11 @@ impl Accounts {
     ///
     /// This is an auxiliary function and not part of public API.
     /// Use [Accounts::background_fetch] instead.
+    ///
+    /// This function is cancellation-safe.
+    /// It is intended to be cancellable,
+    /// either because of the timeout or because background
+    /// fetch was explicitly cancelled.
     async fn background_fetch_no_timeout(accounts: Vec<Context>, events: Events) {
         let n_accounts = accounts.len();
         events.emit(Event {
@@ -378,14 +395,33 @@ impl Accounts {
     }
 
     /// Auxiliary function for [Accounts::background_fetch].
+    ///
+    /// Runs `background_fetch` until it finishes
+    /// or until the timeout.
+    ///
+    /// Produces `AccountsBackgroundFetchDone` event in every case
+    /// and clears [`Self::background_fetch_interrupt_sender`]
+    /// so a new background fetch can be started.
+    ///
+    /// This function is not cancellation-safe.
+    /// Cancelling it before it returns may result
+    /// in not being able to run any new background fetch
+    /// if interrupt sender was not cleared.
     async fn background_fetch_with_timeout(
         accounts: Vec<Context>,
         events: Events,
         timeout: std::time::Duration,
+        interrupt_sender: Arc<parking_lot::Mutex<Option<Sender<()>>>>,
+        interrupt_receiver: Option<Receiver<()>>,
     ) {
+        let Some(interrupt_receiver) = interrupt_receiver else {
+            // Nothing to do if we got no interrupt receiver.
+            return;
+        };
         if let Err(_err) = tokio::time::timeout(
             timeout,
-            Self::background_fetch_no_timeout(accounts, events.clone()),
+            Self::background_fetch_no_timeout(accounts, events.clone())
+                .race(interrupt_receiver.recv().map(|_| ())),
         )
         .await
         {
@@ -398,10 +434,16 @@ impl Accounts {
             id: 0,
             typ: EventType::AccountsBackgroundFetchDone,
         });
+        (*interrupt_sender.lock()) = None;
     }
 
     /// Performs a background fetch for all accounts in parallel with a timeout.
     ///
+    /// Ongoing background fetch can also be cancelled manually
+    /// by calling `stop_background_fetch()`, in which case it will
+    /// return immediately even before the timeout expiration
+    /// or finishing fetching.
+    ///
     /// The `AccountsBackgroundFetchDone` event is emitted at the end,
     /// process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
@@ -414,7 +456,39 @@ impl Accounts {
     ) -> impl Future<Output = ()> + use<> {
         let accounts: Vec<Context> = self.accounts.values().cloned().collect();
         let events = self.events.clone();
-        Self::background_fetch_with_timeout(accounts, events, timeout)
+        let (sender, receiver) = async_channel::bounded(1);
+        let receiver = {
+            let mut lock = self.background_fetch_interrupt_sender.lock();
+            if (*lock).is_some() {
+                // Another background_fetch() is already running,
+                // return immeidately.
+                None
+            } else {
+                *lock = Some(sender);
+                Some(receiver)
+            }
+        };
+        Self::background_fetch_with_timeout(
+            accounts,
+            events,
+            timeout,
+            self.background_fetch_interrupt_sender.clone(),
+            receiver,
+        )
+    }
+
+    /// Interrupts ongoing background_fetch() call,
+    /// making it return early.
+    ///
+    /// This method allows to cancel background_fetch() early,
+    /// e.g. on Android, when `Service.onTimeout` is called.
+    ///
+    /// If there is no ongoing background_fetch(), does nothing.
+    pub fn stop_background_fetch(&self) {
+        let mut lock = self.background_fetch_interrupt_sender.lock();
+        if let Some(sender) = lock.take() {
+            sender.try_send(()).ok();
+        }
     }
 
     /// Emits a single event.

src/aheader.rs

@@ -61,9 +61,11 @@ impl fmt::Display for Aheader {
         if self.prefer_encrypt == EncryptPreference::Mutual {
             write!(fmt, " prefer-encrypt=mutual;")?;
         }
-        if self.verified {
-            write!(fmt, " _verified=1;")?;
-        }
+        // TODO After we reset all existing verifications,
+        // we want to start sending the _verified attribute
+        // if self.verified {
+        //     write!(fmt, " _verified=1;")?;
+        // }
 
         // adds a whitespace every 78 characters, this allows
         // email crate to wrap the lines according to RFC 5322
@@ -282,8 +284,9 @@ mod tests {
             .contains("test@example.com")
         );
 
+        // We don't send the _verified header yet:
         assert!(
-            format!(
+            !format!(
                 "{}",
                 Aheader {
                     addr: "test@example.com".to_string(),

src/blob.rs

@@ -20,7 +20,7 @@ use crate::config::Config;
 use crate::constants::{self, MediaQuality};
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::{LogExt, error, info, warn};
+use crate::log::{LogExt, warn};
 use crate::message::Viewtype;
 use crate::tools::sanitize_filename;
 
@@ -234,8 +234,13 @@ impl<'a> BlobObject<'a> {
     /// If `data` represents an image of known format, this adds the corresponding extension.
     ///
     /// Even though this function is not async, it's OK to call it from an async context.
-    pub(crate) fn store_from_base64(context: &Context, data: &str) -> Result<String> {
-        let buf = base64::engine::general_purpose::STANDARD.decode(data)?;
+    ///
+    /// Returns an error if there is an I/O problem,
+    /// but in case of a failure to decode base64 returns `Ok(None)`.
+    pub(crate) fn store_from_base64(context: &Context, data: &str) -> Result<Option<String>> {
+        let Ok(buf) = base64::engine::general_purpose::STANDARD.decode(data) else {
+            return Ok(None);
+        };
         let name = if let Ok(format) = image::guess_format(&buf) {
             if let Some(ext) = format.extensions_str().first() {
                 format!("file.{ext}")
@@ -246,7 +251,7 @@ impl<'a> BlobObject<'a> {
             String::new()
         };
         let blob = BlobObject::create_and_deduplicate_from_bytes(context, &buf, &name)?;
-        Ok(blob.as_name().to_string())
+        Ok(Some(blob.as_name().to_string()))
     }
 
     /// Recode image to avatar size.

src/calls.rs

@@ -9,7 +9,7 @@ use crate::contact::ContactId;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::message::{self, Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::net::dns::lookup_host_with_cache;

src/chat.rs

@@ -32,7 +32,7 @@ use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers};
 use crate::events::EventType;
 use crate::key::self_fingerprint;
 use crate::location;
-use crate::log::{LogExt, error, info, warn};
+use crate::log::{LogExt, warn};
 use crate::logged_debug_assert;
 use crate::message::{self, Message, MessageState, MsgId, Viewtype};
 use crate::mimefactory::MimeFactory;
@@ -43,13 +43,15 @@ use crate::smtp::send_msg_to_smtp;
 use crate::stock_str;
 use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
-    IsNoneOrEmpty, SystemTime, buf_compress, create_id, create_outgoing_rfc724_mid,
-    create_smeared_timestamp, create_smeared_timestamps, get_abs_path, gm2local_offset,
-    smeared_time, time, truncate_msg_text,
+    IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id,
+    create_outgoing_rfc724_mid, create_smeared_timestamp, create_smeared_timestamps, get_abs_path,
+    gm2local_offset, smeared_time, time, truncate_msg_text,
 };
 use crate::webxdc::StatusUpdateSerial;
 use crate::{chatlist_events, imap};
 
+pub(crate) const PARAM_BROADCAST_SECRET: Param = Param::Arg3;
+
 /// An chat item, such as a message or a marker.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum ChatItem {
@@ -304,7 +306,7 @@ impl ChatId {
 
         info!(
             context,
-            "Created group/mailinglist '{}' grpid={} as {}, blocked={}.",
+            "Created group/broadcast '{}' grpid={} as {}, blocked={}.",
             &grpname,
             grpid,
             chat_id,
@@ -460,7 +462,11 @@ impl ChatId {
     }
 
     /// Adds message "Messages are end-to-end encrypted".
-    async fn add_encrypted_msg(self, context: &Context, timestamp_sort: i64) -> Result<()> {
+    pub(crate) async fn add_encrypted_msg(
+        self,
+        context: &Context,
+        timestamp_sort: i64,
+    ) -> Result<()> {
         let text = stock_str::messages_e2e_encrypted(context).await;
         add_info_msg_with_cmd(
             context,
@@ -1489,8 +1495,9 @@ impl Chat {
     pub async fn is_self_in_chat(&self, context: &Context) -> Result<bool> {
         match self.typ {
             Chattype::Single | Chattype::OutBroadcast | Chattype::Mailinglist => Ok(true),
-            Chattype::Group => is_contact_in_chat(context, self.id, ContactId::SELF).await,
-            Chattype::InBroadcast => Ok(false),
+            Chattype::Group | Chattype::InBroadcast => {
+                is_contact_in_chat(context, self.id, ContactId::SELF).await
+            }
         }
     }
 
@@ -1636,36 +1643,37 @@ impl Chat {
 
     /// Returns true if the chat is encrypted.
     pub async fn is_encrypted(&self, context: &Context) -> Result<bool> {
-        let is_encrypted = match self.typ {
-            Chattype::Single => {
-                match context
-                    .sql
-                    .query_row_optional(
-                        "SELECT cc.contact_id, c.fingerprint<>''
+        let is_encrypted = self.is_self_talk()
+            || match self.typ {
+                Chattype::Single => {
+                    match context
+                        .sql
+                        .query_row_optional(
+                            "SELECT cc.contact_id, c.fingerprint<>''
                              FROM chats_contacts cc LEFT JOIN contacts c
                                  ON c.id=cc.contact_id
                              WHERE cc.chat_id=?
                             ",
-                        (self.id,),
-                        |row| {
-                            let id: ContactId = row.get(0)?;
-                            let is_key: bool = row.get(1)?;
-                            Ok((id, is_key))
-                        },
-                    )
-                    .await?
-                {
-                    Some((id, is_key)) => is_key || id == ContactId::DEVICE,
-                    None => true,
+                            (self.id,),
+                            |row| {
+                                let id: ContactId = row.get(0)?;
+                                let is_key: bool = row.get(1)?;
+                                Ok((id, is_key))
+                            },
+                        )
+                        .await?
+                    {
+                        Some((id, is_key)) => is_key || id == ContactId::DEVICE,
+                        None => true,
+                    }
                 }
-            }
-            Chattype::Group => {
-                // Do not encrypt ad-hoc groups.
-                !self.grpid.is_empty()
-            }
-            Chattype::Mailinglist => false,
-            Chattype::OutBroadcast | Chattype::InBroadcast => true,
-        };
+                Chattype::Group => {
+                    // Do not encrypt ad-hoc groups.
+                    !self.grpid.is_empty()
+                }
+                Chattype::Mailinglist => false,
+                Chattype::OutBroadcast | Chattype::InBroadcast => true,
+            };
         Ok(is_encrypted)
     }
 
@@ -2029,7 +2037,10 @@ impl Chat {
                     ON c.id=cc.contact_id \
                     WHERE cc.chat_id=? AND cc.add_timestamp >= cc.remove_timestamp",
                     (self.id,),
-                    |row| row.get::<_, String>(0),
+                    |row| {
+                        let addr: String = row.get(0)?;
+                        Ok(addr)
+                    },
                 )
                 .await?;
             self.sync(context, SyncAction::SetContacts(addrs)).await?;
@@ -2568,8 +2579,9 @@ pub async fn is_contact_in_chat(
 ) -> Result<bool> {
     // this function works for group and for normal chats, however, it is more useful
     // for group chats.
-    // ContactId::SELF may be used to check, if the user itself is in a group
-    // chat (ContactId::SELF is not added to normal chats)
+    // ContactId::SELF may be used to check whether oneself
+    // is in a group or incoming broadcast chat
+    // (ContactId::SELF is not added to 1:1 chats or outgoing broadcast channels)
 
     let exists = context
         .sql
@@ -2659,8 +2671,12 @@ async fn prepare_send_msg(
         // Allow to send "Member removed" messages so we can leave the group/broadcast.
         // Necessary checks should be made anyway before removing contact
         // from the chat.
-        CantSendReason::NotAMember | CantSendReason::InBroadcast => {
-            msg.param.get_cmd() == SystemMessage::MemberRemovedFromGroup
+        CantSendReason::NotAMember => msg.param.get_cmd() == SystemMessage::MemberRemovedFromGroup,
+        CantSendReason::InBroadcast => {
+            matches!(
+                msg.param.get_cmd(),
+                SystemMessage::MemberRemovedFromGroup | SystemMessage::SecurejoinMessage
+            )
         }
         CantSendReason::MissingKey => msg
             .param
@@ -3038,7 +3054,7 @@ pub async fn get_chat_msgs_ex(
             ))
         }
     };
-    let process_rows = |rows: rusqlite::MappedRows<_>| {
+    let process_rows = |rows: rusqlite::AndThenRows<_>| {
         // It is faster to sort here rather than
         // let sqlite execute an ORDER BY clause.
         let mut sorted_rows = Vec::new();
@@ -3120,7 +3136,10 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()>
                     LEFT JOIN chats c ON m.chat_id=c.id
                     WHERE m.state=10 AND m.hidden=0 AND m.chat_id>9 AND c.archived=1",
                 (),
-                |row| row.get::<_, ChatId>(0),
+                |row| {
+                    let chat_id: ChatId = row.get(0)?;
+                    Ok(chat_id)
+                },
             )
             .await?;
         if chat_ids_in_archive.is_empty() {
@@ -3298,7 +3317,10 @@ pub async fn get_chat_media(
                     DC_CHAT_ID_TRASH,
                     Viewtype::Webxdc,
                 ),
-                |row| row.get::<_, MsgId>(0),
+                |row| {
+                    let msg_id: MsgId = row.get(0)?;
+                    Ok(msg_id)
+                },
             )
             .await?
     } else {
@@ -3328,7 +3350,10 @@ pub async fn get_chat_media(
                         msg_type
                     },
                 ),
-                |row| row.get::<_, MsgId>(0),
+                |row| {
+                    let msg_id: MsgId = row.get(0)?;
+                    Ok(msg_id)
+                },
             )
             .await?
     };
@@ -3349,7 +3374,10 @@ pub async fn get_chat_contacts(context: &Context, chat_id: ChatId) -> Result<Vec
               WHERE cc.chat_id=? AND cc.add_timestamp >= cc.remove_timestamp
               ORDER BY c.id=1, c.last_seen DESC, c.id DESC;",
             (chat_id,),
-            |row| row.get::<_, ContactId>(0),
+            |row| {
+                let contact_id: ContactId = row.get(0)?;
+                Ok(contact_id)
+            },
         )
         .await
 }
@@ -3371,7 +3399,10 @@ pub async fn get_past_chat_contacts(context: &Context, chat_id: ChatId) -> Resul
              AND ? < cc.remove_timestamp
              ORDER BY c.id=1, cc.remove_timestamp DESC, c.id DESC",
             (chat_id, now.saturating_sub(60 * 24 * 3600)),
-            |row| row.get::<_, ContactId>(0),
+            |row| {
+                let contact_id: ContactId = row.get(0)?;
+                Ok(contact_id)
+            },
         )
         .await
 }
@@ -3432,7 +3463,13 @@ pub(crate) async fn create_group_ex(
     if !context.get_config_bool(Config::Bot).await?
         && !context.get_config_bool(Config::SkipStartMessages).await?
     {
-        let text = stock_str::new_group_send_first_message(context).await;
+        let text = if !grpid.is_empty() {
+            // Add "Others will only see this group after you sent a first message." message.
+            stock_str::new_group_send_first_message(context).await
+        } else {
+            // Add "Messages in this chat use classic email and are not encrypted." message.
+            stock_str::chat_unencrypted_explanation(context).await
+        };
         add_info_msg(context, chat_id, &text, create_smeared_timestamp(context)).await?;
     }
     if let (true, true) = (sync.into(), !grpid.is_empty()) {
@@ -3443,7 +3480,7 @@ pub(crate) async fn create_group_ex(
     Ok(chat_id)
 }
 
-/// Create a new **broadcast channel**
+/// Create a new, outgoing **broadcast channel**
 /// (called "Channel" in the UI).
 ///
 /// Broadcast channels are similar to groups on the sending device,
@@ -3460,60 +3497,99 @@ pub(crate) async fn create_group_ex(
 /// Returns the created chat's id.
 pub async fn create_broadcast(context: &Context, chat_name: String) -> Result<ChatId> {
     let grpid = create_id();
-    create_broadcast_ex(context, Sync, grpid, chat_name).await
+    let secret = create_broadcast_secret();
+    create_out_broadcast_ex(context, Sync, grpid, chat_name, secret).await
 }
 
-pub(crate) async fn create_broadcast_ex(
+const SQL_INSERT_BROADCAST_SECRET: &str =
+    "INSERT INTO broadcast_secrets (chat_id, secret) VALUES (?, ?)
+    ON CONFLICT(chat_id) DO UPDATE SET secret=excluded.secret";
+
+pub(crate) async fn create_out_broadcast_ex(
     context: &Context,
     sync: sync::Sync,
     grpid: String,
     chat_name: String,
+    secret: String,
 ) -> Result<ChatId> {
-    let row_id = {
-        let chat_name = &chat_name;
-        let grpid = &grpid;
-        let trans_fn = |t: &mut rusqlite::Transaction| {
-            let cnt = t.execute("UPDATE chats SET name=? WHERE grpid=?", (chat_name, grpid))?;
-            ensure!(cnt <= 1, "{cnt} chats exist with grpid {grpid}");
-            if cnt == 1 {
-                return Ok(t.query_row(
-                    "SELECT id FROM chats WHERE grpid=? AND type=?",
-                    (grpid, Chattype::OutBroadcast),
-                    |row| {
-                        let id: isize = row.get(0)?;
-                        Ok(id)
-                    },
-                )?);
-            }
-            t.execute(
-                "INSERT INTO chats \
-                (type, name, grpid, param, created_timestamp) \
-                VALUES(?, ?, ?, \'U=1\', ?);",
-                (
-                    Chattype::OutBroadcast,
-                    &chat_name,
-                    &grpid,
-                    create_smeared_timestamp(context),
-                ),
-            )?;
-            Ok(t.last_insert_rowid().try_into()?)
-        };
-        context.sql.transaction(trans_fn).await?
+    let chat_name = sanitize_single_line(&chat_name);
+    if chat_name.is_empty() {
+        bail!("Invalid broadcast channel name: {chat_name}.");
+    }
+
+    let timestamp = create_smeared_timestamp(context);
+    let trans_fn = |t: &mut rusqlite::Transaction| -> Result<ChatId> {
+        let cnt: u32 = t.query_row(
+            "SELECT COUNT(*) FROM chats WHERE grpid=?",
+            (&grpid,),
+            |row| row.get(0),
+        )?;
+        ensure!(cnt == 0, "{cnt} chats exist with grpid {grpid}");
+
+        t.execute(
+            "INSERT INTO chats
+            (type, name, grpid, created_timestamp)
+            VALUES(?, ?, ?, ?);",
+            (Chattype::OutBroadcast, &chat_name, &grpid, timestamp),
+        )?;
+        let chat_id = ChatId::new(t.last_insert_rowid().try_into()?);
+
+        t.execute(SQL_INSERT_BROADCAST_SECRET, (chat_id, &secret))?;
+        Ok(chat_id)
     };
-    let chat_id = ChatId::new(u32::try_from(row_id)?);
+    let chat_id = context.sql.transaction(trans_fn).await?;
+    chat_id.add_encrypted_msg(context, timestamp).await?;
 
     context.emit_msgs_changed_without_ids();
     chatlist_events::emit_chatlist_changed(context);
+    chatlist_events::emit_chatlist_item_changed(context, chat_id);
 
     if sync.into() {
         let id = SyncId::Grpid(grpid);
-        let action = SyncAction::CreateBroadcast(chat_name);
+        let action = SyncAction::CreateOutBroadcast { chat_name, secret };
         self::sync(context, id, action).await.log_err(context).ok();
     }
 
     Ok(chat_id)
 }
 
+pub(crate) async fn load_broadcast_secret(
+    context: &Context,
+    chat_id: ChatId,
+) -> Result<Option<String>> {
+    context
+        .sql
+        .query_get_value(
+            "SELECT secret FROM broadcast_secrets WHERE chat_id=?",
+            (chat_id,),
+        )
+        .await
+}
+
+pub(crate) async fn save_broadcast_secret(
+    context: &Context,
+    chat_id: ChatId,
+    secret: &str,
+) -> Result<()> {
+    info!(context, "Saving broadcast secret for chat {chat_id}");
+    context
+        .sql
+        .execute(SQL_INSERT_BROADCAST_SECRET, (chat_id, secret))
+        .await?;
+
+    Ok(())
+}
+
+pub(crate) async fn delete_broadcast_secret(context: &Context, chat_id: ChatId) -> Result<()> {
+    info!(context, "Removing broadcast secret for chat {chat_id}");
+    context
+        .sql
+        .execute("DELETE FROM broadcast_secrets WHERE chat_id=?", (chat_id,))
+        .await?;
+
+    Ok(())
+}
+
 /// Set chat contacts in the `chats_contacts` table.
 pub(crate) async fn update_chat_contacts_table(
     context: &Context,
@@ -3601,6 +3677,30 @@ pub(crate) async fn remove_from_chat_contacts_table(
     Ok(())
 }
 
+/// Removes a contact from the chat
+/// without leaving a trace.
+///
+/// Note that if we call this function,
+/// and then receive a message from another device
+/// that doesn't know that this this member was removed
+/// then the group membership algorithm will wrongly re-add this member.
+pub(crate) async fn remove_from_chat_contacts_table_without_trace(
+    context: &Context,
+    chat_id: ChatId,
+    contact_id: ContactId,
+) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "DELETE FROM chats_contacts
+            WHERE chat_id=? AND contact_id=?",
+            (chat_id, contact_id),
+        )
+        .await?;
+
+    Ok(())
+}
+
 /// Adds a contact to the chat.
 /// If the group is promoted, also sends out a system message to all group members
 pub async fn add_contact_to_chat(
@@ -3628,14 +3728,13 @@ pub(crate) async fn add_contact_to_chat_ex(
     // this also makes sure, no contacts are added to special or normal chats
     let mut chat = Chat::load_from_db(context, chat_id).await?;
     ensure!(
-        chat.typ == Chattype::Group || chat.typ == Chattype::OutBroadcast,
-        "{chat_id} is not a group/broadcast where one can add members"
+        chat.typ == Chattype::Group || (from_handshake && chat.typ == Chattype::OutBroadcast),
+        "{chat_id} is not a group where one can add members",
     );
     ensure!(
         Contact::real_exists_by_id(context, contact_id).await? || contact_id == ContactId::SELF,
         "invalid contact_id {contact_id} for adding to group"
     );
-    ensure!(!chat.is_mailing_list(), "Mailing lists can't be changed");
     ensure!(
         chat.typ != Chattype::OutBroadcast || contact_id != ContactId::SELF,
         "Cannot add SELF to broadcast channel."
@@ -3649,7 +3748,11 @@ pub(crate) async fn add_contact_to_chat_ex(
         context.emit_event(EventType::ErrorSelfNotInGroup(
             "Cannot add contact to group; self not in group.".into(),
         ));
-        bail!("can not add contact because the account is not part of the group/broadcast");
+        warn!(
+            context,
+            "Can not add contact because the account is not part of the group/broadcast."
+        );
+        return Ok(false);
     }
 
     let sync_qr_code_tokens;
@@ -3679,21 +3782,35 @@ pub(crate) async fn add_contact_to_chat_ex(
         }
     } else {
         // else continue and send status mail
-        if is_contact_in_chat(context, chat_id, contact_id).await? {
-            return Ok(false);
-        }
         add_to_chat_contacts_table(context, time(), chat_id, &[contact_id]).await?;
     }
-    if chat.typ == Chattype::Group && chat.is_promoted() {
+    if chat.is_promoted() {
         msg.viewtype = Viewtype::Text;
 
         let contact_addr = contact.get_addr().to_lowercase();
-        msg.text = stock_str::msg_add_member_local(context, contact.id, ContactId::SELF).await;
+        let added_by = if from_handshake && chat.typ == Chattype::OutBroadcast {
+            // The contact was added via a QR code rather than explicit user action,
+            // so it could be confusing to say 'You added member Alice'.
+            // And in a broadcast, SELF is the only one who can add members,
+            // so, no information is lost by just writing 'Member Alice added' instead.
+            ContactId::UNDEFINED
+        } else {
+            ContactId::SELF
+        };
+        msg.text = stock_str::msg_add_member_local(context, contact.id, added_by).await;
         msg.param.set_cmd(SystemMessage::MemberAddedToGroup);
         msg.param.set(Param::Arg, contact_addr);
         msg.param.set_int(Param::Arg2, from_handshake.into());
+        let fingerprint = contact.fingerprint().map(|f| f.hex());
+        msg.param.set_optional(Param::Arg4, fingerprint);
         msg.param
             .set_int(Param::ContactAddedRemoved, contact.id.to_u32() as i32);
+        if chat.typ == Chattype::OutBroadcast {
+            let secret = load_broadcast_secret(context, chat_id)
+                .await?
+                .context("Failed to find broadcast shared secret")?;
+            msg.param.set(PARAM_BROADCAST_SECRET, secret);
+        }
         send_msg(context, chat_id, &mut msg).await?;
 
         sync = Nosync;
@@ -3734,11 +3851,13 @@ pub(crate) async fn shall_attach_selfavatar(context: &Context, chat_id: ChatId)
              LEFT JOIN contacts c ON c.id=cc.contact_id
              WHERE cc.chat_id=? AND cc.contact_id!=? AND cc.add_timestamp >= cc.remove_timestamp",
             (chat_id, ContactId::SELF),
-            |row| Ok(row.get::<_, i64>(0)),
+            |row| {
+                let selfavatar_sent: i64 = row.get(0)?;
+                Ok(selfavatar_sent)
+            },
             |rows| {
                 let mut needs_attach = false;
                 for row in rows {
-                    let row = row?;
                     let selfavatar_sent = row?;
                     if selfavatar_sent < timestamp_some_days_ago {
                         needs_attach = true;
@@ -3847,7 +3966,18 @@ pub async fn remove_contact_from_chat(
     );
 
     let chat = Chat::load_from_db(context, chat_id).await?;
-    if chat.typ == Chattype::Group || chat.typ == Chattype::OutBroadcast {
+    if chat.typ == Chattype::InBroadcast {
+        ensure!(
+            contact_id == ContactId::SELF,
+            "Cannot remove other member from incoming broadcast channel"
+        );
+        delete_broadcast_secret(context, chat_id).await?;
+    }
+
+    if matches!(
+        chat.typ,
+        Chattype::Group | Chattype::OutBroadcast | Chattype::InBroadcast
+    ) {
         if !chat.is_self_in_chat(context).await? {
             let err_msg = format!(
                 "Cannot remove contact {contact_id} from chat {chat_id}: self not in group."
@@ -3860,28 +3990,28 @@ pub async fn remove_contact_from_chat(
             if chat.is_promoted() {
                 remove_from_chat_contacts_table(context, chat_id, contact_id).await?;
             } else {
-                context
-                    .sql
-                    .execute(
-                        "DELETE FROM chats_contacts
-                         WHERE chat_id=? AND contact_id=?",
-                        (chat_id, contact_id),
-                    )
-                    .await?;
+                remove_from_chat_contacts_table_without_trace(context, chat_id, contact_id).await?;
             }
 
             // We do not return an error if the contact does not exist in the database.
             // This allows to delete dangling references to deleted contacts
             // in case of the database becoming inconsistent due to a bug.
             if let Some(contact) = Contact::get_by_id_optional(context, contact_id).await? {
-                if chat.typ == Chattype::Group && chat.is_promoted() {
+                if chat.is_promoted() {
                     let addr = contact.get_addr();
+                    let fingerprint = contact.fingerprint().map(|f| f.hex());
 
-                    let res = send_member_removal_msg(context, &chat, contact_id, addr).await;
+                    let res = send_member_removal_msg(
+                        context,
+                        &chat,
+                        contact_id,
+                        addr,
+                        fingerprint.as_deref(),
+                    )
+                    .await;
 
                     if contact_id == ContactId::SELF {
                         res?;
-                        set_group_explicitly_left(context, &chat.grpid).await?;
                     } else if let Err(e) = res {
                         warn!(
                             context,
@@ -3897,11 +4027,6 @@ pub async fn remove_contact_from_chat(
                 chat.sync_contacts(context).await.log_err(context).ok();
             }
         }
-    } else if chat.typ == Chattype::InBroadcast && contact_id == ContactId::SELF {
-        // For incoming broadcast channels, it's not possible to remove members,
-        // but it's possible to leave:
-        let self_addr = context.get_primary_self_addr().await?;
-        send_member_removal_msg(context, &chat, contact_id, &self_addr).await?;
     } else {
         bail!("Cannot remove members from non-group chats.");
     }
@@ -3914,6 +4039,7 @@ async fn send_member_removal_msg(
     chat: &Chat,
     contact_id: ContactId,
     addr: &str,
+    fingerprint: Option<&str>,
 ) -> Result<MsgId> {
     let mut msg = Message::new(Viewtype::Text);
 
@@ -3929,31 +4055,13 @@ async fn send_member_removal_msg(
 
     msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup);
     msg.param.set(Param::Arg, addr.to_lowercase());
+    msg.param.set_optional(Param::Arg4, fingerprint);
     msg.param
         .set(Param::ContactAddedRemoved, contact_id.to_u32());
 
     send_msg(context, chat.id, &mut msg).await
 }
 
-async fn set_group_explicitly_left(context: &Context, grpid: &str) -> Result<()> {
-    if !is_group_explicitly_left(context, grpid).await? {
-        context
-            .sql
-            .execute("INSERT INTO leftgrps (grpid) VALUES(?);", (grpid,))
-            .await?;
-    }
-
-    Ok(())
-}
-
-pub(crate) async fn is_group_explicitly_left(context: &Context, grpid: &str) -> Result<bool> {
-    let exists = context
-        .sql
-        .exists("SELECT COUNT(*) FROM leftgrps WHERE grpid=?;", (grpid,))
-        .await?;
-    Ok(exists)
-}
-
 /// Sets group or mailing list chat name.
 pub async fn set_chat_name(context: &Context, chat_id: ChatId, new_name: &str) -> Result<()> {
     rename_ex(context, Sync, chat_id, new_name).await
@@ -4714,7 +4822,10 @@ pub(crate) enum SyncAction {
     SetVisibility(ChatVisibility),
     SetMuted(MuteDuration),
     /// Create broadcast channel with the given name.
-    CreateBroadcast(String),
+    CreateOutBroadcast {
+        chat_name: String,
+        secret: String,
+    },
     /// Create encrypted group chat with the given name.
     CreateGroupEncrypted(String),
     Rename(String),
@@ -4779,12 +4890,23 @@ impl Context {
                     .id
             }
             SyncId::Grpid(grpid) => {
-                if let SyncAction::CreateBroadcast(name) = action {
-                    create_broadcast_ex(self, Nosync, grpid.clone(), name.clone()).await?;
-                    return Ok(());
-                } else if let SyncAction::CreateGroupEncrypted(name) = action {
-                    create_group_ex(self, Nosync, grpid.clone(), name).await?;
-                    return Ok(());
+                match action {
+                    SyncAction::CreateOutBroadcast { chat_name, secret } => {
+                        create_out_broadcast_ex(
+                            self,
+                            Nosync,
+                            grpid.to_string(),
+                            chat_name.clone(),
+                            secret.to_string(),
+                        )
+                        .await?;
+                        return Ok(());
+                    }
+                    SyncAction::CreateGroupEncrypted(name) => {
+                        create_group_ex(self, Nosync, grpid.clone(), name).await?;
+                        return Ok(());
+                    }
+                    _ => {}
                 }
                 get_chat_id_by_grpid(self, grpid)
                     .await?
@@ -4806,7 +4928,8 @@ impl Context {
             SyncAction::Accept => chat_id.accept_ex(self, Nosync).await,
             SyncAction::SetVisibility(v) => chat_id.set_visibility_ex(self, Nosync, *v).await,
             SyncAction::SetMuted(duration) => set_muted_ex(self, Nosync, chat_id, *duration).await,
-            SyncAction::CreateBroadcast(_) | SyncAction::CreateGroupEncrypted(..) => {
+            SyncAction::CreateOutBroadcast { .. } | SyncAction::CreateGroupEncrypted(..) => {
+                // Create action should have been handled above already.
                 Err(anyhow!("sync_alter_chat({id:?}, {action:?}): Bad request."))
             }
             SyncAction::Rename(to) => rename_ex(self, Nosync, chat_id, to).await,

src/chatlist.rs

@@ -107,11 +107,6 @@ impl Chatlist {
             Ok((chat_id, msg_id))
         };
 
-        let process_rows = |rows: rusqlite::MappedRows<_>| {
-            rows.collect::<std::result::Result<Vec<_>, _>>()
-                .map_err(Into::into)
-        };
-
         let skip_id = if flag_for_forwarding {
             ChatId::lookup_by_contact(context, ContactId::DEVICE)
                 .await?
@@ -132,7 +127,7 @@ impl Chatlist {
         // groups. Otherwise it would be hard to follow conversations.
         let ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
-            context.sql.query_map(
+            context.sql.query_map_vec(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -150,7 +145,6 @@ impl Chatlist {
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                 (MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned),
                 process_row,
-                process_rows,
             ).await?
         } else if flag_archived_only {
             // show archived chats
@@ -159,7 +153,7 @@ impl Chatlist {
             // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context
                 .sql
-                .query_map(
+                .query_map_vec(
                     "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -177,7 +171,6 @@ impl Chatlist {
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft,),
                     process_row,
-                    process_rows,
                 )
                 .await?
         } else if let Some(query) = query {
@@ -195,7 +188,7 @@ impl Chatlist {
             let str_like_cmd = format!("%{query}%");
             context
                 .sql
-                .query_map(
+                .query_map_vec(
                     "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -214,7 +207,6 @@ impl Chatlist {
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, str_like_cmd, only_unread, MessageState::InFresh),
                     process_row,
-                    process_rows,
                 )
                 .await?
         } else {
@@ -229,7 +221,7 @@ impl Chatlist {
                     let msg_id: Option<MsgId> = row.get(3)?;
                     Ok((chat_id, typ, param, msg_id))
                 };
-                let process_rows = |rows: rusqlite::MappedRows<_>| {
+                let process_rows = |rows: rusqlite::AndThenRows<_>| {
                     rows.filter_map(|row: std::result::Result<(_, _, Params, _), _>| match row {
                         Ok((chat_id, typ, param, msg_id)) => {
                             if typ == Chattype::Mailinglist
@@ -243,7 +235,6 @@ impl Chatlist {
                         Err(e) => Some(Err(e)),
                     })
                     .collect::<std::result::Result<Vec<_>, _>>()
-                    .map_err(Into::into)
                 };
                 context.sql.query_map(
                     "SELECT c.id, c.type, c.param, m.id
@@ -272,7 +263,7 @@ impl Chatlist {
                 ).await?
             } else {
                 //  show normal chatlist
-                context.sql.query_map(
+                context.sql.query_map_vec(
                     "SELECT c.id, m.id
                      FROM chats c
                      LEFT JOIN msgs m
@@ -290,7 +281,6 @@ impl Chatlist {
                      ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, ChatVisibility::Archived, ChatVisibility::Pinned),
                     process_row,
-                    process_rows,
                 ).await?
             };
             if !flag_no_specials && get_archived_cnt(context).await? > 0 {

src/config.rs

@@ -16,12 +16,12 @@ use crate::blob::BlobObject;
 use crate::configure::EnteredLoginParam;
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::{LogExt, info};
-use crate::login_param::ConfiguredLoginParam;
+use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{Provider, get_provider_by_id};
 use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::get_abs_path;
+use crate::transport::ConfiguredLoginParam;
 use crate::{constants, stats};
 
 /// The available configuration keys.
@@ -438,8 +438,19 @@ pub enum Config {
     /// storing the same token multiple times on the server.
     EncryptedDeviceToken,
 
+    /// Enables running test hooks, e.g. see `InnerContext::pre_encrypt_mime_hook`.
+    /// This way is better than conditional compilation, i.e. `#[cfg(test)]`, because tests not
+    /// using this still run unmodified code.
+    TestHooks,
+
     /// Return an error from `receive_imf_inner()` for a fully downloaded message. For tests.
     FailOnReceivingFullMsg,
+
+    /// Enable composing emails with Header Protection as defined in
+    /// <https://www.rfc-editor.org/rfc/rfc9788.html> "Header Protection for Cryptographically
+    /// Protected Email".
+    #[strum(props(default = "1"))]
+    StdHeaderProtectionComposing,
 }
 
 impl Config {
@@ -665,7 +676,7 @@ impl Context {
             Config::Selfavatar if value.is_empty() => None,
             Config::Selfavatar => {
                 config_value = BlobObject::store_from_base64(self, value)?;
-                Some(config_value.as_str())
+                config_value.as_deref()
             }
             _ => Some(value),
         };

src/configure.rs

@@ -27,19 +27,21 @@ use crate::config::{self, Config};
 use crate::constants::NON_ALPHANUMERIC_WITHOUT_DOT;
 use crate::context::Context;
 use crate::imap::Imap;
-use crate::log::{LogExt, info, warn};
+use crate::log::{LogExt, warn};
+use crate::login_param::EnteredCertificateChecks;
 pub use crate::login_param::EnteredLoginParam;
-use crate::login_param::{
-    ConfiguredCertificateChecks, ConfiguredLoginParam, ConfiguredServerLoginParam,
-    ConnectionCandidate, EnteredCertificateChecks, ProxyConfig,
-};
 use crate::message::Message;
+use crate::net::proxy::ProxyConfig;
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Provider, Socket, UsernamePattern};
 use crate::qr::{login_param_from_account_qr, login_param_from_login_qr};
 use crate::smtp::Smtp;
 use crate::sync::Sync::*;
 use crate::tools::time;
+use crate::transport::{
+    ConfiguredCertificateChecks, ConfiguredLoginParam, ConfiguredServerLoginParam,
+    ConnectionCandidate,
+};
 use crate::{EventType, stock_str};
 use crate::{chat, provider};
 use deltachat_contact_tools::addr_cmp;
@@ -192,16 +194,11 @@ impl Context {
     pub async fn list_transports(&self) -> Result<Vec<EnteredLoginParam>> {
         let transports = self
             .sql
-            .query_map(
-                "SELECT entered_param FROM transports",
-                (),
-                |row| row.get::<_, String>(0),
-                |rows| {
-                    rows.flatten()
-                        .map(|s| Ok(serde_json::from_str(&s)?))
-                        .collect::<Result<Vec<EnteredLoginParam>>>()
-                },
-            )
+            .query_map_vec("SELECT entered_param FROM transports", (), |row| {
+                let entered_param: String = row.get(0)?;
+                let transport: EnteredLoginParam = serde_json::from_str(&entered_param)?;
+                Ok(transport)
+            })
             .await?;
 
         Ok(transports)
@@ -568,14 +565,6 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Option<&'
 
     progress!(ctx, 910);
 
-    if let Some(configured_addr) = ctx.get_config(Config::ConfiguredAddr).await? {
-        if configured_addr != param.addr {
-            // Switched account, all server UIDs we know are invalid
-            info!(ctx, "Scheduling resync because the address has changed.");
-            ctx.schedule_resync().await?;
-        }
-    }
-
     let provider = configured_param.provider;
     configured_param
         .save_to_transports_table(ctx, param)

src/configure/auto_mozilla.rs

@@ -28,8 +28,9 @@ struct MozAutoconfigure {
     pub outgoing_servers: Vec<Server>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 enum MozConfigTag {
+    #[default]
     Undefined,
     Hostname,
     Port,
@@ -37,12 +38,6 @@ enum MozConfigTag {
     Username,
 }
 
-impl Default for MozConfigTag {
-    fn default() -> Self {
-        Self::Undefined
-    }
-}
-
 impl FromStr for MozConfigTag {
     type Err = ();
 

src/constants.rs

@@ -223,6 +223,9 @@ pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 5;
 // `max_smtp_rcpt_to` in the provider db.
 pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
 
+/// Same as `DEFAULT_MAX_SMTP_RCPT_TO`, but for chatmail relays.
+pub(crate) const DEFAULT_CHATMAIL_MAX_SMTP_RCPT_TO: usize = 999;
+
 /// How far the last quota check needs to be in the past to be checked by the background function (in seconds).
 pub(crate) const DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT: u64 = 12 * 60 * 60; // 12 hours
 
@@ -250,6 +253,18 @@ pub(crate) const ASM_BODY: &str = "This is the Autocrypt Setup Message \
 /// Period between `sql::housekeeping()` runs.
 pub(crate) const HOUSEKEEPING_PERIOD: i64 = 24 * 60 * 60;
 
+pub(crate) const BROADCAST_INCOMPATIBILITY_MSG: &str = r#"The up to now "experimental channels feature" is about to become an officially supported one. By that, privacy will be improved, it will become faster, and less traffic will be consumed.
+
+As we do not guarantee feature-stability for such experiments, this means, that you will need to create the channel again. 
+
+Here is what to do:
+ • Create a new channel
+ • Tap on the channel name
+ • Tap on "QR Invite Code"
+ • Have all recipients scan the QR code, or send them the link
+
+If you have any questions, please send an email to delta@merlinux.eu or ask at https://support.delta.chat/."#;
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/contact.rs

@@ -31,7 +31,7 @@ use crate::key::{
     DcKey, Fingerprint, SignedPublicKey, load_self_public_key, self_fingerprint,
     self_fingerprint_opt,
 };
-use crate::log::{LogExt, info, warn};
+use crate::log::{LogExt, warn};
 use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
@@ -369,16 +369,15 @@ async fn import_vcard_contact(context: &Context, contact: &VcardContact) -> Resu
         return Ok(id);
     }
     let path = match &contact.profile_image {
-        Some(image) => match BlobObject::store_from_base64(context, image) {
-            Err(e) => {
+        Some(image) => match BlobObject::store_from_base64(context, image)? {
+            None => {
                 warn!(
                     context,
-                    "import_vcard_contact: Could not decode and save avatar for {}: {e:#}.",
-                    contact.addr
+                    "import_vcard_contact: Could not decode avatar for {}.", contact.addr
                 );
                 None
             }
-            Ok(path) => Some(path),
+            Some(path) => Some(path),
         },
         None => None,
     };
@@ -1285,7 +1284,10 @@ impl Contact {
             .query_map_vec(
                 "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY last_seen DESC, id DESC;",
                 (ContactId::LAST_SPECIAL,),
-                |row| row.get::<_, ContactId>(0),
+                |row| {
+                    let contact_id: ContactId = row.get(0)?;
+                    Ok(contact_id)
+                }
             )
             .await?;
         Ok(list)
@@ -1571,11 +1573,24 @@ impl Contact {
     }
 
     /// Returns a color for the contact.
-    /// See [`self::get_color`].
+    /// For self-contact this returns gray if own keypair doesn't exist yet.
+    /// See also [`self::get_color`].
     pub fn get_color(&self) -> u32 {
         get_color(self.id == ContactId::SELF, &self.addr, &self.fingerprint())
     }
 
+    /// Returns a color for the contact.
+    /// Ensures that the color isn't gray. For self-contact this generates own keypair if it doesn't
+    /// exist yet.
+    /// See also [`self::get_color`].
+    pub async fn get_or_gen_color(&self, context: &Context) -> Result<u32> {
+        let mut fpr = self.fingerprint();
+        if fpr.is_none() && self.id == ContactId::SELF {
+            fpr = Some(load_self_public_key(context).await?.dc_fingerprint());
+        }
+        Ok(get_color(self.id == ContactId::SELF, &self.addr, &fpr))
+    }
+
     /// Gets the contact's status.
     ///
     /// Status is the last signature received in a message from this contact.
@@ -2044,7 +2059,7 @@ impl RecentlySeenLoop {
         // become unseen in the future.
         let mut unseen_queue: BinaryHeap<MyHeapElem> = context
             .sql
-            .query_map(
+            .query_map_collect(
                 "SELECT id, last_seen FROM contacts
                  WHERE last_seen > ?",
                 (now_ts - SEEN_RECENTLY_SECONDS,),
@@ -2053,10 +2068,6 @@ impl RecentlySeenLoop {
                     let last_seen: i64 = row.get("last_seen")?;
                     Ok((Reverse(last_seen + SEEN_RECENTLY_SECONDS), contact_id))
                 },
-                |rows| {
-                    rows.collect::<std::result::Result<BinaryHeap<MyHeapElem>, _>>()
-                        .map_err(Into::into)
-                },
             )
             .await
             .unwrap_or_default();

src/contact/contact_tests.rs

@@ -4,7 +4,6 @@ use super::*;
 use crate::chat::{Chat, get_chat_contacts, send_text_msg};
 use crate::chatlist::Chatlist;
 use crate::receive_imf::receive_imf;
-use crate::securejoin::get_securejoin_qr;
 use crate::test_utils::{self, TestContext, TestContextManager, TimeShiftFalsePositiveNote};
 
 #[test]
@@ -775,16 +774,21 @@ async fn test_contact_get_color() -> Result<()> {
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_self_color_vs_key() -> Result<()> {
+async fn test_self_color() -> Result<()> {
     let mut tcm = TestContextManager::new();
     let t = &tcm.unconfigured().await;
     t.configure_addr("alice@example.org").await;
     assert!(t.is_configured().await?);
-    let color = Contact::get_by_id(t, ContactId::SELF).await?.get_color();
+    let self_contact = Contact::get_by_id(t, ContactId::SELF).await?;
+    let color = self_contact.get_color();
     assert_eq!(color, 0x808080);
-    get_securejoin_qr(t, None).await?;
-    let color1 = Contact::get_by_id(t, ContactId::SELF).await?.get_color();
-    assert_ne!(color1, color);
+    let color = self_contact.get_or_gen_color(t).await?;
+    assert_ne!(color, 0x808080);
+    let color1 = self_contact.get_or_gen_color(t).await?;
+    assert_eq!(color1, color);
+
+    let bob = &tcm.bob().await;
+    assert_eq!(bob.add_or_lookup_contact(t).await.get_color(), color);
     Ok(())
 }
 

src/context.rs

@@ -4,7 +4,7 @@ use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 
@@ -21,9 +21,9 @@ use crate::debug_logging::DebugLogging;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::imap::{FolderMeaning, Imap, ServerMetadata};
 use crate::key::self_fingerprint;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::logged_debug_assert;
-use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam};
+use crate::login_param::EnteredLoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::net::tls::TlsSessionStore;
 use crate::peer_channels::Iroh;
@@ -34,6 +34,7 @@ use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
 use crate::tools::{self, duration_to_str, time, time_elapsed};
+use crate::transport::ConfiguredLoginParam;
 use crate::{chatlist_events, stats};
 
 /// Builder for the [`Context`].
@@ -137,7 +138,7 @@ impl ContextBuilder {
     ///
     /// This is useful in order to share the same translation strings in all [`Context`]s.
     /// The mapping may be empty when set, it will be populated by
-    /// [`Context::set_stock-translation`] or [`Accounts::set_stock_translation`] calls.
+    /// [`Context::set_stock_translation`] or [`Accounts::set_stock_translation`] calls.
     ///
     /// Note that the [account manager](crate::accounts::Accounts) is designed to handle the
     /// common case for using multiple [`Context`] instances.
@@ -242,9 +243,6 @@ pub struct InnerContext {
     /// Set to `None` if quota was never tried to load.
     pub(crate) quota: RwLock<Option<QuotaInfo>>,
 
-    /// IMAP UID resync request.
-    pub(crate) resync_request: AtomicBool,
-
     /// Notify about new messages.
     ///
     /// This causes [`Context::wait_next_msgs`] to wake up.
@@ -305,10 +303,21 @@ pub struct InnerContext {
     /// `Connectivity` values for mailboxes, unordered. Used to compute the aggregate connectivity,
     /// see [`Context::get_connectivity()`].
     pub(crate) connectivities: parking_lot::Mutex<Vec<ConnectivityStore>>,
+
+    #[expect(clippy::type_complexity)]
+    /// Transforms the root of the cryptographic payload before encryption.
+    pub(crate) pre_encrypt_mime_hook: parking_lot::Mutex<
+        Option<
+            for<'a> fn(
+                &Context,
+                mail_builder::mime::MimePart<'a>,
+            ) -> mail_builder::mime::MimePart<'a>,
+        >,
+    >,
 }
 
 /// The state of ongoing process.
-#[derive(Debug)]
+#[derive(Debug, Default)]
 enum RunningState {
     /// Ongoing process is allocated.
     Running { cancel_sender: Sender<()> },
@@ -317,15 +326,10 @@ enum RunningState {
     ShallStop { request: tools::Time },
 
     /// There is no ongoing process, a new one can be allocated.
+    #[default]
     Stopped,
 }
 
-impl Default for RunningState {
-    fn default() -> Self {
-        Self::Stopped
-    }
-}
-
 /// Return some info about deltachat-core
 ///
 /// This contains information mostly about the library itself, the
@@ -461,7 +465,6 @@ impl Context {
             scheduler: SchedulerState::new(),
             ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow at least 1 message every 10 seconds + a burst of 6.
             quota: RwLock::new(None),
-            resync_request: AtomicBool::new(false),
             new_msgs_notify,
             server_id: RwLock::new(None),
             metadata: RwLock::new(None),
@@ -475,6 +478,7 @@ impl Context {
             iroh: Arc::new(RwLock::new(None)),
             self_fingerprint: OnceLock::new(),
             connectivities: parking_lot::Mutex::new(Vec::new()),
+            pre_encrypt_mime_hook: None.into(),
         };
 
         let ctx = Context {
@@ -553,7 +557,7 @@ impl Context {
             .and_then(|provider| provider.opt.max_smtp_rcpt_to)
             .map_or_else(
                 || match is_chatmail {
-                    true => usize::MAX,
+                    true => constants::DEFAULT_CHATMAIL_MAX_SMTP_RCPT_TO,
                     false => constants::DEFAULT_MAX_SMTP_RCPT_TO,
                 },
                 usize::from,
@@ -620,12 +624,6 @@ impl Context {
         Ok(())
     }
 
-    pub(crate) async fn schedule_resync(&self) -> Result<()> {
-        self.resync_request.store(true, Ordering::Relaxed);
-        self.scheduler.interrupt_inbox().await;
-        Ok(())
-    }
-
     /// Returns a reference to the underlying SQL instance.
     ///
     /// Warning: this is only here for testing, not part of the public API.
@@ -1065,6 +1063,13 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "test_hooks",
+            self.sql
+                .get_raw_config("test_hooks")
+                .await?
+                .unwrap_or_default(),
+        );
         res.insert(
             "fail_on_receiving_full_msg",
             self.sql
@@ -1072,6 +1077,13 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
+        res.insert(
+            "std_header_protection_composing",
+            self.sql
+                .get_raw_config("std_header_protection_composing")
+                .await?
+                .unwrap_or_default(),
+        );
 
         let elapsed = time_elapsed(&self.creation_time);
         res.insert("uptime", duration_to_str(elapsed));
@@ -1105,7 +1117,10 @@ impl Context {
                     " ORDER BY m.timestamp DESC,m.id DESC;"
                 ),
                 (MessageState::InFresh, time()),
-                |row| row.get::<_, MsgId>(0),
+                |row| {
+                    let msg_id: MsgId = row.get(0)?;
+                    Ok(msg_id)
+                },
             )
             .await?;
         Ok(list)
@@ -1209,7 +1224,10 @@ impl Context {
                    AND IFNULL(txt_normalized, txt) LIKE ?
                  ORDER BY m.timestamp,m.id;",
                     (chat_id, str_like_in_text),
-                    |row| row.get::<_, MsgId>("id"),
+                    |row| {
+                        let msg_id: MsgId = row.get("id")?;
+                        Ok(msg_id)
+                    },
                 )
                 .await?
         } else {
@@ -1238,7 +1256,10 @@ impl Context {
                    AND IFNULL(txt_normalized, txt) LIKE ?
                  ORDER BY m.id DESC LIMIT 1000",
                     (str_like_in_text,),
-                    |row| row.get::<_, MsgId>("id"),
+                    |row| {
+                        let msg_id: MsgId = row.get("id")?;
+                        Ok(msg_id)
+                    },
                 )
                 .await?
         };

src/debug_logging.rs

@@ -3,7 +3,6 @@ use crate::chat::ChatId;
 use crate::config::Config;
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::{error, info};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::param::Param;
 use crate::tools::time;

src/decrypt.rs

@@ -10,17 +10,19 @@ use crate::pgp;
 
 /// Tries to decrypt a message, but only if it is structured as an Autocrypt message.
 ///
-/// If successful and the message is encrypted, returns decrypted body.
+/// If successful and the message was encrypted,
+/// returns the decrypted and decompressed message.
 pub fn try_decrypt<'a>(
     mail: &'a ParsedMail<'a>,
     private_keyring: &'a [SignedSecretKey],
+    shared_secrets: &[String],
 ) -> Result<Option<::pgp::composed::Message<'static>>> {
     let Some(encrypted_data_part) = get_encrypted_mime(mail) else {
         return Ok(None);
     };
 
     let data = encrypted_data_part.get_body_raw()?;
-    let msg = pgp::pk_decrypt(data, private_keyring)?;
+    let msg = pgp::decrypt(data, private_keyring, shared_secrets)?;
 
     Ok(Some(msg))
 }

src/dehtml.rs

@@ -13,6 +13,7 @@ use quick_xml::{
 
 use crate::simplify::{SimplifiedText, simplify_quote};
 
+#[derive(Default)]
 struct Dehtml {
     strbuilder: String,
     quote: String,
@@ -25,6 +26,9 @@ struct Dehtml {
     /// Everything between `<div name="quote">` and `<div name="quoted-content">` is usually metadata
     /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
     divs_since_quoted_content_div: u32,
+    /// `<div class="header-protection-legacy-display">` elements should be omitted, see
+    /// <https://www.rfc-editor.org/rfc/rfc9788.html#section-4.5.3.3>.
+    divs_since_hp_legacy_display: u32,
     /// All-Inkl just puts the quote into `<blockquote> </blockquote>`. This count is
     /// increased at each `<blockquote>` and decreased at each `</blockquote>`.
     blockquotes_since_blockquote: u32,
@@ -48,20 +52,25 @@ impl Dehtml {
     }
 
     fn get_add_text(&self) -> AddText {
-        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
-            AddText::No // Everything between `<div name="quoted">` and `<div name="quoted_content">` is metadata which we don't want
+        // Everything between `<div name="quoted">` and `<div name="quoted_content">` is
+        // metadata which we don't want.
+        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0
+            || self.divs_since_hp_legacy_display > 0
+        {
+            AddText::No
         } else {
             self.add_text
         }
     }
 }
 
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, Default, PartialEq, Clone, Copy)]
 enum AddText {
     /// Inside `<script>`, `<style>` and similar tags
     /// which contents should not be displayed.
     No,
 
+    #[default]
     YesRemoveLineEnds,
 
     /// Inside `<pre>`.
@@ -121,12 +130,7 @@ fn dehtml_quick_xml(buf: &str) -> (String, String) {
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
-        quote: String::new(),
-        add_text: AddText::YesRemoveLineEnds,
-        last_href: None,
-        divs_since_quote_div: 0,
-        divs_since_quoted_content_div: 0,
-        blockquotes_since_blockquote: 0,
+        ..Default::default()
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -244,6 +248,7 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
         "div" => {
             pop_tag(&mut dehtml.divs_since_quote_div);
             pop_tag(&mut dehtml.divs_since_quoted_content_div);
+            pop_tag(&mut dehtml.divs_since_hp_legacy_display);
 
             *dehtml.get_buf() += "\n\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
@@ -295,6 +300,8 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
         "div" => {
             maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
             maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
+            maybe_push_tag(event, reader, "header-protection-legacy-display",
+                &mut dehtml.divs_since_hp_legacy_display);
 
             *dehtml.get_buf() += "\n\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
@@ -539,6 +546,27 @@ mod tests {
         assert_eq!(txt.text.trim(), "two\nlines");
     }
 
+    #[test]
+    fn test_hp_legacy_display() {
+        let input = r#"
+<html><head><title></title></head><body>
+<div class="header-protection-legacy-display">
+<pre>Subject: Dinner plans</pre>
+</div>
+<p>
+Let's meet at Rama's Roti Shop at 8pm and go to the park
+from there.
+</p>
+</body>
+</html>
+        "#;
+        let txt = dehtml(input).unwrap();
+        assert_eq!(
+            txt.text.trim(),
+            "Let's meet at Rama's Roti Shop at 8pm and go to the park from there."
+        );
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_quote_div() {
         let input = include_str!("../test-data/message/gmx-quote-body.eml");

src/download.rs

@@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize};
 use crate::config::Config;
 use crate::context::Context;
 use crate::imap::session::Session;
-use crate::log::info;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::tools::time;
@@ -221,21 +220,14 @@ impl MimeMessage {
     /// To create the placeholder, only the outermost header can be used,
     /// the mime-structure itself is not available.
     ///
-    /// The placeholder part currently contains a text with size and availability of the message;
-    /// `error` is set as the part error;
-    /// in the future, we may do more advanced things as previews here.
+    /// The placeholder part currently contains a text with size and availability of the message.
     pub(crate) async fn create_stub_from_partial_download(
         &mut self,
         context: &Context,
         org_bytes: u32,
-        error: Option<String>,
     ) -> Result<()> {
-        let prefix = match error {
-            None => "",
-            Some(_) => "[❗] ",
-        };
         let mut text = format!(
-            "{prefix}[{}]",
+            "[{}]",
             stock_str::partial_download_msg_body(context, org_bytes).await
         );
         if let Some(delete_server_after) = context.get_config_delete_server_after().await? {
@@ -252,7 +244,6 @@ impl MimeMessage {
         self.do_add_single_part(Part {
             typ: Viewtype::Text,
             msg: text,
-            error,
             ..Default::default()
         });
 

src/e2ee.rs

@@ -46,6 +46,7 @@ impl EncryptHelper {
         keyring: Vec<SignedPublicKey>,
         mail_to_encrypt: MimePart<'static>,
         compress: bool,
+        anonymous_recipients: bool,
     ) -> Result<String> {
         let sign_key = load_self_secret_key(context).await?;
 
@@ -53,7 +54,35 @@ impl EncryptHelper {
         let cursor = Cursor::new(&mut raw_message);
         mail_to_encrypt.clone().write_part(cursor).ok();
 
-        let ctext = pgp::pk_encrypt(raw_message, keyring, Some(sign_key), compress).await?;
+        let ctext = pgp::pk_encrypt(
+            raw_message,
+            keyring,
+            Some(sign_key),
+            compress,
+            anonymous_recipients,
+        )
+        .await?;
+
+        Ok(ctext)
+    }
+
+    /// Symmetrically encrypt the message. This is used for broadcast channels.
+    /// `shared secret` is the secret that will be used for symmetric encryption.
+    pub async fn encrypt_symmetrically(
+        self,
+        context: &Context,
+        shared_secret: &str,
+        mail_to_encrypt: MimePart<'static>,
+        compress: bool,
+    ) -> Result<String> {
+        let sign_key = load_self_secret_key(context).await?;
+
+        let mut raw_message = Vec::new();
+        let cursor = Cursor::new(&mut raw_message);
+        mail_to_encrypt.clone().write_part(cursor).ok();
+
+        let ctext =
+            pgp::symm_encrypt_message(raw_message, sign_key, shared_secret, compress).await?;
 
         Ok(ctext)
     }

src/ephemeral.rs

@@ -80,7 +80,7 @@ use crate::contact::ContactId;
 use crate::context::Context;
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::log::{LogExt, error, info, warn};
+use crate::log::{LogExt, warn};
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::stock_str;
@@ -88,9 +88,10 @@ use crate::tools::{SystemTime, duration_to_str, time};
 use crate::{location, stats};
 
 /// Ephemeral timer value.
-#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize, Default)]
 pub enum Timer {
     /// Timer is disabled.
+    #[default]
     Disabled,
 
     /// Timer is enabled.
@@ -125,12 +126,6 @@ impl Timer {
     }
 }
 
-impl Default for Timer {
-    fn default() -> Self {
-        Self::Disabled
-    }
-}
-
 impl fmt::Display for Timer {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}", self.to_u32())
@@ -246,10 +241,9 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     match timer {
         Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
-            0..=59 => {
+            0..=60 => {
                 stock_str::msg_ephemeral_timer_enabled(context, &timer.to_string(), from_id).await
             }
-            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
             61..=3599 => {
                 stock_str::msg_ephemeral_timer_minutes(
                     context,

src/ephemeral/ephemeral_tests.rs

@@ -38,7 +38,7 @@ async fn test_stock_ephemeral_messages() {
     assert_eq!(
         stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, ContactId::SELF)
             .await,
-        "You set message deletion timer to 1 minute."
+        "You set message deletion timer to 60 s."
     );
     assert_eq!(
         stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 90 }, ContactId::SELF)
@@ -142,7 +142,7 @@ async fn test_ephemeral_enable_disable() -> Result<()> {
     let bob_received_message = bob.recv_msg(&sent).await;
     assert_eq!(
         bob_received_message.text,
-        "Message deletion timer is set to 1 minute by alice@example.org."
+        "Message deletion timer is set to 60 s by alice@example.org."
     );
     assert_eq!(
         chat_bob.get_ephemeral_timer(bob).await?,

src/headerdef.rs

@@ -39,6 +39,8 @@ pub enum HeaderDef {
     /// Mailing list ID defined in [RFC 2919](https://tools.ietf.org/html/rfc2919).
     ListId,
     ListPost,
+    /// Mailing list id, belonging to a broadcast channel created by Delta Chat
+    ChatListId,
 
     /// List-Help header defined in [RFC 2369](https://datatracker.ietf.org/doc/html/rfc2369).
     ListHelp,
@@ -63,7 +65,9 @@ pub enum HeaderDef {
     ChatUserAvatar,
     ChatVoiceMessage,
     ChatGroupMemberRemoved,
+    ChatGroupMemberRemovedFpr,
     ChatGroupMemberAdded,
+    ChatGroupMemberAddedFpr,
     ChatContent,
 
     /// Past members of the group.
@@ -94,6 +98,11 @@ pub enum HeaderDef {
     /// This message obsoletes the text of the message defined here by rfc724_mid.
     ChatEdit,
 
+    /// The secret shared amongst all recipients of this broadcast channel,
+    /// used to encrypt and decrypt messages.
+    /// This secret is sent to a new member in the member-addition message.
+    ChatBroadcastSecret,
+
     /// [Autocrypt](https://autocrypt.org/) header.
     Autocrypt,
     AutocryptGossip,
@@ -129,6 +138,9 @@ pub enum HeaderDef {
     /// Advertised gossip topic for one webxdc.
     IrohGossipTopic,
 
+    /// See <https://www.rfc-editor.org/rfc/rfc9788.html#name-hp-outer-header-field>.
+    HpOuter,
+
     #[cfg(test)]
     TestHeader,
 }

src/imap.rs

@@ -24,7 +24,7 @@ use ratelimit::Ratelimit;
 use url::Url;
 
 use crate::calls::{create_fallback_ice_servers, create_ice_servers_from_metadata};
-use crate::chat::{self, ChatId, ChatIdBlocked};
+use crate::chat::{self, ChatId, ChatIdBlocked, add_device_msg};
 use crate::chatlist_events;
 use crate::config::Config;
 use crate::constants::{self, Blocked, Chattype, ShowEmails};
@@ -32,10 +32,7 @@ use crate::contact::{Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::log::{LogExt, error, info, warn};
-use crate::login_param::{
-    ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params,
-};
+use crate::log::{LogExt, warn};
 use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
 use crate::mimeparser;
 use crate::net::proxy::ProxyConfig;
@@ -48,6 +45,9 @@ use crate::receive_imf::{
 use crate::scheduler::connectivity::ConnectivityStore;
 use crate::stock_str;
 use crate::tools::{self, create_id, duration_to_str, time};
+use crate::transport::{
+    ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params,
+};
 
 pub(crate) mod capabilities;
 mod client;
@@ -104,6 +104,12 @@ pub(crate) struct Imap {
     /// immediately after logging in or returning an error in response to LOGIN command
     /// due to internal server error.
     ratelimit: Ratelimit,
+
+    /// IMAP UID resync request sender.
+    pub(crate) resync_request_sender: async_channel::Sender<()>,
+
+    /// IMAP UID resync request receiver.
+    pub(crate) resync_request_receiver: async_channel::Receiver<()>,
 }
 
 #[derive(Debug)]
@@ -254,6 +260,7 @@ impl Imap {
         oauth2: bool,
         idle_interrupt_receiver: Receiver<()>,
     ) -> Self {
+        let (resync_request_sender, resync_request_receiver) = async_channel::bounded(1);
         Imap {
             idle_interrupt_receiver,
             addr: addr.to_string(),
@@ -268,6 +275,8 @@ impl Imap {
             conn_backoff_ms: 0,
             // 1 connection per minute + a burst of 2.
             ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
+            resync_request_sender,
+            resync_request_receiver,
         }
     }
 
@@ -392,6 +401,7 @@ impl Imap {
             match login_res {
                 Ok(mut session) => {
                     let capabilities = determine_capabilities(&mut session).await?;
+                    let resync_request_sender = self.resync_request_sender.clone();
 
                     let session = if capabilities.can_compress {
                         info!(context, "Enabling IMAP compression.");
@@ -402,9 +412,9 @@ impl Imap {
                             })
                             .await
                             .context("Failed to enable IMAP compression")?;
-                        Session::new(compressed_session, capabilities)
+                        Session::new(compressed_session, capabilities, resync_request_sender)
                     } else {
-                        Session::new(session, capabilities)
+                        Session::new(session, capabilities, resync_request_sender)
                     };
 
                     // Store server ID in the context to display in account info.
@@ -1468,29 +1478,19 @@ impl Session {
                     context,
                     "Passing message UID {} to receive_imf().", request_uid
                 );
-                let res = receive_imf_inner(
-                    context,
-                    rfc724_mid,
-                    body,
-                    is_seen,
-                    partial.map(|msg_size| (msg_size, None)),
-                )
-                .await;
-                let received_msg = if let Err(err) = res {
-                    warn!(context, "receive_imf error: {:#}.", err);
-                    if partial.is_some() {
-                        return Err(err);
+                let res = receive_imf_inner(context, rfc724_mid, body, is_seen, partial).await;
+                let received_msg = match res {
+                    Err(err) => {
+                        warn!(context, "receive_imf error: {err:#}.");
+
+                        let text = format!(
+                            "❌ Failed to receive a message: {err:#}. Please report this bug to delta@merlinux.eu or https://support.delta.chat/."
+                        );
+                        let mut msg = Message::new_text(text);
+                        add_device_msg(context, None, Some(&mut msg)).await?;
+                        None
                     }
-                    receive_imf_inner(
-                        context,
-                        rfc724_mid,
-                        body,
-                        is_seen,
-                        Some((body.len().try_into()?, Some(format!("{err:#}")))),
-                    )
-                    .await?
-                } else {
-                    res?
+                    Ok(msg) => msg,
                 };
                 received_msgs_channel
                     .send((request_uid, received_msg))
@@ -2501,21 +2501,6 @@ pub(crate) async fn get_imap_self_sent_search_command(context: &Context) -> Resu
     Ok(search_command)
 }
 
-/// Deprecated, use get_uid_next() and get_uidvalidity()
-pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result<(u32, u32)> {
-    let key = format!("imap.mailbox.{folder}");
-    if let Some(entry) = context.sql.get_raw_config(&key).await? {
-        // the entry has the format `imap.mailbox.<folder>=<uidvalidity>:<lastseenuid>`
-        let mut parts = entry.split(':');
-        Ok((
-            parts.next().unwrap_or_default().parse().unwrap_or(0),
-            parts.next().unwrap_or_default().parse().unwrap_or(0),
-        ))
-    } else {
-        Ok((0, 0))
-    }
-}
-
 /// Whether to ignore fetching messages from a folder.
 ///
 /// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders

src/imap/client.rs

@@ -8,14 +8,15 @@ use tokio::io::BufWriter;
 
 use super::capabilities::Capabilities;
 use crate::context::Context;
-use crate::log::{LoggingStream, info, warn};
-use crate::login_param::{ConnectionCandidate, ConnectionSecurity};
+use crate::log::{LoggingStream, warn};
 use crate::net::dns::{lookup_host_with_cache, update_connect_timestamp};
 use crate::net::proxy::ProxyConfig;
 use crate::net::session::SessionStream;
 use crate::net::tls::wrap_tls;
 use crate::net::{connect_tcp_inner, run_connection_attempts, update_connection_history};
 use crate::tools::time;
+use crate::transport::ConnectionCandidate;
+use crate::transport::ConnectionSecurity;
 
 #[derive(Debug)]
 pub(crate) struct Client {

src/imap/idle.rs

@@ -8,7 +8,7 @@ use tokio::time::timeout;
 use super::Imap;
 use super::session::Session;
 use crate::context::Context;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::net::TIMEOUT;
 use crate::tools::{self, time_elapsed};
 

src/imap/scan_folders.rs

@@ -5,7 +5,7 @@ use anyhow::{Context as _, Result};
 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
 use crate::imap::{Imap, session::Session};
-use crate::log::{LogExt, info};
+use crate::log::LogExt;
 use crate::tools::{self, time_elapsed};
 use crate::{context::Context, imap::FolderMeaning};
 

src/imap/select_folder.rs

@@ -5,7 +5,7 @@ use anyhow::Context as _;
 use super::session::Session as ImapSession;
 use super::{get_uid_next, get_uidvalidity, set_modseq, set_uid_next, set_uidvalidity};
 use crate::context::Context;
-use crate::log::{info, warn};
+use crate::log::warn;
 
 type Result<T> = std::result::Result<T, Error>;
 
@@ -206,7 +206,7 @@ impl ImapSession {
                             "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
                         );
                         set_uid_next(context, folder, new_uid_next).await?;
-                        context.schedule_resync().await?;
+                        self.resync_request_sender.try_send(()).ok();
                     }
 
                     // If UIDNEXT changed, there are new emails.
@@ -243,7 +243,7 @@ impl ImapSession {
             .await?;
 
         if old_uid_validity != 0 || old_uid_next != 0 {
-            context.schedule_resync().await?;
+            self.resync_request_sender.try_send(()).ok();
         }
         info!(
             context,

src/imap/session.rs

@@ -48,6 +48,8 @@ pub(crate) struct Session {
     ///
     /// Should be false if no folder is currently selected.
     pub new_mail: bool,
+
+    pub resync_request_sender: async_channel::Sender<()>,
 }
 
 impl Deref for Session {
@@ -68,6 +70,7 @@ impl Session {
     pub(crate) fn new(
         inner: ImapSession<Box<dyn SessionStream>>,
         capabilities: Capabilities,
+        resync_request_sender: async_channel::Sender<()>,
     ) -> Self {
         Self {
             inner,
@@ -77,6 +80,7 @@ impl Session {
             selected_folder_needs_expunge: false,
             last_full_folder_scan: Mutex::new(None),
             new_mail: false,
+            resync_request_sender,
         }
     }
 

src/imex.rs

@@ -20,7 +20,7 @@ use crate::context::Context;
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
-use crate::log::{LogExt, error, info, warn};
+use crate::log::{LogExt, warn};
 use crate::pgp;
 use crate::qr::DCBACKUP_VERSION;
 use crate::sql;
@@ -377,7 +377,15 @@ async fn import_backup_stream_inner<R: tokio::io::AsyncRead + Unpin>(
         res = check_backup_version(context).await;
     }
     if res.is_ok() {
-        res = adjust_bcc_self(context).await;
+        // All recent backups have `bcc_self` set to "1" before export.
+        //
+        // Setting `bcc_self` to "1" on export was introduced on 2024-12-17
+        // in commit 21664125d798021be75f47d5b0d5006d338b4531
+        //
+        // We additionally try to set `bcc_self` to "1" after import here
+        // for compatibility with older backups,
+        // but eventually this code can be removed.
+        res = context.set_config(Config::BccSelf, Some("1")).await;
     }
     fs::remove_file(unpacked_database)
         .await
@@ -751,7 +759,7 @@ async fn export_database(
         .to_str()
         .with_context(|| format!("path {} is not valid unicode", dest.display()))?;
 
-    adjust_bcc_self(context).await?;
+    context.set_config(Config::BccSelf, Some("1")).await?;
     context
         .sql
         .set_raw_config_int("backup_time", timestamp)
@@ -785,18 +793,6 @@ async fn export_database(
         .await
 }
 
-/// Sets `Config::BccSelf` (and `DeleteServerAfter` to "never" in effect) if needed so that new
-/// messages are present on the server after a backup restoration or available for all devices in
-/// multi-device case. NB: Calling this after a backup import isn't reliable as we can crash in
-/// between, but this is a problem only for old backups, new backups already have `BccSelf` set if
-/// necessary.
-async fn adjust_bcc_self(context: &Context) -> Result<()> {
-    if context.is_chatmail().await? && !context.config_exists(Config::BccSelf).await? {
-        context.set_config(Config::BccSelf, Some("1")).await?;
-    }
-    Ok(())
-}
-
 async fn check_backup_version(context: &Context) -> Result<()> {
     let version = (context.sql.get_raw_config_int("backup_version").await?).unwrap_or(2);
     ensure!(

src/imex/key_transfer.rs

@@ -93,7 +93,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
     let private_key = load_self_secret_key(context).await?;
     let ac_headers = Some(("Autocrypt-Prefer-Encrypt", "mutual"));
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(passphrase, private_key_asc.into_bytes())
+    let encr = pgp::symm_encrypt_autocrypt_setup(passphrase, private_key_asc.into_bytes())
         .await?
         .replace('\n', "\r\n");
 

src/imex/transfer.rs

@@ -41,7 +41,7 @@ use tokio_util::sync::CancellationToken;
 use crate::chat::add_device_msg;
 use crate::context::Context;
 use crate::imex::BlobDirContents;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::message::Message;
 use crate::qr::Qr;
 use crate::stock_str::backup_transfer_msg_body;
@@ -69,7 +69,7 @@ pub struct BackupProvider {
     _endpoint: Endpoint,
 
     /// iroh address.
-    node_addr: iroh::EndpointAddr,
+    node_addr: iroh::NodeAddr,
 
     /// Authentication token that should be submitted
     /// to retrieve the backup.
@@ -96,11 +96,12 @@ impl BackupProvider {
     pub async fn prepare(context: &Context) -> Result<Self> {
         let relay_mode = RelayMode::Disabled;
         let endpoint = Endpoint::builder()
+            .tls_x509() // For compatibility with iroh <0.34.0
             .alpns(vec![BACKUP_ALPN.to_vec()])
             .relay_mode(relay_mode)
             .bind()
             .await?;
-        let node_addr = endpoint.addr();
+        let node_addr = endpoint.node_addr().await?;
 
         // Acquire global "ongoing" mutex.
         let cancel_token = context.alloc_ongoing().await?;
@@ -297,12 +298,16 @@ impl Future for BackupProvider {
 
 pub async fn get_backup2(
     context: &Context,
-    node_addr: iroh::EndpointAddr,
+    node_addr: iroh::NodeAddr,
     auth_token: String,
 ) -> Result<()> {
     let relay_mode = RelayMode::Disabled;
 
-    let endpoint = Endpoint::builder().relay_mode(relay_mode).bind().await?;
+    let endpoint = Endpoint::builder()
+        .tls_x509() // For compatibility with iroh <0.34.0
+        .relay_mode(relay_mode)
+        .bind()
+        .await?;
 
     let conn = endpoint.connect(node_addr, BACKUP_ALPN).await?;
     let (mut send_stream, mut recv_stream) = conn.open_bi().await?;

src/internals_for_benches.rs

@@ -0,0 +1,38 @@
+//! Re-exports of `pub(crate)` functions that are needed for benchmarks.
+#![allow(missing_docs)] // Not necessary to put a doc comment on the pub functions here
+
+use anyhow::Result;
+use deltachat_contact_tools::EmailAddress;
+
+use crate::chat::ChatId;
+use crate::context::Context;
+use crate::key;
+use crate::key::DcKey;
+use crate::mimeparser::MimeMessage;
+use crate::pgp;
+use crate::pgp::KeyPair;
+
+pub fn key_from_asc(data: &str) -> Result<key::SignedSecretKey> {
+    key::SignedSecretKey::from_asc(data)
+}
+
+pub async fn store_self_keypair(context: &Context, keypair: &KeyPair) -> Result<()> {
+    key::store_self_keypair(context, keypair).await
+}
+
+pub async fn parse_and_get_text(context: &Context, imf_raw: &[u8]) -> Result<String> {
+    let mime_parser = MimeMessage::from_bytes(context, imf_raw, None).await?;
+    Ok(mime_parser.parts.into_iter().next().unwrap().msg)
+}
+
+pub async fn save_broadcast_secret(context: &Context, chat_id: ChatId, secret: &str) -> Result<()> {
+    crate::chat::save_broadcast_secret(context, chat_id, secret).await
+}
+
+pub fn create_dummy_keypair(addr: &str) -> Result<KeyPair> {
+    pgp::create_keypair(EmailAddress::new(addr)?)
+}
+
+pub fn create_broadcast_secret() -> String {
+    crate::tools::create_broadcast_secret()
+}

src/key.rs

@@ -15,7 +15,7 @@ use tokio::runtime::Handle;
 
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::{LogExt, info};
+use crate::log::LogExt;
 use crate::pgp::KeyPair;
 use crate::tools::{self, time_elapsed};
 
@@ -159,13 +159,15 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
 pub(crate) async fn load_self_public_keyring(context: &Context) -> Result<Vec<SignedPublicKey>> {
     let keys = context
         .sql
-        .query_map(
+        .query_map_vec(
             r#"SELECT public_key
                FROM keypairs
                ORDER BY id=(SELECT value FROM config WHERE keyname='key_id') DESC"#,
             (),
-            |row| row.get::<_, Vec<u8>>(0),
-            |keys| keys.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+            |row| {
+                let public_key_bytes: Vec<u8> = row.get(0)?;
+                Ok(public_key_bytes)
+            },
         )
         .await?
         .into_iter()
@@ -232,13 +234,15 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
 pub(crate) async fn load_self_secret_keyring(context: &Context) -> Result<Vec<SignedSecretKey>> {
     let keys = context
         .sql
-        .query_map(
+        .query_map_vec(
             r#"SELECT private_key
                FROM keypairs
                ORDER BY id=(SELECT value FROM config WHERE keyname='key_id') DESC"#,
             (),
-            |row| row.get::<_, Vec<u8>>(0),
-            |keys| keys.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+            |row| {
+                let bytes: Vec<u8> = row.get(0)?;
+                Ok(bytes)
+            },
         )
         .await?
         .into_iter()

src/lib.rs

@@ -75,7 +75,10 @@ mod mimefactory;
 pub mod mimeparser;
 pub mod oauth2;
 mod param;
+#[cfg(not(feature = "internals"))]
 mod pgp;
+#[cfg(feature = "internals")]
+pub mod pgp;
 pub mod provider;
 pub mod qr;
 pub mod qr_code_generator;
@@ -89,6 +92,7 @@ pub mod stock_str;
 mod sync;
 mod timesmearing;
 mod token;
+mod transport;
 mod update_helper;
 pub mod webxdc;
 #[macro_use]
@@ -98,7 +102,7 @@ pub mod color;
 pub mod html;
 pub mod net;
 pub mod plaintext;
-mod push;
+pub mod push;
 mod stats;
 pub use stats::SecurejoinSource;
 pub use stats::SecurejoinUiPath;
@@ -112,6 +116,9 @@ pub mod accounts;
 pub mod peer_channels;
 pub mod reaction;
 
+#[cfg(feature = "internals")]
+pub mod internals_for_benches;
+
 /// If set IMAP/incoming and SMTP/outgoing MIME messages will be printed.
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 

src/location.rs

@@ -22,7 +22,7 @@ use crate::constants::DC_CHAT_ID_TRASH;
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::tools::{duration_to_str, time};
@@ -348,7 +348,10 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         .query_map_vec(
             "SELECT id FROM chats WHERE locations_send_until>?;",
             (now,),
-            |row| row.get::<_, i32>(0),
+            |row| {
+                let id: i32 = row.get(0)?;
+                Ok(id)
+            },
         )
         .await?;
 

src/log.rs

@@ -23,8 +23,6 @@ macro_rules! info {
     }};
 }
 
-pub(crate) use info;
-
 // Workaround for <https://github.com/rust-lang/rust/issues/133708>.
 #[macro_use]
 mod warn_macro_mod {
@@ -60,8 +58,6 @@ macro_rules! error {
     }};
 }
 
-pub(crate) use error;
-
 impl Context {
     /// Set last error string.
     /// Implemented as blocking as used from macros in different, not always async blocks.

src/login_param.rs

@@ -1,21 +1,22 @@
 //! # Login parameters.
+//!
+//! Login parameters are entered by the user
+//! to configure a new transport.
+//! Login parameters may also be entered
+//! implicitly by scanning a QR code
+//! of `dcaccount:` or `dclogin:` scheme.
 
 use std::fmt;
 
-use anyhow::{Context as _, Result, bail, ensure, format_err};
-use deltachat_contact_tools::{EmailAddress, addr_cmp, addr_normalize};
+use anyhow::{Context as _, Result};
 use num_traits::ToPrimitive as _;
 use serde::{Deserialize, Serialize};
 
 use crate::config::Config;
-use crate::configure::server_params::{ServerParams, expand_param_vector};
 use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_OAUTH2};
 use crate::context::Context;
-use crate::net::load_connection_timestamp;
 pub use crate::net::proxy::ProxyConfig;
 pub use crate::provider::Socket;
-use crate::provider::{Protocol, Provider, UsernamePattern, get_provider_by_id};
-use crate::sql::Sql;
 use crate::tools::ToOption;
 
 /// User-entered setting for certificate checks.
@@ -55,45 +56,6 @@ pub enum EnteredCertificateChecks {
     AcceptInvalidCertificates2 = 3,
 }
 
-/// Values saved into `imap_certificate_checks`.
-#[derive(
-    Copy, Clone, Debug, Display, FromPrimitive, ToPrimitive, PartialEq, Eq, Serialize, Deserialize,
-)]
-#[repr(u32)]
-#[strum(serialize_all = "snake_case")]
-pub(crate) enum ConfiguredCertificateChecks {
-    /// Use configuration from the provider database.
-    /// If there is no provider database setting for certificate checks,
-    /// accept invalid certificates.
-    ///
-    /// Must not be saved by new versions.
-    ///
-    /// Previous Delta Chat versions before core 1.133.0
-    /// stored this in `configured_imap_certificate_checks`
-    /// if Automatic configuration
-    /// was selected, configuration with strict TLS checks failed
-    /// and configuration without strict TLS checks succeeded.
-    OldAutomatic = 0,
-
-    /// Ensure that TLS certificate is valid for the server hostname.
-    Strict = 1,
-
-    /// Accept certificates that are expired, self-signed
-    /// or otherwise not valid for the server hostname.
-    AcceptInvalidCertificates = 2,
-
-    /// Accept certificates that are expired, self-signed
-    /// or otherwise not valid for the server hostname.
-    ///
-    /// Alias to `AcceptInvalidCertificates` for compatibility.
-    AcceptInvalidCertificates2 = 3,
-
-    /// Use configuration from the provider database.
-    /// If there is no provider database setting for certificate checks,
-    /// apply strict checks to TLS certificates.
-    Automatic = 4,
-}
-
 /// Login parameters for a single server, either IMAP or SMTP
 #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct EnteredServerLoginParam {
@@ -333,584 +295,20 @@ fn unset_empty(s: &str) -> &str {
     if s.is_empty() { "unset" } else { s }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct ConnectionCandidate {
-    /// Server hostname or IP address.
-    pub host: String,
-
-    /// Server port.
-    pub port: u16,
-
-    /// Transport layer security.
-    pub security: ConnectionSecurity,
-}
-
-impl fmt::Display for ConnectionCandidate {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{}:{}", &self.host, self.port, self.security)?;
-        Ok(())
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) enum ConnectionSecurity {
-    /// Implicit TLS.
-    Tls,
-
-    // STARTTLS.
-    Starttls,
-
-    /// Plaintext.
-    Plain,
-}
-
-impl fmt::Display for ConnectionSecurity {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Tls => write!(f, "tls")?,
-            Self::Starttls => write!(f, "starttls")?,
-            Self::Plain => write!(f, "plain")?,
-        }
-        Ok(())
-    }
-}
-
-impl TryFrom<Socket> for ConnectionSecurity {
-    type Error = anyhow::Error;
-
-    fn try_from(socket: Socket) -> Result<Self> {
-        match socket {
-            Socket::Automatic => Err(format_err!("Socket security is not configured")),
-            Socket::Ssl => Ok(Self::Tls),
-            Socket::Starttls => Ok(Self::Starttls),
-            Socket::Plain => Ok(Self::Plain),
-        }
-    }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-pub(crate) struct ConfiguredServerLoginParam {
-    pub connection: ConnectionCandidate,
-
-    /// Username.
-    pub user: String,
-}
-
-impl fmt::Display for ConfiguredServerLoginParam {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}:{}", self.connection, &self.user)?;
-        Ok(())
-    }
-}
-
-pub(crate) async fn prioritize_server_login_params(
-    sql: &Sql,
-    params: &[ConfiguredServerLoginParam],
-    alpn: &str,
-) -> Result<Vec<ConfiguredServerLoginParam>> {
-    let mut res: Vec<(Option<i64>, ConfiguredServerLoginParam)> = Vec::with_capacity(params.len());
-    for param in params {
-        let timestamp = load_connection_timestamp(
-            sql,
-            alpn,
-            &param.connection.host,
-            param.connection.port,
-            None,
-        )
-        .await?;
-        res.push((timestamp, param.clone()));
-    }
-    res.sort_by_key(|(ts, _param)| std::cmp::Reverse(*ts));
-    Ok(res.into_iter().map(|(_ts, param)| param).collect())
-}
-
-/// Login parameters saved to the database
-/// after successful configuration.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct ConfiguredLoginParam {
-    /// `From:` address that was used at the time of configuration.
-    pub addr: String,
-
-    pub imap: Vec<ConfiguredServerLoginParam>,
-
-    // Custom IMAP user.
-    //
-    // This overwrites autoconfig from the provider database
-    // if non-empty.
-    pub imap_user: String,
-
-    pub imap_password: String,
-
-    pub smtp: Vec<ConfiguredServerLoginParam>,
-
-    // Custom SMTP user.
-    //
-    // This overwrites autoconfig from the provider database
-    // if non-empty.
-    pub smtp_user: String,
-
-    pub smtp_password: String,
-
-    pub provider: Option<&'static Provider>,
-
-    /// TLS options: whether to allow invalid certificates and/or
-    /// invalid hostnames
-    pub certificate_checks: ConfiguredCertificateChecks,
-
-    /// If true, login via OAUTH2 (not recommended anymore)
-    pub oauth2: bool,
-}
-
-/// The representation of ConfiguredLoginParam in the database,
-/// saved as Json.
-#[derive(Debug, Serialize, Deserialize)]
-struct ConfiguredLoginParamJson {
-    pub addr: String,
-    pub imap: Vec<ConfiguredServerLoginParam>,
-    pub imap_user: String,
-    pub imap_password: String,
-    pub smtp: Vec<ConfiguredServerLoginParam>,
-    pub smtp_user: String,
-    pub smtp_password: String,
-    pub provider_id: Option<String>,
-    pub certificate_checks: ConfiguredCertificateChecks,
-    pub oauth2: bool,
-}
-
-impl fmt::Display for ConfiguredLoginParam {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let addr = &self.addr;
-        let provider_id = match self.provider {
-            Some(provider) => provider.id,
-            None => "none",
-        };
-        let certificate_checks = self.certificate_checks;
-        write!(f, "{addr} imap:[")?;
-        let mut first = true;
-        for imap in &self.imap {
-            if !first {
-                write!(f, ", ")?;
-            }
-            write!(f, "{imap}")?;
-            first = false;
-        }
-        write!(f, "] smtp:[")?;
-        let mut first = true;
-        for smtp in &self.smtp {
-            if !first {
-                write!(f, ", ")?;
-            }
-            write!(f, "{smtp}")?;
-            first = false;
-        }
-        write!(f, "] provider:{provider_id} cert_{certificate_checks}")?;
-        Ok(())
-    }
-}
-
-impl ConfiguredLoginParam {
-    /// Load configured account settings from the database.
-    ///
-    /// Returns `None` if account is not configured.
-    pub(crate) async fn load(context: &Context) -> Result<Option<Self>> {
-        let Some(self_addr) = context.get_config(Config::ConfiguredAddr).await? else {
-            return Ok(None);
-        };
-
-        let json: Option<String> = context
-            .sql
-            .query_get_value(
-                "SELECT configured_param FROM transports WHERE addr=?",
-                (&self_addr,),
-            )
-            .await?;
-        if let Some(json) = json {
-            Ok(Some(Self::from_json(&json)?))
-        } else {
-            bail!("Self address {self_addr} doesn't have a corresponding transport");
-        }
-    }
-
-    /// Loads legacy configured param. Only used for tests and the migration.
-    pub(crate) async fn load_legacy(context: &Context) -> Result<Option<Self>> {
-        if !context.get_config_bool(Config::Configured).await? {
-            return Ok(None);
-        }
-
-        let addr = context
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .unwrap_or_default()
-            .trim()
-            .to_string();
-
-        let certificate_checks: ConfiguredCertificateChecks = if let Some(certificate_checks) =
-            context
-                .get_config_parsed::<i32>(Config::ConfiguredImapCertificateChecks)
-                .await?
-        {
-            num_traits::FromPrimitive::from_i32(certificate_checks)
-                .context("Invalid configured_imap_certificate_checks value")?
-        } else {
-            // This is true for old accounts configured using C core
-            // which did not check TLS certificates.
-            ConfiguredCertificateChecks::OldAutomatic
-        };
-
-        let send_pw = context
-            .get_config(Config::ConfiguredSendPw)
-            .await?
-            .context("SMTP password is not configured")?;
-        let mail_pw = context
-            .get_config(Config::ConfiguredMailPw)
-            .await?
-            .context("IMAP password is not configured")?;
-
-        let server_flags = context
-            .get_config_parsed::<i32>(Config::ConfiguredServerFlags)
-            .await?
-            .unwrap_or_default();
-        let oauth2 = matches!(server_flags & DC_LP_AUTH_FLAGS, DC_LP_AUTH_OAUTH2);
-
-        let provider = context.get_configured_provider().await?;
-
-        let imap;
-        let smtp;
-
-        let mail_user = context
-            .get_config(Config::ConfiguredMailUser)
-            .await?
-            .unwrap_or_default();
-        let send_user = context
-            .get_config(Config::ConfiguredSendUser)
-            .await?
-            .unwrap_or_default();
-
-        if let Some(provider) = provider {
-            let parsed_addr = EmailAddress::new(&addr).context("Bad email-address")?;
-            let addr_localpart = parsed_addr.local;
-
-            if provider.server.is_empty() {
-                let servers = vec![
-                    ServerParams {
-                        protocol: Protocol::Imap,
-                        hostname: context
-                            .get_config(Config::ConfiguredMailServer)
-                            .await?
-                            .unwrap_or_default(),
-                        port: context
-                            .get_config_parsed::<u16>(Config::ConfiguredMailPort)
-                            .await?
-                            .unwrap_or_default(),
-                        socket: context
-                            .get_config_parsed::<i32>(Config::ConfiguredMailSecurity)
-                            .await?
-                            .and_then(num_traits::FromPrimitive::from_i32)
-                            .unwrap_or_default(),
-                        username: mail_user.clone(),
-                    },
-                    ServerParams {
-                        protocol: Protocol::Smtp,
-                        hostname: context
-                            .get_config(Config::ConfiguredSendServer)
-                            .await?
-                            .unwrap_or_default(),
-                        port: context
-                            .get_config_parsed::<u16>(Config::ConfiguredSendPort)
-                            .await?
-                            .unwrap_or_default(),
-                        socket: context
-                            .get_config_parsed::<i32>(Config::ConfiguredSendSecurity)
-                            .await?
-                            .and_then(num_traits::FromPrimitive::from_i32)
-                            .unwrap_or_default(),
-                        username: send_user.clone(),
-                    },
-                ];
-                let servers = expand_param_vector(servers, &addr, &parsed_addr.domain);
-                imap = servers
-                    .iter()
-                    .filter_map(|params| {
-                        let Ok(security) = params.socket.try_into() else {
-                            return None;
-                        };
-                        if params.protocol == Protocol::Imap {
-                            Some(ConfiguredServerLoginParam {
-                                connection: ConnectionCandidate {
-                                    host: params.hostname.clone(),
-                                    port: params.port,
-                                    security,
-                                },
-                                user: params.username.clone(),
-                            })
-                        } else {
-                            None
-                        }
-                    })
-                    .collect();
-                smtp = servers
-                    .iter()
-                    .filter_map(|params| {
-                        let Ok(security) = params.socket.try_into() else {
-                            return None;
-                        };
-                        if params.protocol == Protocol::Smtp {
-                            Some(ConfiguredServerLoginParam {
-                                connection: ConnectionCandidate {
-                                    host: params.hostname.clone(),
-                                    port: params.port,
-                                    security,
-                                },
-                                user: params.username.clone(),
-                            })
-                        } else {
-                            None
-                        }
-                    })
-                    .collect();
-            } else {
-                imap = provider
-                    .server
-                    .iter()
-                    .filter_map(|server| {
-                        if server.protocol != Protocol::Imap {
-                            return None;
-                        }
-
-                        let Ok(security) = server.socket.try_into() else {
-                            return None;
-                        };
-
-                        Some(ConfiguredServerLoginParam {
-                            connection: ConnectionCandidate {
-                                host: server.hostname.to_string(),
-                                port: server.port,
-                                security,
-                            },
-                            user: if !mail_user.is_empty() {
-                                mail_user.clone()
-                            } else {
-                                match server.username_pattern {
-                                    UsernamePattern::Email => addr.to_string(),
-                                    UsernamePattern::Emaillocalpart => addr_localpart.clone(),
-                                }
-                            },
-                        })
-                    })
-                    .collect();
-                smtp = provider
-                    .server
-                    .iter()
-                    .filter_map(|server| {
-                        if server.protocol != Protocol::Smtp {
-                            return None;
-                        }
-
-                        let Ok(security) = server.socket.try_into() else {
-                            return None;
-                        };
-
-                        Some(ConfiguredServerLoginParam {
-                            connection: ConnectionCandidate {
-                                host: server.hostname.to_string(),
-                                port: server.port,
-                                security,
-                            },
-                            user: if !send_user.is_empty() {
-                                send_user.clone()
-                            } else {
-                                match server.username_pattern {
-                                    UsernamePattern::Email => addr.to_string(),
-                                    UsernamePattern::Emaillocalpart => addr_localpart.clone(),
-                                }
-                            },
-                        })
-                    })
-                    .collect();
-            }
-        } else if let (Some(configured_mail_servers), Some(configured_send_servers)) = (
-            context.get_config(Config::ConfiguredImapServers).await?,
-            context.get_config(Config::ConfiguredSmtpServers).await?,
-        ) {
-            imap = serde_json::from_str(&configured_mail_servers)
-                .context("Failed to parse configured IMAP servers")?;
-            smtp = serde_json::from_str(&configured_send_servers)
-                .context("Failed to parse configured SMTP servers")?;
-        } else {
-            // Load legacy settings storing a single IMAP and single SMTP server.
-            let mail_server = context
-                .get_config(Config::ConfiguredMailServer)
-                .await?
-                .unwrap_or_default();
-            let mail_port = context
-                .get_config_parsed::<u16>(Config::ConfiguredMailPort)
-                .await?
-                .unwrap_or_default();
-
-            let mail_security: Socket = context
-                .get_config_parsed::<i32>(Config::ConfiguredMailSecurity)
-                .await?
-                .and_then(num_traits::FromPrimitive::from_i32)
-                .unwrap_or_default();
-
-            let send_server = context
-                .get_config(Config::ConfiguredSendServer)
-                .await?
-                .context("SMTP server is not configured")?;
-            let send_port = context
-                .get_config_parsed::<u16>(Config::ConfiguredSendPort)
-                .await?
-                .unwrap_or_default();
-            let send_security: Socket = context
-                .get_config_parsed::<i32>(Config::ConfiguredSendSecurity)
-                .await?
-                .and_then(num_traits::FromPrimitive::from_i32)
-                .unwrap_or_default();
-
-            imap = vec![ConfiguredServerLoginParam {
-                connection: ConnectionCandidate {
-                    host: mail_server,
-                    port: mail_port,
-                    security: mail_security.try_into()?,
-                },
-                user: mail_user.clone(),
-            }];
-            smtp = vec![ConfiguredServerLoginParam {
-                connection: ConnectionCandidate {
-                    host: send_server,
-                    port: send_port,
-                    security: send_security.try_into()?,
-                },
-                user: send_user.clone(),
-            }];
-        }
-
-        Ok(Some(ConfiguredLoginParam {
-            addr,
-            imap,
-            imap_user: mail_user,
-            imap_password: mail_pw,
-            smtp,
-            smtp_user: send_user,
-            smtp_password: send_pw,
-            certificate_checks,
-            provider,
-            oauth2,
-        }))
-    }
-
-    pub(crate) async fn save_to_transports_table(
-        self,
-        context: &Context,
-        entered_param: &EnteredLoginParam,
-    ) -> Result<()> {
-        let addr = addr_normalize(&self.addr);
-        let provider_id = self.provider.map(|provider| provider.id);
-        let configured_addr = context.get_config(Config::ConfiguredAddr).await?;
-        if let Some(configured_addr) = &configured_addr {
-            ensure!(
-                addr_cmp(configured_addr, &addr),
-                "Adding a second transport is not supported right now."
-            );
-        }
-        context
-            .sql
-            .execute(
-                "INSERT INTO transports (addr, entered_param, configured_param)
-                VALUES (?, ?, ?)
-                ON CONFLICT (addr)
-                DO UPDATE SET entered_param=excluded.entered_param, configured_param=excluded.configured_param",
-                (
-                    self.addr.clone(),
-                    serde_json::to_string(entered_param)?,
-                    self.into_json()?,
-                ),
-            )
-            .await?;
-        if configured_addr.is_none() {
-            // If there is no transport yet, set the new transport as the primary one
-            context
-                .sql
-                .set_raw_config(Config::ConfiguredProvider.as_ref(), provider_id)
-                .await?;
-            context
-                .sql
-                .set_raw_config(Config::ConfiguredAddr.as_ref(), Some(&addr))
-                .await?;
-        }
-        Ok(())
-    }
-
-    pub(crate) fn from_json(json: &str) -> Result<Self> {
-        let json: ConfiguredLoginParamJson = serde_json::from_str(json)?;
-
-        let provider = json.provider_id.and_then(|id| get_provider_by_id(&id));
-
-        Ok(ConfiguredLoginParam {
-            addr: json.addr,
-            imap: json.imap,
-            imap_user: json.imap_user,
-            imap_password: json.imap_password,
-            smtp: json.smtp,
-            smtp_user: json.smtp_user,
-            smtp_password: json.smtp_password,
-            provider,
-            certificate_checks: json.certificate_checks,
-            oauth2: json.oauth2,
-        })
-    }
-
-    pub(crate) fn into_json(self) -> Result<String> {
-        let json = ConfiguredLoginParamJson {
-            addr: self.addr,
-            imap: self.imap,
-            imap_user: self.imap_user,
-            imap_password: self.imap_password,
-            smtp: self.smtp,
-            smtp_user: self.smtp_user,
-            smtp_password: self.smtp_password,
-            provider_id: self.provider.map(|p| p.id.to_string()),
-            certificate_checks: self.certificate_checks,
-            oauth2: self.oauth2,
-        };
-        Ok(serde_json::to_string(&json)?)
-    }
-
-    pub(crate) fn strict_tls(&self, connected_through_proxy: bool) -> bool {
-        let provider_strict_tls = self.provider.map(|provider| provider.opt.strict_tls);
-        match self.certificate_checks {
-            ConfiguredCertificateChecks::OldAutomatic => {
-                provider_strict_tls.unwrap_or(connected_through_proxy)
-            }
-            ConfiguredCertificateChecks::Automatic => provider_strict_tls.unwrap_or(true),
-            ConfiguredCertificateChecks::Strict => true,
-            ConfiguredCertificateChecks::AcceptInvalidCertificates
-            | ConfiguredCertificateChecks::AcceptInvalidCertificates2 => false,
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::log::LogExt as _;
-    use crate::provider::get_provider_by_id;
     use crate::test_utils::TestContext;
     use pretty_assertions::assert_eq;
 
     #[test]
-    fn test_certificate_checks_display() {
+    fn test_entered_certificate_checks_display() {
         use std::string::ToString;
 
         assert_eq!(
             "accept_invalid_certificates".to_string(),
             EnteredCertificateChecks::AcceptInvalidCertificates.to_string()
         );
-
-        assert_eq!(
-            "accept_invalid_certificates".to_string(),
-            ConfiguredCertificateChecks::AcceptInvalidCertificates.to_string()
-        );
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -976,267 +374,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_save_load_login_param() -> Result<()> {
-        let t = TestContext::new().await;
-
-        let param = ConfiguredLoginParam {
-            addr: "alice@example.org".to_string(),
-            imap: vec![ConfiguredServerLoginParam {
-                connection: ConnectionCandidate {
-                    host: "imap.example.com".to_string(),
-                    port: 123,
-                    security: ConnectionSecurity::Starttls,
-                },
-                user: "alice".to_string(),
-            }],
-            imap_user: "".to_string(),
-            imap_password: "foo".to_string(),
-            smtp: vec![ConfiguredServerLoginParam {
-                connection: ConnectionCandidate {
-                    host: "smtp.example.com".to_string(),
-                    port: 456,
-                    security: ConnectionSecurity::Tls,
-                },
-                user: "alice@example.org".to_string(),
-            }],
-            smtp_user: "".to_string(),
-            smtp_password: "bar".to_string(),
-            provider: None,
-            certificate_checks: ConfiguredCertificateChecks::Strict,
-            oauth2: false,
-        };
-
-        param
-            .clone()
-            .save_to_transports_table(&t, &EnteredLoginParam::default())
-            .await?;
-        let expected_param = r#"{"addr":"alice@example.org","imap":[{"connection":{"host":"imap.example.com","port":123,"security":"Starttls"},"user":"alice"}],"imap_user":"","imap_password":"foo","smtp":[{"connection":{"host":"smtp.example.com","port":456,"security":"Tls"},"user":"alice@example.org"}],"smtp_user":"","smtp_password":"bar","provider_id":null,"certificate_checks":"Strict","oauth2":false}"#;
-        assert_eq!(
-            t.sql
-                .query_get_value::<String>("SELECT configured_param FROM transports", ())
-                .await?
-                .unwrap(),
-            expected_param
-        );
-        assert_eq!(t.is_configured().await?, true);
-        let loaded = ConfiguredLoginParam::load(&t).await?.unwrap();
-        assert_eq!(param, loaded);
-
-        // Legacy ConfiguredImapCertificateChecks config is ignored
-        t.set_config(Config::ConfiguredImapCertificateChecks, Some("999"))
-            .await?;
-        assert!(ConfiguredLoginParam::load(&t).await.is_ok());
-
-        // Test that we don't panic on unknown ConfiguredImapCertificateChecks values.
-        let wrong_param = expected_param.replace("Strict", "Stricct");
-        assert_ne!(expected_param, wrong_param);
-        t.sql
-            .execute("UPDATE transports SET configured_param=?", (wrong_param,))
-            .await?;
-        assert!(ConfiguredLoginParam::load(&t).await.is_err());
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_posteo_alias() -> Result<()> {
-        let t = TestContext::new().await;
-
-        let user = "alice@posteo.de";
-
-        // Alice has old config with "alice@posteo.at" address
-        // and "alice@posteo.de" username.
-        t.set_config(Config::Configured, Some("1")).await?;
-        t.set_config(Config::ConfiguredProvider, Some("posteo"))
-            .await?;
-        t.sql
-            .set_raw_config(Config::ConfiguredAddr.as_ref(), Some("alice@posteo.at"))
-            .await?;
-        t.set_config(Config::ConfiguredMailServer, Some("posteo.de"))
-            .await?;
-        t.set_config(Config::ConfiguredMailPort, Some("993"))
-            .await?;
-        t.set_config(Config::ConfiguredMailSecurity, Some("1"))
-            .await?; // TLS
-        t.set_config(Config::ConfiguredMailUser, Some(user)).await?;
-        t.set_config(Config::ConfiguredMailPw, Some("foobarbaz"))
-            .await?;
-        t.set_config(Config::ConfiguredImapCertificateChecks, Some("1"))
-            .await?; // Strict
-        t.set_config(Config::ConfiguredSendServer, Some("posteo.de"))
-            .await?;
-        t.set_config(Config::ConfiguredSendPort, Some("465"))
-            .await?;
-        t.set_config(Config::ConfiguredSendSecurity, Some("1"))
-            .await?; // TLS
-        t.set_config(Config::ConfiguredSendUser, Some(user)).await?;
-        t.set_config(Config::ConfiguredSendPw, Some("foobarbaz"))
-            .await?;
-        t.set_config(Config::ConfiguredSmtpCertificateChecks, Some("1"))
-            .await?; // Strict
-        t.set_config(Config::ConfiguredServerFlags, Some("0"))
-            .await?;
-
-        let param = ConfiguredLoginParam {
-            addr: "alice@posteo.at".to_string(),
-            imap: vec![
-                ConfiguredServerLoginParam {
-                    connection: ConnectionCandidate {
-                        host: "posteo.de".to_string(),
-                        port: 993,
-                        security: ConnectionSecurity::Tls,
-                    },
-                    user: user.to_string(),
-                },
-                ConfiguredServerLoginParam {
-                    connection: ConnectionCandidate {
-                        host: "posteo.de".to_string(),
-                        port: 143,
-                        security: ConnectionSecurity::Starttls,
-                    },
-                    user: user.to_string(),
-                },
-            ],
-            imap_user: "alice@posteo.de".to_string(),
-            imap_password: "foobarbaz".to_string(),
-            smtp: vec![
-                ConfiguredServerLoginParam {
-                    connection: ConnectionCandidate {
-                        host: "posteo.de".to_string(),
-                        port: 465,
-                        security: ConnectionSecurity::Tls,
-                    },
-                    user: user.to_string(),
-                },
-                ConfiguredServerLoginParam {
-                    connection: ConnectionCandidate {
-                        host: "posteo.de".to_string(),
-                        port: 587,
-                        security: ConnectionSecurity::Starttls,
-                    },
-                    user: user.to_string(),
-                },
-            ],
-            smtp_user: "alice@posteo.de".to_string(),
-            smtp_password: "foobarbaz".to_string(),
-            provider: get_provider_by_id("posteo"),
-            certificate_checks: ConfiguredCertificateChecks::Strict,
-            oauth2: false,
-        };
-
-        let loaded = ConfiguredLoginParam::load_legacy(&t).await?.unwrap();
-        assert_eq!(loaded, param);
-
-        migrate_configured_login_param(&t).await;
-        let loaded = ConfiguredLoginParam::load(&t).await?.unwrap();
-        assert_eq!(loaded, param);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_empty_server_list_legacy() -> Result<()> {
-        // Find a provider that does not have server list set.
-        //
-        // There is at least one such provider in the provider database.
-        let (domain, provider) = crate::provider::data::PROVIDER_DATA
-            .iter()
-            .find(|(_domain, provider)| provider.server.is_empty())
-            .unwrap();
-
-        let t = TestContext::new().await;
-
-        let addr = format!("alice@{domain}");
-
-        t.set_config(Config::Configured, Some("1")).await?;
-        t.set_config(Config::ConfiguredProvider, Some(provider.id))
-            .await?;
-        t.sql
-            .set_raw_config(Config::ConfiguredAddr.as_ref(), Some(&addr))
-            .await?;
-        t.set_config(Config::ConfiguredMailPw, Some("foobarbaz"))
-            .await?;
-        t.set_config(Config::ConfiguredImapCertificateChecks, Some("1"))
-            .await?; // Strict
-        t.set_config(Config::ConfiguredSendPw, Some("foobarbaz"))
-            .await?;
-        t.set_config(Config::ConfiguredSmtpCertificateChecks, Some("1"))
-            .await?; // Strict
-        t.set_config(Config::ConfiguredServerFlags, Some("0"))
-            .await?;
-
-        let loaded = ConfiguredLoginParam::load_legacy(&t).await?.unwrap();
-        assert_eq!(loaded.provider, Some(*provider));
-        assert_eq!(loaded.imap.is_empty(), false);
-        assert_eq!(loaded.smtp.is_empty(), false);
-
-        migrate_configured_login_param(&t).await;
-
-        let loaded = ConfiguredLoginParam::load(&t).await?.unwrap();
-        assert_eq!(loaded.provider, Some(*provider));
-        assert_eq!(loaded.imap.is_empty(), false);
-        assert_eq!(loaded.smtp.is_empty(), false);
-
-        Ok(())
-    }
-
-    async fn migrate_configured_login_param(t: &TestContext) {
-        t.sql.execute("DROP TABLE transports;", ()).await.unwrap();
-        t.sql.set_raw_config_int("dbversion", 130).await.unwrap();
-        t.sql.run_migrations(t).await.log_err(t).ok();
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_empty_server_list() -> Result<()> {
-        // Find a provider that does not have server list set.
-        //
-        // There is at least one such provider in the provider database.
-        let (domain, provider) = crate::provider::data::PROVIDER_DATA
-            .iter()
-            .find(|(_domain, provider)| provider.server.is_empty())
-            .unwrap();
-
-        let t = TestContext::new().await;
-
-        let addr = format!("alice@{domain}");
-
-        ConfiguredLoginParam {
-            addr: addr.clone(),
-            imap: vec![ConfiguredServerLoginParam {
-                connection: ConnectionCandidate {
-                    host: "example.org".to_string(),
-                    port: 100,
-                    security: ConnectionSecurity::Tls,
-                },
-                user: addr.clone(),
-            }],
-            imap_user: addr.clone(),
-            imap_password: "foobarbaz".to_string(),
-            smtp: vec![ConfiguredServerLoginParam {
-                connection: ConnectionCandidate {
-                    host: "example.org".to_string(),
-                    port: 100,
-                    security: ConnectionSecurity::Tls,
-                },
-                user: addr.clone(),
-            }],
-            smtp_user: addr.clone(),
-            smtp_password: "foobarbaz".to_string(),
-            provider: Some(provider),
-            certificate_checks: ConfiguredCertificateChecks::Automatic,
-            oauth2: false,
-        }
-        .save_to_transports_table(&t, &EnteredLoginParam::default())
-        .await?;
-
-        let loaded = ConfiguredLoginParam::load(&t).await?.unwrap();
-        assert_eq!(loaded.provider, Some(*provider));
-        assert_eq!(loaded.imap.is_empty(), false);
-        assert_eq!(loaded.smtp.is_empty(), false);
-        assert_eq!(t.get_configured_provider().await?, Some(*provider));
-
-        Ok(())
-    }
 }

src/message.rs

@@ -24,7 +24,7 @@ use crate::ephemeral::{Timer as EphemeralTimer, start_ephemeral_timers_msgids};
 use crate::events::EventType;
 use crate::imap::markseen_on_imap_table;
 use crate::location::delete_poi_location;
-use crate::log::{error, info, warn};
+use crate::log::warn;
 use crate::mimeparser::{SystemMessage, parse_message_id};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
@@ -381,9 +381,11 @@ impl rusqlite::types::FromSql for MsgId {
     ToSql,
     Serialize,
     Deserialize,
+    Default,
 )]
 #[repr(u8)]
 pub(crate) enum MessengerMessage {
+    #[default]
     No = 0,
     Yes = 1,
 
@@ -391,12 +393,6 @@ pub(crate) enum MessengerMessage {
     Reply = 2,
 }
 
-impl Default for MessengerMessage {
-    fn default() -> Self {
-        Self::No
-    }
-}
-
 /// An object representing a single message in memory.
 /// The message object is not updated.
 /// If you want an update, you have to recreate the object.
@@ -1444,7 +1440,15 @@ pub(crate) fn guess_msgtype_from_path_suffix(path: &Path) -> Option<(Viewtype, &
     let extension: &str = &path.extension()?.to_str()?.to_lowercase();
     let info = match extension {
         // before using viewtype other than Viewtype::File,
-        // make sure, all target UIs support that type in the context of the used viewer/player.
+        // make sure, all target UIs support that type.
+        //
+        // it is a non-goal to support as many formats as possible in-app.
+        // additional parser come at security and maintainance costs and
+        // should only be added when strictly neccessary,
+        // eg. when a format comes from the camera app on a significant number of devices.
+        // it is okay, when eg. dragging some video from a browser results in a "File"
+        // for everyone, sender as well as all receivers.
+        //
         // if in doubt, it is better to default to Viewtype::File that passes handing to an external app.
         // (cmp. <https://developer.android.com/guide/topics/media/media-formats>)
         "3gp" => (Viewtype::Video, "video/3gpp"),
@@ -1507,7 +1511,7 @@ pub(crate) fn guess_msgtype_from_path_suffix(path: &Path) -> Option<(Viewtype, &
         "vcf" => (Viewtype::Vcard, "text/vcard"),
         "wav" => (Viewtype::Audio, "audio/wav"),
         "weba" => (Viewtype::File, "audio/webm"),
-        "webm" => (Viewtype::Video, "video/webm"),
+        "webm" => (Viewtype::File, "video/webm"), // not supported natively by iOS nor by SDWebImage
         "webp" => (Viewtype::Image, "image/webp"), // iOS via SDWebImage, Android since 4.0
         "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
         "xdc" => (Viewtype::Webxdc, "application/webxdc+zip"),

src/mimefactory.rs

@@ -3,7 +3,7 @@
 use std::collections::{BTreeSet, HashSet};
 use std::io::Cursor;
 
-use anyhow::{Context as _, Result, bail, ensure};
+use anyhow::{Context as _, Result, bail, format_err};
 use base64::Engine as _;
 use data_encoding::BASE32_NOPAD;
 use deltachat_contact_tools::sanitize_bidi_characters;
@@ -15,9 +15,9 @@ use tokio::fs;
 
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::blob::BlobObject;
-use crate::chat::{self, Chat};
+use crate::chat::{self, Chat, PARAM_BROADCAST_SECRET, load_broadcast_secret};
 use crate::config::Config;
-use crate::constants::ASM_SUBJECT;
+use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG};
 use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
@@ -27,7 +27,7 @@ use crate::ephemeral::Timer as EphemeralTimer;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, SignedPublicKey, self_fingerprint};
 use crate::location;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{SystemMessage, is_hidden};
 use crate::param::Param;
@@ -94,7 +94,7 @@ pub struct MimeFactory {
     /// to use for encryption.
     ///
     /// `None` if the message is not encrypted.
-    encryption_keys: Option<Vec<(String, SignedPublicKey)>>,
+    encryption_pubkeys: Option<Vec<(String, SignedPublicKey)>>,
 
     /// Vector of pairs of recipient name and address that goes into the `To` field.
     ///
@@ -182,7 +182,7 @@ impl MimeFactory {
         let now = time();
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
         let attach_profile_data = Self::should_attach_profile_data(&msg);
-        let undisclosed_recipients = chat.typ == Chattype::OutBroadcast;
+        let undisclosed_recipients = should_hide_recipients(&msg, &chat);
 
         let from_addr = context.get_primary_self_addr().await?;
         let config_displayname = context
@@ -208,14 +208,14 @@ impl MimeFactory {
         let mut recipient_ids = HashSet::new();
         let mut req_mdn = false;
 
-        let encryption_keys;
+        let encryption_pubkeys;
 
         let self_fingerprint = self_fingerprint(context).await?;
 
         if chat.is_self_talk() {
             to.push((from_displayname.to_string(), from_addr.to_string()));
 
-            encryption_keys = if msg.param.get_bool(Param::ForcePlaintext).unwrap_or(false) {
+            encryption_pubkeys = if msg.param.get_bool(Param::ForcePlaintext).unwrap_or(false) {
                 None
             } else {
                 // Encrypt, but only to self.
@@ -230,7 +230,7 @@ impl MimeFactory {
             recipients.push(list_post.to_string());
 
             // Do not encrypt messages to mailing lists.
-            encryption_keys = None;
+            encryption_pubkeys = None;
         } else {
             let email_to_remove = if msg.param.get_cmd() == SystemMessage::MemberRemovedFromGroup {
                 msg.param.get(Param::Arg)
@@ -290,6 +290,14 @@ impl MimeFactory {
                         for row in rows {
                             let (authname, addr, fingerprint, id, add_timestamp, remove_timestamp, public_key_bytes_opt) = row?;
 
+                            // In a broadcast channel, only send member-added/removed messages
+                            // to the affected member:
+                            if let Some(fp) = must_have_only_one_recipient(&msg, &chat) {
+                                if fp? != fingerprint {
+                                    continue;
+                                }
+                            }
+
                             let public_key_opt = if let Some(public_key_bytes) = &public_key_bytes_opt {
                                 Some(SignedPublicKey::from_slice(public_key_bytes)?)
                             } else {
@@ -329,7 +337,7 @@ impl MimeFactory {
 
                                 if let Some(public_key) = public_key_opt {
                                     keys.push((addr.clone(), public_key))
-                                } else if id != ContactId::SELF {
+                                } else if id != ContactId::SELF && !should_encrypt_symmetrically(&msg, &chat) {
                                     missing_key_addresses.insert(addr.clone());
                                     if is_encrypted {
                                         warn!(context, "Missing key for {addr}");
@@ -350,7 +358,7 @@ impl MimeFactory {
 
                                             if let Some(public_key) = public_key_opt {
                                                 keys.push((addr.clone(), public_key))
-                                            } else if id != ContactId::SELF {
+                                            } else if id != ContactId::SELF && !should_encrypt_symmetrically(&msg, &chat)  {
                                                 missing_key_addresses.insert(addr.clone());
                                                 if is_encrypted {
                                                     warn!(context, "Missing key for {addr}");
@@ -415,8 +423,10 @@ impl MimeFactory {
                 req_mdn = true;
             }
 
-            encryption_keys = if !is_encrypted {
+            encryption_pubkeys = if !is_encrypted {
                 None
+            } else if should_encrypt_symmetrically(&msg, &chat) {
+                Some(Vec::new())
             } else {
                 if keys.is_empty() && !recipients.is_empty() {
                     bail!("No recipient keys are available, cannot encrypt to {recipients:?}.");
@@ -474,7 +484,7 @@ impl MimeFactory {
             sender_displayname,
             selfstatus,
             recipients,
-            encryption_keys,
+            encryption_pubkeys,
             to,
             past_members,
             member_fingerprints,
@@ -503,7 +513,7 @@ impl MimeFactory {
         let timestamp = create_smeared_timestamp(context);
 
         let addr = contact.get_addr().to_string();
-        let encryption_keys = if contact.is_key_contact() {
+        let encryption_pubkeys = if contact.is_key_contact() {
             if let Some(key) = contact.public_key(context).await? {
                 Some(vec![(addr.clone(), key)])
             } else {
@@ -519,7 +529,7 @@ impl MimeFactory {
             sender_displayname: None,
             selfstatus: "".to_string(),
             recipients: vec![addr],
-            encryption_keys,
+            encryption_pubkeys,
             to: vec![("".to_string(), contact.get_addr().to_string())],
             past_members: vec![],
             member_fingerprints: vec![],
@@ -560,6 +570,10 @@ impl MimeFactory {
             //   messages are auto-sent unlike usual unencrypted messages.
             step == "vg-request-with-auth"
                 || step == "vc-request-with-auth"
+                // Note that for "vg-member-added"
+                // get_cmd() returns `MemberAddedToGroup` rather than `SecurejoinMessage`,
+                // so, it wouldn't actually be necessary to have them in the list here.
+                // Still, they are here for completeness.
                 || step == "vg-member-added"
                 || step == "vc-contact-confirm"
         }
@@ -812,16 +826,25 @@ impl MimeFactory {
             }
         }
 
-        if let Loaded::Message { chat, .. } = &self.loaded {
+        if let Loaded::Message { msg, chat } = &self.loaded {
             if chat.typ == Chattype::OutBroadcast || chat.typ == Chattype::InBroadcast {
                 headers.push((
-                    "List-ID",
+                    "Chat-List-ID",
                     mail_builder::headers::text::Text::new(format!(
                         "{} <{}>",
                         chat.name, chat.grpid
                     ))
                     .into(),
                 ));
+
+                if msg.param.get_cmd() == SystemMessage::MemberAddedToGroup {
+                    if let Some(secret) = msg.param.get(PARAM_BROADCAST_SECRET) {
+                        headers.push((
+                            "Chat-Broadcast-Secret",
+                            mail_builder::headers::text::Text::new(secret.to_string()).into(),
+                        ));
+                    }
+                }
             }
         }
 
@@ -872,7 +895,7 @@ impl MimeFactory {
             ));
         }
 
-        let is_encrypted = self.encryption_keys.is_some();
+        let is_encrypted = self.encryption_pubkeys.is_some();
 
         // Add ephemeral timer for non-MDN messages.
         // For MDNs it does not matter because they are not visible
@@ -998,6 +1021,12 @@ impl MimeFactory {
                 } else {
                     unprotected_headers.push(header.clone());
                 }
+            } else if header_name == "chat-broadcast-secret" {
+                if is_encrypted {
+                    protected_headers.push(header.clone());
+                } else {
+                    bail!("Message is unecrypted, cannot include broadcast secret");
+                }
             } else if is_encrypted && header_name == "date" {
                 protected_headers.push(header.clone());
 
@@ -1054,7 +1083,10 @@ impl MimeFactory {
             }
         }
 
-        let outer_message = if let Some(encryption_keys) = self.encryption_keys {
+        let use_std_header_protection = context
+            .get_config_bool(Config::StdHeaderProtectionComposing)
+            .await?;
+        let outer_message = if let Some(encryption_pubkeys) = self.encryption_pubkeys {
             // Store protected headers in the inner message.
             let message = protected_headers
                 .into_iter()
@@ -1069,17 +1101,33 @@ impl MimeFactory {
                     message.header(header, value)
                 });
 
+            if use_std_header_protection {
+                message = unprotected_headers
+                    .iter()
+                    // Structural headers shouldn't be added as "HP-Outer". They are defined in
+                    // <https://www.rfc-editor.org/rfc/rfc9787.html#structural-header-fields>.
+                    .filter(|(name, _)| {
+                        !(name.eq_ignore_ascii_case("mime-version")
+                            || name.eq_ignore_ascii_case("content-type")
+                            || name.eq_ignore_ascii_case("content-transfer-encoding")
+                            || name.eq_ignore_ascii_case("content-disposition"))
+                    })
+                    .fold(message, |message, (name, value)| {
+                        message.header(format!("HP-Outer: {name}"), value.clone())
+                    });
+            }
+
             // Add gossip headers in chats with multiple recipients
             let multiple_recipients =
-                encryption_keys.len() > 1 || context.get_config_bool(Config::BccSelf).await?;
+                encryption_pubkeys.len() > 1 || context.get_config_bool(Config::BccSelf).await?;
 
             let gossip_period = context.get_config_i64(Config::GossipPeriod).await?;
             let now = time();
 
             match &self.loaded {
                 Loaded::Message { chat, msg } => {
-                    if chat.typ != Chattype::OutBroadcast {
-                        for (addr, key) in &encryption_keys {
+                    if !should_hide_recipients(msg, chat) {
+                        for (addr, key) in &encryption_pubkeys {
                             let fingerprint = key.dc_fingerprint().hex();
                             let cmd = msg.param.get_cmd();
                             let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup
@@ -1158,7 +1206,13 @@ impl MimeFactory {
             for (h, v) in &mut message.headers {
                 if h == "Content-Type" {
                     if let mail_builder::headers::HeaderType::ContentType(ct) = v {
-                        *ct = ct.clone().attribute("protected-headers", "v1");
+                        let mut ct_new = ct.clone();
+                        ct_new = ct_new.attribute("protected-headers", "v1");
+                        if use_std_header_protection {
+                            ct_new = ct_new.attribute("hp", "cipher");
+                        }
+                        *ct = ct_new;
+                        break;
                     }
                 }
             }
@@ -1173,18 +1227,70 @@ impl MimeFactory {
                 Loaded::Mdn { .. } => true,
             };
 
-            // Encrypt to self unconditionally,
-            // even for a single-device setup.
-            let mut encryption_keyring = vec![encrypt_helper.public_key.clone()];
-            encryption_keyring.extend(encryption_keys.iter().map(|(_addr, key)| (*key).clone()));
+            let shared_secret: Option<String> = match &self.loaded {
+                Loaded::Message { chat, msg }
+                    if should_encrypt_with_broadcast_secret(msg, chat) =>
+                {
+                    let secret = load_broadcast_secret(context, chat.id).await?;
+                    if secret.is_none() {
+                        // If there is no shared secret yet
+                        // because this is an old broadcast channel,
+                        // created before we had symmetric encryption,
+                        // we show an error message.
+                        let text = BROADCAST_INCOMPATIBILITY_MSG;
+                        chat::add_info_msg(context, chat.id, text, time()).await?;
+                        bail!(text);
+                    }
+                    secret
+                }
+                _ => None,
+            };
+
+            // Do not anonymize OpenPGP recipients.
+            //
+            // This is disabled to avoid interoperability problems
+            // with old core versions <1.160.0 that do not support
+            // receiving messages with wildcard Key IDs:
+            // <https://github.com/chatmail/core/issues/7378>
+            //
+            // The option should be changed to true
+            // once new core versions are sufficiently deployed.
+            let anonymous_recipients = false;
+
+            if context.get_config_bool(Config::TestHooks).await? {
+                if let Some(hook) = &*context.pre_encrypt_mime_hook.lock() {
+                    message = hook(context, message);
+                }
+            }
+
+            let encrypted = if let Some(shared_secret) = shared_secret {
+                encrypt_helper
+                    .encrypt_symmetrically(context, &shared_secret, message, compress)
+                    .await?
+            } else {
+                // Asymmetric encryption
+
+                // Encrypt to self unconditionally,
+                // even for a single-device setup.
+                let mut encryption_keyring = vec![encrypt_helper.public_key.clone()];
+                encryption_keyring
+                    .extend(encryption_pubkeys.iter().map(|(_addr, key)| (*key).clone()));
+
+                encrypt_helper
+                    .encrypt(
+                        context,
+                        encryption_keyring,
+                        message,
+                        compress,
+                        anonymous_recipients,
+                    )
+                    .await?
+            };
 
             // XXX: additional newline is needed
             // to pass filtermail at
-            // <https://github.com/deltachat/chatmail/blob/4d915f9800435bf13057d41af8d708abd34dbfa8/chatmaild/src/chatmaild/filtermail.py#L84-L86>
-            let encrypted = encrypt_helper
-                .encrypt(context, encryption_keyring, message, compress)
-                .await?
-                + "\n";
+            // <https://github.com/deltachat/chatmail/blob/4d915f9800435bf13057d41af8d708abd34dbfa8/chatmaild/src/chatmaild/filtermail.py#L84-L86>:
+            let encrypted = encrypted + "\n";
 
             // Set the appropriate Content-Type for the outer message
             MimePart::new(
@@ -1244,7 +1350,13 @@ impl MimeFactory {
                 for (h, v) in &mut message.headers {
                     if h == "Content-Type" {
                         if let mail_builder::headers::HeaderType::ContentType(ct) = v {
-                            *ct = ct.clone().attribute("protected-headers", "v1");
+                            let mut ct_new = ct.clone();
+                            ct_new = ct_new.attribute("protected-headers", "v1");
+                            if use_std_header_protection {
+                                ct_new = ct_new.attribute("hp", "clear");
+                            }
+                            *ct = ct_new;
+                            break;
                         }
                     }
                 }
@@ -1352,6 +1464,43 @@ impl MimeFactory {
         let command = msg.param.get_cmd();
         let mut placeholdertext = None;
 
+        let send_verified_headers = match chat.typ {
+            Chattype::Single => true,
+            Chattype::Group => true,
+            // Mailinglists and broadcast channels can actually never be verified:
+            Chattype::Mailinglist => false,
+            Chattype::OutBroadcast | Chattype::InBroadcast => false,
+        };
+
+        if send_verified_headers {
+            let was_protected: bool = context
+                .sql
+                .query_get_value("SELECT protected FROM chats WHERE id=?", (chat.id,))
+                .await?
+                .unwrap_or_default();
+
+            if was_protected {
+                let unverified_member_exists = context
+                    .sql
+                    .exists(
+                        "SELECT COUNT(*)
+                        FROM contacts, chats_contacts
+                        WHERE chats_contacts.contact_id=contacts.id AND chats_contacts.chat_id=?
+                        AND contacts.id>9
+                        AND contacts.verifier=0",
+                        (chat.id,),
+                    )
+                    .await?;
+
+                if !unverified_member_exists {
+                    headers.push((
+                        "Chat-Verified",
+                        mail_builder::headers::raw::Raw::new("1").into(),
+                    ));
+                }
+            }
+        }
+
         if chat.typ == Chattype::Group {
             // Send group ID unless it is an ad hoc group that has no ID.
             if !chat.grpid.is_empty() {
@@ -1379,8 +1528,8 @@ impl MimeFactory {
 
             match command {
                 SystemMessage::MemberRemovedFromGroup => {
-                    ensure!(chat.typ != Chattype::OutBroadcast);
                     let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default();
+                    let fingerprint_to_remove = msg.param.get(Param::Arg4).unwrap_or_default();
 
                     if email_to_remove
                         == context
@@ -1401,12 +1550,19 @@ impl MimeFactory {
                                 .into(),
                         ));
                     }
+
+                    if !fingerprint_to_remove.is_empty() {
+                        headers.push((
+                            "Chat-Group-Member-Removed-Fpr",
+                            mail_builder::headers::raw::Raw::new(fingerprint_to_remove.to_string())
+                                .into(),
+                        ));
+                    }
                 }
                 SystemMessage::MemberAddedToGroup => {
-                    ensure!(chat.typ != Chattype::OutBroadcast);
-                    // TODO: lookup the contact by ID rather than email address.
-                    // We are adding key-contacts, the cannot be looked up by address.
                     let email_to_add = msg.param.get(Param::Arg).unwrap_or_default();
+                    let fingerprint_to_add = msg.param.get(Param::Arg4).unwrap_or_default();
+
                     placeholdertext =
                         Some(stock_str::msg_add_member_remote(context, email_to_add).await);
 
@@ -1416,15 +1572,19 @@ impl MimeFactory {
                             mail_builder::headers::raw::Raw::new(email_to_add.to_string()).into(),
                         ));
                     }
+                    if !fingerprint_to_add.is_empty() {
+                        headers.push((
+                            "Chat-Group-Member-Added-Fpr",
+                            mail_builder::headers::raw::Raw::new(fingerprint_to_add.to_string())
+                                .into(),
+                        ));
+                    }
                     if 0 != msg.param.get_int(Param::Arg2).unwrap_or_default() & DC_FROM_HANDSHAKE {
-                        info!(
-                            context,
-                            "Sending secure-join message {:?}.", "vg-member-added",
-                        );
+                        let step = "vg-member-added";
+                        info!(context, "Sending secure-join message {:?}.", step);
                         headers.push((
                             "Secure-Join",
-                            mail_builder::headers::raw::Raw::new("vg-member-added".to_string())
-                                .into(),
+                            mail_builder::headers::raw::Raw::new(step.to_string()).into(),
                         ));
                     }
                 }
@@ -1545,7 +1705,7 @@ impl MimeFactory {
 
                 // We should not send `null` as relay URL
                 // as this is the only way to reach the node.
-                debug_assert_eq!(node_addr.relay_urls().count(), 1);
+                debug_assert!(node_addr.relay_url().is_some());
                 headers.push((
                     HeaderDef::IrohNodeAddr.into(),
                     mail_builder::headers::text::Text::new(serde_json::to_string(&node_addr)?)
@@ -1835,6 +1995,37 @@ fn hidden_recipients() -> Address<'static> {
     Address::new_group(Some("hidden-recipients".to_string()), Vec::new())
 }
 
+fn should_encrypt_with_broadcast_secret(msg: &Message, chat: &Chat) -> bool {
+    chat.typ == Chattype::OutBroadcast && must_have_only_one_recipient(msg, chat).is_none()
+}
+
+fn should_hide_recipients(msg: &Message, chat: &Chat) -> bool {
+    should_encrypt_with_broadcast_secret(msg, chat)
+}
+
+fn should_encrypt_symmetrically(msg: &Message, chat: &Chat) -> bool {
+    should_encrypt_with_broadcast_secret(msg, chat)
+}
+
+/// Some messages sent into outgoing broadcast channels (member-added/member-removed)
+/// should only go to a single recipient,
+/// rather than all recipients.
+/// This function returns the fingerprint of the recipient the message should be sent to.
+fn must_have_only_one_recipient<'a>(msg: &'a Message, chat: &Chat) -> Option<Result<&'a str>> {
+    if chat.typ == Chattype::OutBroadcast
+        && matches!(
+            msg.param.get_cmd(),
+            SystemMessage::MemberRemovedFromGroup | SystemMessage::MemberAddedToGroup
+        )
+    {
+        let Some(fp) = msg.param.get(Param::Arg4) else {
+            return Some(Err(format_err!("Missing removed/added member")));
+        };
+        return Some(Ok(fp));
+    }
+    None
+}
+
 async fn build_body_file(context: &Context, msg: &Message) -> Result<MimePart<'static>> {
     let file_name = msg.get_filename().context("msg has no file")?;
     let blob = msg

src/mimefactory/mimefactory_tests.rs

@@ -832,8 +832,32 @@ async fn test_protected_headers_directive() -> Result<()> {
             .count(),
         1
     );
-    assert_eq!(part.match_indices("Subject:").count(), 1);
+    assert_eq!(part.match_indices("Subject:").count(), 2);
+    assert_eq!(part.match_indices("HP-Outer: Subject:").count(), 1);
+    Ok(())
+}
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_hp_outer_headers() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let t = &tcm.alice().await;
+    let chat_id = t.get_self_chat().await.id;
+
+    for std_hp_composing in [false, true] {
+        t.set_config_bool(Config::StdHeaderProtectionComposing, std_hp_composing)
+            .await?;
+        chat::send_text_msg(t, chat_id, "hi!".to_string()).await?;
+        let sent_msg = t.pop_sent_msg().await;
+        let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes(), None).await?;
+        assert_eq!(msg.header_exists(HeaderDef::HpOuter), std_hp_composing);
+        for hdr in ["Date", "From", "Message-ID"] {
+            assert_eq!(
+                msg.decoded_data_contains(&format!("HP-Outer: {hdr}:")),
+                std_hp_composing,
+            );
+        }
+        assert!(!msg.decoded_data_contains("HP-Outer: Content-Type"));
+    }
     Ok(())
 }
 

src/mimeparser.rs

@@ -26,7 +26,7 @@ use crate::dehtml::dehtml;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring};
-use crate::log::{error, info, warn};
+use crate::log::warn;
 use crate::message::{self, Message, MsgId, Viewtype, get_vcard_summary, set_msg_failed};
 use crate::param::{Param, Params};
 use crate::simplify::{SimplifiedText, simplify};
@@ -240,12 +240,11 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 impl MimeMessage {
     /// Parse a mime message.
     ///
-    /// If `partial` is set, it contains the full message size in bytes and an optional error text
-    /// for the partially downloaded message, and `body` contains the HEADER only.
+    /// If `partial` is set, it contains the full message size in bytes.
     pub(crate) async fn from_bytes(
         context: &Context,
         body: &[u8],
-        partial: Option<(u32, Option<String>)>,
+        partial: Option<u32>,
     ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
@@ -272,7 +271,7 @@ impl MimeMessage {
             &mut from,
             &mut list_post,
             &mut chat_disposition_notification_to,
-            &mail.headers,
+            &mail,
         );
         headers.retain(|k, _| {
             !is_hidden(k) || {
@@ -300,7 +299,7 @@ impl MimeMessage {
                         &mut from,
                         &mut list_post,
                         &mut chat_disposition_notification_to,
-                        &part.headers,
+                        part,
                     );
                     (part, part.ctype.mimetype.parse::<Mime>()?)
                 } else {
@@ -355,9 +354,16 @@ impl MimeMessage {
 
         let mail_raw; // Memory location for a possible decrypted message.
         let decrypted_msg; // Decrypted signed OpenPGP message.
+        let secrets: Vec<String> = context
+            .sql
+            .query_map_vec("SELECT secret FROM broadcast_secrets", (), |row| {
+                let secret: String = row.get(0)?;
+                Ok(secret)
+            })
+            .await?;
 
         let (mail, is_encrypted) =
-            match tokio::task::block_in_place(|| try_decrypt(&mail, &private_keyring)) {
+            match tokio::task::block_in_place(|| try_decrypt(&mail, &private_keyring, &secrets)) {
                 Ok(Some(mut msg)) => {
                     mail_raw = msg.as_data_vec().unwrap_or_default();
 
@@ -530,7 +536,7 @@ impl MimeMessage {
                 &mut inner_from,
                 &mut list_post,
                 &mut chat_disposition_notification_to,
-                &mail.headers,
+                mail,
             );
 
             if !signatures.is_empty() {
@@ -612,9 +618,9 @@ impl MimeMessage {
         };
 
         match partial {
-            Some((org_bytes, err)) => {
+            Some(org_bytes) => {
                 parser
-                    .create_stub_from_partial_download(context, org_bytes, err)
+                    .create_stub_from_partial_download(context, org_bytes)
                     .await?;
             }
             None => match mail {
@@ -708,14 +714,16 @@ impl MimeMessage {
     }
 
     /// Parses avatar action headers.
-    fn parse_avatar_headers(&mut self, context: &Context) {
+    fn parse_avatar_headers(&mut self, context: &Context) -> Result<()> {
         if let Some(header_value) = self.get_header(HeaderDef::ChatGroupAvatar) {
-            self.group_avatar = self.avatar_action_from_header(context, header_value.to_string());
+            self.group_avatar =
+                self.avatar_action_from_header(context, header_value.to_string())?;
         }
 
         if let Some(header_value) = self.get_header(HeaderDef::ChatUserAvatar) {
-            self.user_avatar = self.avatar_action_from_header(context, header_value.to_string());
+            self.user_avatar = self.avatar_action_from_header(context, header_value.to_string())?;
         }
+        Ok(())
     }
 
     fn parse_videochat_headers(&mut self) {
@@ -822,7 +830,7 @@ impl MimeMessage {
 
     async fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context);
-        self.parse_avatar_headers(context);
+        self.parse_avatar_headers(context)?;
         self.parse_videochat_headers();
         if self.delivery_report.is_none() {
             self.squash_attachment_parts();
@@ -854,6 +862,10 @@ impl MimeMessage {
                         .iter_mut()
                         .find(|part| !part.msg.is_empty() && !part.is_reaction);
                     if let Some(part) = part_with_text {
+                        // Message bubbles are small, so we use en dash to save space. In some
+                        // languages there may be em dashes in the message text added by the author,
+                        // they may look stronger than Subject separation, this is a known thing.
+                        // Anyway, classic email support isn't a priority as of 2025.
                         part.msg = format!("{} – {}", subject, part.msg);
                     }
                 }
@@ -920,21 +932,18 @@ impl MimeMessage {
         &mut self,
         context: &Context,
         header_value: String,
-    ) -> Option<AvatarAction> {
-        if header_value == "0" {
+    ) -> Result<Option<AvatarAction>> {
+        let res = if header_value == "0" {
             Some(AvatarAction::Delete)
         } else if let Some(base64) = header_value
             .split_ascii_whitespace()
             .collect::<String>()
             .strip_prefix("base64:")
         {
-            match BlobObject::store_from_base64(context, base64) {
-                Ok(path) => Some(AvatarAction::Change(path)),
-                Err(err) => {
-                    warn!(
-                        context,
-                        "Could not decode and save avatar to blob file: {:#}", err,
-                    );
+            match BlobObject::store_from_base64(context, base64)? {
+                Some(path) => Some(AvatarAction::Change(path)),
+                None => {
+                    warn!(context, "Could not decode avatar base64");
                     None
                 }
             }
@@ -948,7 +957,7 @@ impl MimeMessage {
                         if let Some(blob) = part.param.get(Param::File) {
                             let res = Some(AvatarAction::Change(blob.to_string()));
                             self.parts.remove(i);
-                            return res;
+                            return Ok(res);
                         }
                         break;
                     }
@@ -956,7 +965,8 @@ impl MimeMessage {
                 i += 1;
             }
             None
-        }
+        };
+        Ok(res)
     }
 
     /// Returns true if the message was encrypted as defined in
@@ -997,6 +1007,14 @@ impl MimeMessage {
         self.headers.contains_key(hname) || self.headers_removed.contains(hname)
     }
 
+    #[cfg(test)]
+    /// Returns whether the decrypted data contains the given `&str`.
+    pub(crate) fn decoded_data_contains(&self, s: &str) -> bool {
+        assert!(!self.decrypting_failed);
+        let decoded_str = str::from_utf8(&self.decoded_data).unwrap();
+        decoded_str.contains(s)
+    }
+
     /// Returns `Chat-Group-ID` header value if it is a valid group ID.
     pub fn get_chat_group_id(&self) -> Option<&str> {
         self.get_header(HeaderDef::ChatGroupId)
@@ -1313,6 +1331,10 @@ impl MimeMessage {
                             let is_html = mime_type == mime::TEXT_HTML;
                             if is_html {
                                 self.is_mime_modified = true;
+                                // NB: This unconditionally removes Legacy Display Elements (see
+                                // <https://www.rfc-editor.org/rfc/rfc9788.html#section-4.5.3.3>). We
+                                // don't check for the "hp-legacy-display" Content-Type parameter
+                                // for simplicity.
                                 if let Some(text) = dehtml(&decoded_data) {
                                     text
                                 } else {
@@ -1340,16 +1362,30 @@ impl MimeMessage {
 
                         let (simplified_txt, simplified_quote) = if mime_type.type_() == mime::TEXT
                             && mime_type.subtype() == mime::PLAIN
-                            && is_format_flowed
                         {
-                            let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
-                                delsp.as_str().eq_ignore_ascii_case("yes")
-                            } else {
-                                false
+                            // Don't check that we're inside an encrypted or signed part for
+                            // simplicity.
+                            let simplified_txt = match mail
+                                .ctype
+                                .params
+                                .get("hp-legacy-display")
+                                .is_some_and(|v| v == "1")
+                            {
+                                false => simplified_txt,
+                                true => rm_legacy_display_elements(&simplified_txt),
                             };
-                            let unflowed_text = unformat_flowed(&simplified_txt, delsp);
-                            let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
-                            (unflowed_text, unflowed_quote)
+                            if is_format_flowed {
+                                let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                    delsp.as_str().eq_ignore_ascii_case("yes")
+                                } else {
+                                    false
+                                };
+                                let unflowed_text = unformat_flowed(&simplified_txt, delsp);
+                                let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp));
+                                (unflowed_text, unflowed_quote)
+                            } else {
+                                (simplified_txt, top_quote)
+                            }
                         } else {
                             (simplified_txt, top_quote)
                         };
@@ -1563,6 +1599,8 @@ impl MimeMessage {
             // The message belongs to a mailing list and has a `ListId:`-header
             // that should be used to get a unique id.
             return Some(list_id);
+        } else if let Some(chat_list_id) = self.get_header(HeaderDef::ChatListId) {
+            return Some(chat_list_id);
         } else if let Some(sender) = self.get_header(HeaderDef::Sender) {
             // the `Sender:`-header alone is no indicator for mailing list
             // as also used for bot-impersonation via `set_override_sender_name()`
@@ -1620,6 +1658,11 @@ impl MimeMessage {
         }
     }
 
+    /// Merges headers from the email `part` into `headers` respecting header protection.
+    /// Should only be called with nonempty `headers` if `part` is a root of the Cryptographic
+    /// Payload as defined in <https://www.rfc-editor.org/rfc/rfc9788.html> "Header Protection for
+    /// Cryptographically Protected Email", otherwise this may unnecessarily discard headers from
+    /// outer parts.
     #[allow(clippy::too_many_arguments)]
     fn merge_headers(
         context: &Context,
@@ -1630,10 +1673,14 @@ impl MimeMessage {
         from: &mut Option<SingleInfo>,
         list_post: &mut Option<String>,
         chat_disposition_notification_to: &mut Option<SingleInfo>,
-        fields: &[mailparse::MailHeader<'_>],
+        part: &mailparse::ParsedMail,
     ) {
+        let fields = &part.headers;
+        // See <https://www.rfc-editor.org/rfc/rfc9788.html>.
+        let has_header_protection = part.ctype.params.contains_key("hp");
+
         headers.retain(|k, _| {
-            !is_protected(k) || {
+            !(has_header_protection || is_protected(k)) || {
                 headers_removed.insert(k.to_string());
                 false
             }
@@ -1960,6 +2007,20 @@ impl MimeMessage {
     }
 }
 
+fn rm_legacy_display_elements(text: &str) -> String {
+    let mut res = None;
+    for l in text.lines() {
+        res = res.map(|r: String| match r.is_empty() {
+            true => l.to_string(),
+            false => r + "\r\n" + l,
+        });
+        if l.is_empty() {
+            res = Some(String::new());
+        }
+    }
+    res.unwrap_or_default()
+}
+
 fn remove_header(
     headers: &mut HashMap<String, String>,
     key: &str,
@@ -2084,7 +2145,8 @@ pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
 }
 
 /// Returns whether the outer header value must be ignored if the message contains a signed (and
-/// optionally encrypted) part.
+/// optionally encrypted) part. This is independent from the modern Header Protection defined in
+/// <https://www.rfc-editor.org/rfc/rfc9788.html>.
 ///
 /// NB: There are known cases when Subject and List-ID only appear in the outer headers of
 /// signed-only messages. Such messages are shown as unencrypted anyway.

src/mimeparser/mimeparser_tests.rs

@@ -1401,22 +1401,28 @@ async fn test_x_microsoft_original_message_id_precedence() -> Result<()> {
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_extra_imf_chat_header() -> Result<()> {
+async fn test_extra_imf_headers() -> Result<()> {
     let mut tcm = TestContextManager::new();
     let t = &tcm.alice().await;
     let chat_id = t.get_self_chat().await.id;
 
-    chat::send_text_msg(t, chat_id, "hi!".to_string()).await?;
-    let sent_msg = t.pop_sent_msg().await;
-    // Check removal of some nonexistent "Chat-*" header to protect the code from future breakages.
-    let payload = sent_msg
-        .payload
-        .replace("Message-ID:", "Chat-Forty-Two: 42\r\nMessage-ID:");
-    let msg = MimeMessage::from_bytes(t, payload.as_bytes(), None)
-        .await
-        .unwrap();
-    assert!(msg.headers.contains_key("chat-version"));
-    assert!(!msg.headers.contains_key("chat-forty-two"));
+    for std_hp_composing in [false, true] {
+        t.set_config_bool(Config::StdHeaderProtectionComposing, std_hp_composing)
+            .await?;
+        chat::send_text_msg(t, chat_id, "hi!".to_string()).await?;
+        let sent_msg = t.pop_sent_msg().await;
+        // Check removal of some nonexistent "Chat-*" header to protect the code from future
+        // breakages. But headers not prefixed with "Chat-" remain unless a message has standard
+        // Header Protection.
+        let payload = sent_msg.payload.replace(
+            "Message-ID:",
+            "Chat-Forty-Two: 42\r\nForty-Two: 42\r\nMessage-ID:",
+        );
+        let msg = MimeMessage::from_bytes(t, payload.as_bytes(), None).await?;
+        assert!(msg.headers.contains_key("chat-version"));
+        assert!(!msg.headers.contains_key("chat-forty-two"));
+        assert_ne!(msg.headers.contains_key("forty-two"), std_hp_composing);
+    }
     Ok(())
 }
 
@@ -1751,6 +1757,39 @@ async fn test_time_in_future() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_hp_legacy_display() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let mut msg = Message::new_text(
+        "Subject: Dinner plans\n\
+        \n\
+        Let's eat"
+            .to_string(),
+    );
+    msg.set_subject("Dinner plans".to_string());
+    let chat_id = alice.create_chat(bob).await.id;
+    alice.set_config_bool(Config::TestHooks, true).await?;
+    *alice.pre_encrypt_mime_hook.lock() = Some(|_, mut mime| {
+        for (h, v) in &mut mime.headers {
+            if h == "Content-Type" {
+                if let mail_builder::headers::HeaderType::ContentType(ct) = v {
+                    *ct = ct.clone().attribute("hp-legacy-display", "1");
+                }
+            }
+        }
+        mime
+    });
+    let sent_msg = alice.send_msg(chat_id, &mut msg).await;
+
+    let msg_bob = bob.recv_msg(&sent_msg).await;
+    assert_eq!(msg_bob.subject, "Dinner plans");
+    assert_eq!(msg_bob.text, "Let's eat");
+    Ok(())
+}
+
 /// Tests that subject is not prepended to the message
 /// when bot receives it.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

src/net/dns.rs

@@ -50,7 +50,7 @@ use tokio::time::timeout;
 
 use super::load_connection_timestamp;
 use crate::context::Context;
-use crate::log::{info, warn};
+use crate::log::warn;
 use crate::tools::time;
 
 /// Inserts entry into DNS cache