chore(release): prepare for 1.135.0 (#5264)

after merging:

6. Tag the release: `git tag -a v1.135.0`.

7. Push the release tag: `git push origin v1.135.0`.

8. Create a GitHub release: `gh release create v1.135.0 -n ''`.

---------

Co-authored-by: link2xt <link2xt@testrun.org>

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.74.0
+      RUSTUP_TOOLCHAIN: 1.75.0
     steps:
       - uses: actions/checkout@v3
       - name: Install rustfmt and clippy
@@ -76,11 +76,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.74.0
+            rust: 1.75.0
           - os: windows-latest
-            rust: 1.74.0
+            rust: 1.75.0
           - os: macos-latest
-            rust: 1.74.0
+            rust: 1.75.0
 
           # Minimum Supported Rust Version = 1.70.0
           - os: ubuntu-latest

CHANGELOG.md

@@ -1,5 +1,252 @@
 # Changelog
 
+## [1.135.0] - 2024-02-13
+
+### Features / Changes
+
+- Add wildcard pattern support to provider database.
+- Add device message about outgoing undecryptable messages ([#5164](https://github.com/deltachat/deltachat-core-rust/pull/5164)).
+- Context::set_config(): Restart IO scheduler if needed ([#5111](https://github.com/deltachat/deltachat-core-rust/pull/5111)).
+- Server_sent_unsolicited_exists(): Log folder name.
+- Cache system time instead of looking at the clock several times in a row.
+
+### Fixes
+
+- Dehtml: Don't just truncate text when trying to decode ([#5223](https://github.com/deltachat/deltachat-core-rust/pull/5223)).
+- Mark the gossip keys from the message as verified, not the ones from the db ([#5247](https://github.com/deltachat/deltachat-core-rust/pull/5247)).
+- Guarantee immediate message deletion if delete_server_after == 0 ([#5201](https://github.com/deltachat/deltachat-core-rust/pull/5201)).
+- Never allow a message timestamp to be a lot in the future ([#5249](https://github.com/deltachat/deltachat-core-rust/pull/5249)).
+- Imap::configure_mvbox: Do select_with_uidvalidity() before return.
+- ImapSession::select_or_create_folder(): Don't fail if folder is created in parallel.
+- Emit ConfigSynced event on the second device.
+- Create mvbox on setting mvbox_move.
+- Use SystemTime instead of Instant everywhere.
+- Restore database rows removed in previous release; this ensures compatibility when adding second device or importing backup and not all devices run the new core ([#5254](https://github.com/deltachat/deltachat-core-rust/pull/5254))
+
+### Miscellaneous Tasks
+
+- cargo: Bump image from 0.24.7 to 0.24.8.
+- cargo: Bump chrono from 0.4.31 to 0.4.33.
+- cargo: Bump futures-lite from 2.1.0 to 2.2.0.
+- cargo: Bump pin-project from 1.1.3 to 1.1.4.
+- cargo: Bump iana-time-zone from yanked 0.1.59 to 0.1.60.
+- cargo: Bump smallvec from 1.11.2 to 1.13.1.
+- cargo: Bump base64 from 0.21.5 to 0.21.7.
+- cargo: Bump regex from 1.10.2 to 1.10.3.
+- cargo: Bump libc from 0.2.151 to 0.2.153.
+- cargo: Bump reqwest from 0.11.23 to 0.11.24.
+- cargo: Bump axum from 0.7.3 to 0.7.4.
+- cargo: Bump uuid from 1.6.1 to 1.7.0.
+- cargo: Bump fast-socks5 from 0.9.2 to 0.9.5.
+- cargo: Bump serde_json from 1.0.111 to 1.0.113.
+- cargo: Bump syn from 2.0.46 to 2.0.48.
+- cargo: Bump serde from 1.0.194 to 1.0.196.
+- cargo: Bump toml from 0.8.8 to 0.8.10.
+- cargo: Update to strum 0.26.
+- Cargo update.
+- scripts: Do not install deltachat-rpc-client twice.
+
+### Other
+
+- Update welcome image, thanks @paulaluap
+.
+- Merge pull request #5243 from deltachat/dependabot/cargo/pin-project-1.1.4
+
+.
+- Merge pull request #5241 from deltachat/dependabot/cargo/futures-lite-2.2.0
+
+.
+- Merge pull request #5236 from deltachat/dependabot/cargo/chrono-0.4.33
+
+.
+- Merge pull request #5235 from deltachat/dependabot/cargo/image-0.24.8
+
+.
+- Basic self-reporting, core part ([#5129](https://github.com/deltachat/deltachat-core-rust/pull/5129))
+
+Part of https://github.com/deltachat/deltachat-android/issues/2909
+
+For now, this is only sending a few basic metrics..
+- Do not change db schema in an incompatible way ([#5254](https://github.com/deltachat/deltachat-core-rust/pull/5254))
+
+PR #5099 removed some columns in the database that were actually in use.
+
+usually, to not worsen UX unnecessarily
+(releases take time - in between, "Add Second Device", "Backup" etc.
+would fail), we try to avoid such schema changes (checking for
+db-version would avoid import etc. but would still worse UX),
+see discussion at #2294.
+
+these are the errors, the user will be confronted with otherwise:
+
+<img width=400
+src=https://github.com/deltachat/deltachat-core-rust/assets/9800740/e3f0fd6e-a7a9-43f6-9023-0ae003985425>
+
+it is not great to maintain the old columns, but well :)
+
+as no official releases with newer cores are rolled out yet, i think, it
+is fine to change the "107" migration
+and not copy things a second time in a newer migration.
+
+(this issue happens to me during testing, and is probably also the issue
+reported by @lk108 for ubuntu-touch).
+
+### Refactor
+
+- Resultify token::exists.
+
+### Tests
+
+- Delete_server_after="1" should cause immediate message deletion ([#5201](https://github.com/deltachat/deltachat-core-rust/pull/5201)).
+
+## [1.134.0] - 2024-01-31
+
+### API-Changes
+
+- [**breaking**] JSON-RPC: device message api now requires `Option<MessageData>` instead of `String` for the message ([#5211](https://github.com/deltachat/deltachat-core-rust/pull/5211)).
+- CFFI: add `dc_accounts_background_fetch` and event `DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE`.
+- JSON-RPC: add `accounts_background_fetch`.
+
+### Features / Changes
+
+- `Qr::check_qr()`: Accept i.delta.chat invite links ([#5217](https://github.com/deltachat/deltachat-core-rust/pull/5217)).
+- Add support for IMAP METADATA, fetching `/shared/comment` and `/shared/admin` and displaying it in account info.
+
+### Fixes
+
+- Add tolerance for macOS and iOS changing `#` to `%23`.
+- Do not drop unknown report attachments, such as TLS reports.
+- Treat only "Auto-Submitted: auto-generated" messages as bot-sent ([#5213](https://github.com/deltachat/deltachat-core-rust/pull/5213)).
+- `Chat::resend_msgs`: Guarantee strictly increasing time in the `Date` header.
+- Delete resent messages on receiver side ([#5155](https://github.com/deltachat/deltachat-core-rust/pull/5155)).
+- Fix iOS build issue.
+
+### CI
+
+- Add/remove necessary newlines to fix Python lint.
+
+### Tests
+
+- `test_import_export_online_all`: Send the message to the existing address to avoid errors ([#5220](https://github.com/deltachat/deltachat-core-rust/pull/5220)).
+
+## [1.133.2] - 2024-01-24
+
+### Fixes
+
+- Downgrade OpenSSL from 3.2.0 to 3.1.4 ([#5206](https://github.com/deltachat/deltachat-core-rust/issues/5206))
+- No new chats for MDNs with alias ([#5196](https://github.com/deltachat/deltachat-core-rust/issues/5196)) ([#5199](https://github.com/deltachat/deltachat-core-rust/pull/5199)).
+
+## [1.133.1] - 2024-01-21
+
+### API-Changes
+
+- Add `is_bot` to cffi and jsonrpc ([#5197](https://github.com/deltachat/deltachat-core-rust/pull/5197)).
+
+### Features / Changes
+
+- Add system message when provider does not allow unencrypted messages ([#5195](https://github.com/deltachat/deltachat-core-rust/pull/5195)).
+
+### Fixes
+
+- `Chat::send_msg`: Remove encryption-related params from already sent message. This allows to send received encrypted `dc_msg_t` object to unencrypted chat, e.g. in a Python bot.
+- Set message download state to Failure on IMAP errors. This avoids partially downloaded messages getting stuck in "Downloading..." state without actually being in a download queue.
+- BCC-to-self even if server deletion is set to "at once". This is a workaround for SMTP servers which do not return response in time, BCC-self works as a confirmation that message was sent out successfully and does not need more retries.
+- node: Run tests with native ESM modules instead of `esm` ([#5194](https://github.com/deltachat/deltachat-core-rust/pull/5194)).
+- Use Quoted-Printable MIME encoding for the text part ([#3986](https://github.com/deltachat/deltachat-core-rust/pull/3986)).
+
+### Tests
+
+- python: Add `get_protected_chat` to testplugin.py.
+
+## [1.133.0] - 2024-01-14
+
+### Features / Changes
+
+- Securejoin protocol implementation refinements
+  - Track forward and backward verification separately ([#5089](https://github.com/deltachat/deltachat-core-rust/pull/5089)) to avoid inconsistent states.
+  - Mark 1:1 chat as verified for Bob early. 1:1 chat with Alice is verified as soon as Alice's key is verified rather than at the end of the protocol.
+- Put Message-ID into hidden headers and take it from there on receiver ([#4798](https://github.com/deltachat/deltachat-core-rust/pull/4798)). This works around servers which generate their own Message-ID and overwrite the one generated by Delta Chat.
+- deltachat-repl: Enable INFO logging by default and add timestamps.
+- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elments based on the configuration key which is a part of the event.
+- Sync contact creation/rename across devices ([#5163](https://github.com/deltachat/deltachat-core-rust/pull/5163)).
+- Encrypt MDNs ([#5175](https://github.com/deltachat/deltachat-core-rust/pull/5175)).
+- Only try to configure non-strict TLS checks if explicitly set ([#5181](https://github.com/deltachat/deltachat-core-rust/pull/5181)).
+
+### Build system
+
+- Use released version of iroh 0.4.2 for "setup second device" feature.
+
+### CI
+
+- Update to Rust 1.75.0.
+- Downgrade `chai` from 4.4.0 to 4.3.10.
+
+### Documentation
+
+- Add a link <https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html> to autoconfig RFC draft.
+- Update securejoin link in `standards.md` from <https://countermitm.readthedocs.io/> to <https://securejoin.readthedocs.io>.
+- Restore "Constants" page in Doxygen >=1.9.8
+
+### Fixes
+
+- imap: Limit the rate of LOGIN attempts rather than connection attempts. This is to avoid having to wait for rate limiter right after switching from a bad or offline network to a working network while still guarding against reconnection loop.
+- Do not ignore `peerstate.save_to_db()` errors.
+- securejoin: Mark 1:1s as protected regardless of the Config::VerifiedOneOnOneChats.
+- Delete received outgoing messages from SMTP queue ([#5115](https://github.com/deltachat/deltachat-core-rust/pull/5115)).
+- imap: Fail fast on `LIST` errors to avoid busy loop when connection is lost.
+- Split SMTP jobs already in `chat::create_send_msg_jobs()` ([#5115](https://github.com/deltachat/deltachat-core-rust/pull/5115)).
+- Do not remove contents from unencrypted [Schleuder](https://schleuder.org/) mailing lists messages.
+- Reset message error when scheduling resending ([#5119](https://github.com/deltachat/deltachat-core-rust/pull/5119)).
+- Emit events more reliably when starting and stopping I/O ([#5101](https://github.com/deltachat/deltachat-core-rust/pull/5101)).
+- Fix timestamp of chat protection info message for correct message ordering after restoring a backup ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+### Refactor
+
+- sql: Recreate `config` table with UNIQUE constraint.
+- sql: Recreate `keypairs` table to remove unused `addr` and `created` fields and move `is_default` flag to `config` table.
+- Send `Secure-Join-Fingerprint` only in `*-request-with-auth`.
+
+### Tests
+
+- Test joining non-protected group.
+- Test that read receipts don't degrade encryption.
+- Test that changing default private key breaks backward verification.
+- Test recovery from lost vc-contact-confirm.
+- Use `wait_for_incoming_msg_event()` more.
+
+## [1.132.1] - 2023-12-12
+
+### Features / Changes
+
+- Add "From:" to protected headers for signed-only messages.
+- Sync user actions for ad-hoc groups across devices ([#5065](https://github.com/deltachat/deltachat-core-rust/pull/5065)).
+
+### Fixes
+
+- Add padlock to empty part if the whole message is empty.
+- Renew IDLE timeout on keepalives and reduce it to 5 minutes.
+- connectivity: Return false from `all_work_done()` immediately after connecting (iOS notification fix).
+
+### API-Changes
+
+- deltachat-jsonrpc-client: add `Account.{import,export}_self_keys`.
+
+### CI
+
+- Update to Rust 1.74.1.
+
+## [1.132.0] - 2023-12-06
+
+### Features / Changes
+
+- Increase TCP timeouts from 30 to 60 seconds.
+
+### Fixes
+
+- Don't sort message creating a protected group over a protection message ([#4963](https://github.com/deltachat/deltachat-core-rust/pull/4963)).
+- Do not lock accounts.toml on iOS.
+- Protect groups even if some members are not verified and add `test_securejoin_after_contact_resetup` regression test.
+
 ## [1.131.9] - 2023-12-02
 
 ### API-Changes
@@ -3322,3 +3569,10 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.131.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.6...v1.131.7
 [1.131.8]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.7...v1.131.8
 [1.131.9]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.8...v1.131.9
+[1.132.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.9...v1.132.0
+[1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1
+[1.133.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.1...v1.133.0
+[1.133.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.0...v1.133.1
+[1.133.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.1...v1.133.2
+[1.134.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.2...v1.134.0
+[1.135.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.134.0...v1.135.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.131.9"
+version = "1.135.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.70"
@@ -30,9 +30,6 @@ opt-level = "z"
 codegen-units = 1
 strip = true
 
-[patch.crates-io]
-imap-proto = { git = "https://github.com/djc/tokio-imap.git", rev = "01ff256a7e42a9f7d2732706f8b71a16ce93427e" }
-
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 format-flowed = { path = "./format-flowed" }
@@ -40,7 +37,7 @@ ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
 async-channel = "2.0.0"
-async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
+async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
@@ -54,12 +51,12 @@ escaper = "0.1"
 fast-socks5 = "0.9"
 fd-lock = "4"
 futures = "0.3"
-futures-lite = "2.0.0"
+futures-lite = "2.2.0"
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh = { git = "https://github.com/n0-computer/iroh", branch = "maint-0.4", default-features = false }
+image = { version = "0.24.8", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -76,19 +73,20 @@ pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
 quick-xml = "0.31"
+quoted_printable = "0.5"
 rand = "0.8"
-regex = "1.9"
-reqwest = { version = "0.11.20", features = ["json"] }
+regex = "1.10"
+reqwest = { version = "0.11.24", features = ["json"] }
 rusqlite = { version = "0.30", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
-serde_json = "1.0"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-strum = "0.25"
-strum_macros = "0.25"
+strum = "0.26"
+strum_macros = "0.26"
 tagger = "4.3.4"
 textwrap = "0.16.0"
 thiserror = "1"
@@ -101,11 +99,19 @@ toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 
+# Pin OpenSSL to 3.1 releases.
+# OpenSSL 3.2 has a regression tracked at <https://github.com/openssl/openssl/issues/23376>
+# which results in broken `deltachat-rpc-server` binaries when cross-compiled using Zig toolchain.
+# See <https://github.com/deltachat/deltachat-core-rust/issues/5206> for Delta Chat issue.
+# According to <https://www.openssl.org/policies/releasestrat.html>
+# 3.1 branch will be supported until 2025-03-14.
+openssl-src = "~300.1"
+
 [dev-dependencies]
 ansi_term = "0.12.0"
 anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = "2.0.0"
+futures-lite = "2.2.0"
 log = "0.4"
 pretty_env_logger = "0.5"
 proptest = { version = "1", default-features = false, features = ["std"] }

README.md

@@ -27,7 +27,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -121,7 +121,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests

RELEASE.md

@@ -9,13 +9,16 @@ For example, to release version 1.116.0 of the core, do the following steps.
 
 3. Update the changelog: `git cliff --unreleased --tag 1.116.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.116.0 -p CHANGELOG.md`.
 
-4. Update the version by running `scripts/set_core_version.py 1.116.0`.
+4. add a link to compare previous with current version to the end of CHANGELOG.md:
+  `[1.116.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.115.2...v1.116.0`
 
-5. Commit the changes as `chore(release): prepare for 1.116.0`.
+5. Update the version by running `scripts/set_core_version.py 1.116.0`.
+
+6. Commit the changes as `chore(release): prepare for 1.116.0`.
    Optionally, use a separate branch like `prep-1.116.0` for this commit and open a PR for review.
 
-6. Tag the release: `git tag -a v1.116.0`.
+7. Tag the release: `git tag -a v1.116.0`.
 
-7. Push the release tag: `git push origin v1.116.0`.
+8. Push the release tag: `git push origin v1.116.0`.
 
-8. Create a GitHub release: `gh release create v1.116.0 -n ''`.
+9. Create a GitHub release: `gh release create v1.116.0 -n ''`.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.131.9"
+version = "1.135.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/DoxygenLayout.xml

@@ -9,7 +9,7 @@
       <tab type="hierarchy" visible="no" title="" intro=""/>
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="topics" visible="yes" title="Constants" intro="Here is a list of constants:"/>
     <tab type="pages" visible="yes" title="" intro=""/>
     <tab type="namespaces" visible="yes" title="">
       <tab type="namespacelist" visible="yes" title="" intro=""/>

deltachat-ffi/deltachat.h

@@ -423,19 +423,16 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes,
- *                    0=do not watch the `Sent`-folder (default),
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
+ *                    0=do not watch the `Sent`-folder (default).
  * - `mvbox_move`   = 1=detect chat messages,
  *                    move them to the `DeltaChat` folder,
  *                    and watch the `DeltaChat` folder for updates (default),
  *                    0=do not move chat-messages
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `only_fetch_mvbox` = 1=Do not fetch messages from folders other than the
  *                    `DeltaChat` folder. Messages will still be fetched from the
  *                    spam folder and `sendbox_watch` will also still be respected
  *                    if enabled.
  *                    0=watch all folders normally (default)
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `show_emails`  = DC_SHOW_EMAILS_OFF (0)=
  *                    show direct replies to chats only,
  *                    DC_SHOW_EMAILS_ACCEPTED_CONTACTS (1)=
@@ -3151,6 +3148,23 @@ void           dc_accounts_maybe_network        (dc_accounts_t* accounts);
 void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
 
 
+/**
+ * Perform a background fetch for all accounts in parallel with a timeout.
+ * Pauses the scheduler, fetches messages from imap and then resumes the scheduler.
+ *
+ * dc_accounts_background_fetch() was created for the iOS Background fetch.
+ *
+ * The `DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE` event is emitted at the end
+ * even in case of timeout, unless the function fails and returns 0.
+ * Process all events until you get this one and you can safely return to the background
+ * without forgetting to create notifications caused by timing race conditions.
+ *
+ * @memberof dc_accounts_t
+ * @param timeout The timeout in seconds
+ * @return Return 1 if DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE was emitted and 0 otherwise.
+ */
+int            dc_accounts_background_fetch    (dc_accounts_t* accounts, uint64_t timeout);
+
 /**
  * Create the event emitter that is used to receive events.
  *
@@ -4397,6 +4411,9 @@ int             dc_msg_is_info                (const dc_msg_t* msg);
  * Currently, the following types are defined:
  * - DC_INFO_PROTECTION_ENABLED (11) - Info-message for "Chat is now protected"
  * - DC_INFO_PROTECTION_DISABLED (12) - Info-message for "Chat is no longer protected"
+ * - DC_INFO_INVALID_UNENCRYPTED_MAIL (13) - Info-message for "Provider requires end-to-end encryption which is not setup yet",
+ *   the UI should change the corresponding string using #DC_STR_INVALID_UNENCRYPTED_MAIL
+ *   and also offer a way to fix the encryption, eg. by a button offering a QR scan
  *
  * Even when you display an icon,
  * you should still display the text of the informational message using dc_msg_get_text()
@@ -4423,6 +4440,7 @@ int             dc_msg_get_info_type          (const dc_msg_t* msg);
 #define         DC_INFO_EPHEMERAL_TIMER_CHANGED   10
 #define         DC_INFO_PROTECTION_ENABLED        11
 #define         DC_INFO_PROTECTION_DISABLED       12
+#define         DC_INFO_INVALID_UNENCRYPTED_MAIL  13
 #define         DC_INFO_WEBXDC_INFO_MESSAGE       32
 
 /**
@@ -5068,6 +5086,15 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
  */
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
+/**
+ * Returns whether contact is a bot.
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 0 if the contact is not a bot, 1 otherwise.
+ */
+int             dc_contact_is_bot            (dc_contact_t* contact);
+
 
 /**
  * Return the contact ID that verified a contact.
@@ -6206,6 +6233,18 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_SELFAVATAR_CHANGED               2110
 
 
+/**
+ * A multi-device synced config value changed. Maybe the app needs to refresh smth. For uniformity
+ * this is emitted on the source device too. The value isn't reported, otherwise it would be logged
+ * which might not be good for privacy. You can get the new value with
+ * `dc_get_config(context, data2)`.
+ *
+ * @param data1 0
+ * @param data2 (char*) Configuration key.
+ */
+#define DC_EVENT_CONFIG_SYNCED                    2111
+
+
 /**
  * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
@@ -6230,6 +6269,16 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
 
+/**
+ * Tells that the Background fetch was completed (or timed out).
+ *
+ * This event acts as a marker, when you reach this event you can be sure
+ * that all events emitted during the background fetch were processed.
+ * 
+ * This event is only emitted by the account manager
+ */
+
+#define DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE  2200
 
 /**
  * @}
@@ -6998,6 +7047,8 @@ void dc_event_unref(dc_event_t* event);
 /// "You added member %1$s."
 ///
 /// Used in status messages.
+///
+/// `%1$s` will be replaced by the added member's name.
 #define DC_STR_ADD_MEMBER_BY_YOU 128
 
 /// "Member %1$s added by %2$s."
@@ -7219,6 +7270,21 @@ void dc_event_unref(dc_event_t* event);
 /// Used as the first info messages in newly created groups.
 #define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
 
+/// "Member %1$s added."
+///
+/// Used as info messages.
+///
+/// `%1$s` will be replaced by the added member's name.
+#define DC_STR_MESSAGE_ADD_MEMBER 173
+
+/// "Your email provider %1$s requires end-to-end encryption which is not setup yet."
+///
+/// Used as info messages when a message cannot be sent because it cannot be encrypted.
+///
+/// `%1$s` will be replaced by the provider's domain.
+#define DC_STR_INVALID_UNENCRYPTED_MAIL 174
+
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -556,8 +556,10 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::SecurejoinJoinerProgress { .. } => 2061,
         EventType::ConnectivityChanged => 2100,
         EventType::SelfavatarChanged => 2110,
+        EventType::ConfigSynced { .. } => 2111,
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
+        EventType::AccountsBackgroundFetchDone => 2200,
     }
 }
 
@@ -583,8 +585,10 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Error(_)
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
+        | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::ErrorSelfNotInGroup(_) => 0,
+        | EventType::ErrorSelfNotInGroup(_)
+        | EventType::AccountsBackgroundFetchDone => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -643,7 +647,9 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::SelfavatarChanged => 0,
+        | EventType::SelfavatarChanged
+        | EventType::AccountsBackgroundFetchDone
+        | EventType::ConfigSynced { .. } => 0,
         EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
@@ -705,6 +711,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
         | EventType::WebxdcInstanceDeleted { .. }
+        | EventType::AccountsBackgroundFetchDone
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -722,6 +729,10 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             .to_c_string()
             .unwrap_or_default()
             .into_raw(),
+        EventType::ConfigSynced { key } => {
+            let data2 = key.to_string().to_c_string().unwrap_or_default();
+            data2.into_raw()
+        }
     }
 }
 
@@ -4125,6 +4136,15 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_is_bot(contact: *mut dc_contact_t) -> libc::c_int {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_is_bot()");
+        return 0;
+    }
+    (*contact).contact.is_bot() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) -> u32 {
     if contact.is_null() {
@@ -4882,6 +4902,26 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_background_fetch(
+    accounts: *mut dc_accounts_t,
+    timeout_in_seconds: u64,
+) -> libc::c_int {
+    if accounts.is_null() || timeout_in_seconds <= 2 {
+        eprintln!("ignoring careless call to dc_accounts_background_fetch()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    block_on(async move {
+        let accounts = accounts.read().await;
+        accounts
+            .background_fetch(Duration::from_secs(timeout_in_seconds))
+            .await;
+    });
+    1
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     accounts: *mut dc_accounts_t,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.131.9"
+version = "1.135.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -17,11 +17,11 @@ deltachat = { path = ".." }
 num-traits = "0.2"
 schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.8.0"
+tempfile = "3.9.0"
 log = "0.4"
 async-channel = { version = "2.0.0" }
-futures = { version = "0.3.28" }
-serde_json = "1.0.105"
+futures = { version = "0.3.30" }
+serde_json = "1"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
 tokio = { version = "1.33.0" }

deltachat-jsonrpc/src/api.rs

@@ -231,13 +231,27 @@ impl CommandApi {
         Ok(())
     }
 
+    /// Performs a background fetch for all accounts in parallel with a timeout.
+    ///
+    /// The `AccountsBackgroundFetchDone` event is emitted at the end even in case of timeout.
+    /// Process all events until you get this one and you can safely return to the background
+    /// without forgetting to create notifications caused by timing race conditions.
+    async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
+        self.accounts
+            .write()
+            .await
+            .background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
+            .await;
+        Ok(())
+    }
+
     // ---------------------------------------------
     // Methods that work on individual accounts
     // ---------------------------------------------
 
     /// Starts background tasks for a single account.
     async fn start_io(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
@@ -311,6 +325,11 @@ impl CommandApi {
         ctx.get_info().await
     }
 
+    async fn draft_self_report(&self, account_id: u32) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx.draft_self_report().await?.to_u32())
+    }
+
     /// Sets the given configuration key.
     async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
@@ -383,7 +402,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -896,19 +915,35 @@ impl CommandApi {
             .to_u32())
     }
 
-    // for now only text messages, because we only used text messages in desktop thusfar
+    /// Add a message to the device-chat.
+    /// Device-messages usually contain update information
+    /// and some hints that are added during the program runs, multi-device etc.
+    /// The device-message may be defined by a label;
+    /// if a message with the same label was added or skipped before,
+    /// the message is not added again, even if the message was deleted in between.
+    /// If needed, the device-chat is created before.
+    ///
+    /// Sends the `MsgsChanged` event on success.
+    ///
+    /// Setting msg to None will prevent the device message with this label from being added in the future.
     async fn add_device_message(
         &self,
         account_id: u32,
         label: String,
-        text: String,
-    ) -> Result<u32> {
+        msg: Option<MessageData>,
+    ) -> Result<Option<u32>> {
         let ctx = self.get_context(account_id).await?;
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(text);
-        let message_id =
-            deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut msg)).await?;
-        Ok(message_id.to_u32())
+        if let Some(msg) = msg {
+            let mut message = msg.create_message(&ctx).await?;
+            let message_id =
+                deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut message)).await?;
+            if !message_id.is_unset() {
+                return Ok(Some(message_id.to_u32()));
+            }
+        } else {
+            deltachat::chat::add_device_msg(&ctx, Some(&label), None).await?;
+        }
+        Ok(None)
     }
 
     ///  Mark all messages in a chat as _noticed_.
@@ -1808,38 +1843,7 @@ impl CommandApi {
 
     async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
-        let mut message = Message::new(if let Some(viewtype) = data.viewtype {
-            viewtype.into()
-        } else if data.file.is_some() {
-            Viewtype::File
-        } else {
-            Viewtype::Text
-        });
-        message.set_text(data.text.unwrap_or_default());
-        if data.html.is_some() {
-            message.set_html(data.html);
-        }
-        if data.override_sender_name.is_some() {
-            message.set_override_sender_name(data.override_sender_name);
-        }
-        if let Some(file) = data.file {
-            message.set_file(file, None);
-        }
-        if let Some((latitude, longitude)) = data.location {
-            message.set_location(latitude, longitude);
-        }
-        if let Some(id) = data.quoted_message_id {
-            message
-                .set_quote(
-                    &ctx,
-                    Some(
-                        &Message::load_from_db(&ctx, MsgId::new(id))
-                            .await
-                            .context("message to quote could not be loaded")?,
-                    ),
-                )
-                .await?;
-        }
+        let mut message = data.create_message(&ctx).await?;
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
             .await?
             .to_u32();
@@ -2121,13 +2125,6 @@ async fn set_config(
             value,
         )
         .await?;
-
-        match key {
-            "sentbox_watch" | "mvbox_move" | "only_fetch_mvbox" => {
-                ctx.restart_io_if_running().await;
-            }
-            _ => {}
-        }
     }
     Ok(())
 }

deltachat-jsonrpc/src/api/types/chat.rs

@@ -85,7 +85,7 @@ impl FullChat {
         let can_send = chat.can_send(context).await?;
 
         let was_seen_recently = if chat.get_type() == Chattype::Single {
-            match contact_ids.get(0) {
+            match contact_ids.first() {
                 Some(contact) => Contact::get_by_id(context, *contact)
                     .await
                     .context("failed to load contact for was_seen_recently")?

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -102,7 +102,7 @@ pub(crate) async fn get_chat_list_item_by_id(
     let self_in_group = chat_contacts.contains(&ContactId::SELF);
 
     let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
-        let contact = chat_contacts.get(0);
+        let contact = chat_contacts.first();
         let was_seen_recently = match contact {
             Some(contact) => Contact::get_by_id(ctx, *contact)
                 .await

deltachat-jsonrpc/src/api/types/contact.rs

@@ -45,6 +45,9 @@ pub struct ContactObject {
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
+
+    /// If the contact is a bot.
+    is_bot: bool,
 }
 
 impl ContactObject {
@@ -80,6 +83,7 @@ impl ContactObject {
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),
+            is_bot: contact.is_bot(),
         })
     }
 }

deltachat-jsonrpc/src/api/types/events.rs

@@ -28,55 +28,37 @@ pub enum EventType {
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Info {
-        msg: String,
-    },
+    Info { msg: String },
 
     /// Emitted when SMTP connection is established and login was successful.
-    SmtpConnected {
-        msg: String,
-    },
+    SmtpConnected { msg: String },
 
     /// Emitted when IMAP connection is established and login was successful.
-    ImapConnected {
-        msg: String,
-    },
+    ImapConnected { msg: String },
 
     /// Emitted when a message was successfully sent to the SMTP server.
-    SmtpMessageSent {
-        msg: String,
-    },
+    SmtpMessageSent { msg: String },
 
     /// Emitted when an IMAP message has been marked as deleted
-    ImapMessageDeleted {
-        msg: String,
-    },
+    ImapMessageDeleted { msg: String },
 
     /// Emitted when an IMAP message has been moved
-    ImapMessageMoved {
-        msg: String,
-    },
+    ImapMessageMoved { msg: String },
 
     /// Emitted before going into IDLE on the Inbox folder.
     ImapInboxIdle,
 
     /// Emitted when an new file in the $BLOBDIR was created
-    NewBlobFile {
-        file: String,
-    },
+    NewBlobFile { file: String },
 
     /// Emitted when an file in the $BLOBDIR was deleted
-    DeletedBlobFile {
-        file: String,
-    },
+    DeletedBlobFile { file: String },
 
     /// The library-user should write a warning string to the log.
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Warning {
-        msg: String,
-    },
+    Warning { msg: String },
 
     /// The library-user should report an error to the end-user.
     ///
@@ -88,18 +70,14 @@ pub enum EventType {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
-    Error {
-        msg: String,
-    },
+    Error { msg: String },
 
     /// An action cannot be performed because the user is not in the group.
     /// Reported eg. after a call to
     /// setChatName(), setChatProfileImage(),
     /// addContactToChat(), removeContactFromChat(),
     /// and messages sending functions.
-    ErrorSelfNotInGroup {
-        msg: String,
-    },
+    ErrorSelfNotInGroup { msg: String },
 
     /// Messages or chats changed.  One or more messages or chats changed for various
     /// reasons in the database:
@@ -110,10 +88,7 @@ pub enum EventType {
     /// `chatId` is set if only a single chat is affected by the changes, otherwise 0.
     /// `msgId` is set if only a single message is affected by the changes, otherwise 0.
     #[serde(rename_all = "camelCase")]
-    MsgsChanged {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgsChanged { chat_id: u32, msg_id: u32 },
 
     /// Reactions for the message changed.
     #[serde(rename_all = "camelCase")]
@@ -128,10 +103,7 @@ pub enum EventType {
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
     #[serde(rename_all = "camelCase")]
-    IncomingMsg {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    IncomingMsg { chat_id: u32, msg_id: u32 },
 
     /// Downloading a bunch of messages just finished. This is an experimental
     /// event to allow the UI to only show one notification per message bunch,
@@ -139,47 +111,31 @@ pub enum EventType {
     ///
     /// msg_ids contains the message ids.
     #[serde(rename_all = "camelCase")]
-    IncomingMsgBunch {
-        msg_ids: Vec<u32>,
-    },
+    IncomingMsgBunch { msg_ids: Vec<u32> },
 
     /// Messages were seen or noticed.
     /// chat id is always set.
     #[serde(rename_all = "camelCase")]
-    MsgsNoticed {
-        chat_id: u32,
-    },
+    MsgsNoticed { chat_id: u32 },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgDelivered {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDelivered { chat_id: u32, msg_id: u32 },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgFailed {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgFailed { chat_id: u32, msg_id: u32 },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgRead {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgRead { chat_id: u32, msg_id: u32 },
 
     /// A single message is deleted.
     #[serde(rename_all = "camelCase")]
-    MsgDeleted {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDeleted { chat_id: u32, msg_id: u32 },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
@@ -189,24 +145,17 @@ pub enum EventType {
     /// This event does not include ephemeral timer modification, which
     /// is a separate event.
     #[serde(rename_all = "camelCase")]
-    ChatModified {
-        chat_id: u32,
-    },
+    ChatModified { chat_id: u32 },
 
     /// Chat ephemeral timer changed.
     #[serde(rename_all = "camelCase")]
-    ChatEphemeralTimerModified {
-        chat_id: u32,
-        timer: u32,
-    },
+    ChatEphemeralTimerModified { chat_id: u32, timer: u32 },
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[serde(rename_all = "camelCase")]
-    ContactsChanged {
-        contact_id: Option<u32>,
-    },
+    ContactsChanged { contact_id: Option<u32> },
 
     /// Location of one or more contact has changed.
     ///
@@ -214,9 +163,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     this parameter is set to `None`.
     #[serde(rename_all = "camelCase")]
-    LocationChanged {
-        contact_id: Option<u32>,
-    },
+    LocationChanged { contact_id: Option<u32> },
 
     /// Inform about the configuration progress started by configure().
     ConfigureProgress {
@@ -234,9 +181,7 @@ pub enum EventType {
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexProgress {
-        progress: usize,
-    },
+    ImexProgress { progress: usize },
 
     /// A file has been exported. A file has been written by imex().
     /// This event may be sent multiple times by a single call to imex().
@@ -246,9 +191,7 @@ pub enum EventType {
     ///
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexFileWritten {
-        path: String,
-    },
+    ImexFileWritten { path: String },
 
     /// Progress information of a secure-join handshake from the view of the inviter
     /// (Alice, the person who shows the QR code).
@@ -263,10 +206,7 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[serde(rename_all = "camelCase")]
-    SecurejoinInviterProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -277,10 +217,7 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[serde(rename_all = "camelCase")]
-    SecurejoinJoinerProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -288,8 +225,17 @@ pub enum EventType {
     /// getConnectivityHtml() for details.
     ConnectivityChanged,
 
+    /// Deprecated by `ConfigSynced`.
     SelfavatarChanged,
 
+    /// A multi-device synced config value changed. Maybe the app needs to refresh smth. For
+    /// uniformity this is emitted on the source device too. The value isn't here, otherwise it
+    /// would be logged which might not be good for privacy.
+    ConfigSynced {
+        /// Configuration key.
+        key: String,
+    },
+
     #[serde(rename_all = "camelCase")]
     WebxdcStatusUpdate {
         msg_id: u32,
@@ -298,9 +244,14 @@ pub enum EventType {
 
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
-    WebxdcInstanceDeleted {
-        msg_id: u32,
-    },
+    WebxdcInstanceDeleted { msg_id: u32 },
+
+    /// Tells that the Background fetch was completed (or timed out).
+    /// This event acts as a marker, when you reach this event you can be sure
+    /// that all events emitted during the background fetch were processed.
+    ///
+    /// This event is only emitted by the account manager
+    AccountsBackgroundFetchDone,
 }
 
 impl From<CoreEventType> for EventType {
@@ -396,6 +347,9 @@ impl From<CoreEventType> for EventType {
             },
             CoreEventType::ConnectivityChanged => ConnectivityChanged,
             CoreEventType::SelfavatarChanged => SelfavatarChanged,
+            CoreEventType::ConfigSynced { key } => ConfigSynced {
+                key: key.to_string(),
+            },
             CoreEventType::WebxdcStatusUpdate {
                 msg_id,
                 status_update_serial,
@@ -406,6 +360,7 @@ impl From<CoreEventType> for EventType {
             CoreEventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
                 msg_id: msg_id.to_u32(),
             },
+            CoreEventType::AccountsBackgroundFetchDone => AccountsBackgroundFetchDone,
         }
     }
 }

deltachat-jsonrpc/src/api/types/message.rs

@@ -345,6 +345,7 @@ pub enum SystemMessageType {
     SecurejoinMessage,
     LocationStreamingEnabled,
     LocationOnly,
+    InvalidUnencryptedMail,
 
     /// Chat ephemeral message timer is changed.
     EphemeralTimerChanged,
@@ -385,6 +386,7 @@ impl From<deltachat::mimeparser::SystemMessage> for SystemMessageType {
             SystemMessage::MultiDeviceSync => SystemMessageType::MultiDeviceSync,
             SystemMessage::WebxdcStatusUpdate => SystemMessageType::WebxdcStatusUpdate,
             SystemMessage::WebxdcInfoMessage => SystemMessageType::WebxdcInfoMessage,
+            SystemMessage::InvalidUnencryptedMail => SystemMessageType::InvalidUnencryptedMail,
         }
     }
 }
@@ -546,6 +548,44 @@ pub struct MessageData {
     pub quoted_message_id: Option<u32>,
 }
 
+impl MessageData {
+    pub(crate) async fn create_message(self, context: &Context) -> Result<Message> {
+        let mut message = Message::new(if let Some(viewtype) = self.viewtype {
+            viewtype.into()
+        } else if self.file.is_some() {
+            Viewtype::File
+        } else {
+            Viewtype::Text
+        });
+        message.set_text(self.text.unwrap_or_default());
+        if self.html.is_some() {
+            message.set_html(self.html);
+        }
+        if self.override_sender_name.is_some() {
+            message.set_override_sender_name(self.override_sender_name);
+        }
+        if let Some(file) = self.file {
+            message.set_file(file, None);
+        }
+        if let Some((latitude, longitude)) = self.location {
+            message.set_location(latitude, longitude);
+        }
+        if let Some(id) = self.quoted_message_id {
+            message
+                .set_quote(
+                    context,
+                    Some(
+                        &Message::load_from_db(context, MsgId::new(id))
+                            .await
+                            .context("message to quote could not be loaded")?,
+                    ),
+                )
+                .await?;
+        }
+        Ok(message)
+    }
+}
+
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct MessageReadReceipt {

deltachat-jsonrpc/typescript/package.json

@@ -53,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.131.9"
+  "version": "1.135.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.131.9"
+version = "1.135.0"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -12,7 +12,7 @@ dirs = "5"
 log = "0.4.20"
 pretty_env_logger = "0.5"
 rusqlite = "0.30"
-rustyline = "12"
+rustyline = "13"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 
 [features]

deltachat-repl/src/main.rs

@@ -299,8 +299,8 @@ impl Highlighter for DcHelper {
         self.highlighter.highlight(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize) -> bool {
-        self.highlighter.highlight_char(line, pos)
+    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
+        self.highlighter.highlight_char(line, pos, forced)
     }
 }
 
@@ -401,7 +401,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    mut ctx: Context,
+    ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
@@ -481,7 +481,10 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    let _ = pretty_env_logger::try_init();
+    pretty_env_logger::formatted_timed_builder()
+        .parse_default_env()
+        .filter_module("deltachat_repl", log::LevelFilter::Info)
+        .init();
 
     let args = std::env::args().collect();
     start(args).await?;

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -1,4 +1,5 @@
 """Delta Chat JSON-RPC high-level API"""
+
 from ._utils import AttrDict, run_bot_cli, run_client_cli
 from .account import Account
 from .chat import Chat

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -300,3 +300,13 @@ class Account:
     def import_backup(self, path, passphrase: str = "") -> None:
         """Import backup."""
         self._rpc.import_backup(self.id, str(path), passphrase)
+
+    def export_self_keys(self, path) -> None:
+        """Export keys."""
+        passphrase = ""  # Setting passphrase is currently not supported.
+        self._rpc.export_self_keys(self.id, str(path), passphrase)
+
+    def import_self_keys(self, path) -> None:
+        """Import keys."""
+        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
+        self._rpc.import_self_keys(self.id, str(path), passphrase)

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -1,4 +1,5 @@
 """Event loop implementations offering high level event handling/hooking."""
+
 import logging
 from typing import (
     TYPE_CHECKING,

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -1,4 +1,5 @@
 """High-level classes for event processing and filtering."""
+
 import re
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,9 +1,10 @@
 import logging
 
+import pytest
 from deltachat_rpc_client import Chat, SpecialContactId
 
 
-def test_qr_setup_contact(acfactory) -> None:
+def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     qr_code, _svg = alice.get_qr_code()
@@ -23,13 +24,26 @@ def test_qr_setup_contact(acfactory) -> None:
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
+    # Test that if Bob changes the key, backwards verification is lost.
+    logging.info("Bob 2 is created")
+    bob2 = acfactory.new_configured_account()
+    bob2.export_self_keys(tmp_path)
 
-def test_qr_securejoin(acfactory):
+    logging.info("Bob imports a key")
+    bob.import_self_keys(tmp_path / "private-key-default.asc")
+
+    assert bob.get_config("key_id") == "2"
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert not bob_contact_alice_snapshot.is_verified
+
+
+@pytest.mark.parametrize("protect", [True, False])
+def test_qr_securejoin(acfactory, protect):
     alice, bob = acfactory.get_online_accounts(2)
 
     logging.info("Alice creates a verified group")
-    alice_chat = alice.create_group("Verified group", protect=True)
-    assert alice_chat.get_basic_snapshot().is_protected
+    alice_chat = alice.create_group("Verified group", protect=protect)
+    assert alice_chat.get_basic_snapshot().is_protected == protect
 
     logging.info("Bob joins verified group")
     qr_code, _svg = alice_chat.get_qr_code()
@@ -53,7 +67,7 @@ def test_qr_securejoin(acfactory):
 
     snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
-    assert snapshot.chat.get_basic_snapshot().is_protected
+    assert snapshot.chat.get_basic_snapshot().is_protected == protect
 
     # Test that Bob verified Alice's profile.
     bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
@@ -161,7 +175,7 @@ def test_verified_group_recovery(acfactory) -> None:
     logging.info("ac2 joins verified group")
     qr_code, _svg = chat.get_qr_code()
     ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     # ac1 has ac2 directly verified.
     ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
@@ -169,7 +183,8 @@ def test_verified_group_recovery(acfactory) -> None:
 
     logging.info("ac3 joins verified group")
     ac3_chat = ac3.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
 
     logging.info("ac2 logs in on a new device")
     ac2 = acfactory.resetup_account(ac2)
@@ -177,8 +192,7 @@ def test_verified_group_recovery(acfactory) -> None:
     logging.info("ac2 reverifies with ac3")
     qr_code, _svg = ac3.get_qr_code()
     ac2.secure_join(qr_code)
-
-    ac3.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("ac3 sends a message to the group")
     assert len(ac3_chat.get_contacts()) == 3
@@ -225,7 +239,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     logging.info("ac2 joins verified group")
     qr_code, _svg = chat.get_qr_code()
     ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     # ac1 has ac2 directly verified.
     ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
@@ -233,7 +247,8 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
 
     logging.info("ac3 joins verified group")
     ac3_chat = ac3.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
 
     logging.info("ac2 logs in on a new device")
     ac2 = acfactory.resetup_account(ac2)
@@ -241,8 +256,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     logging.info("ac2 reverifies with ac3")
     qr_code, _svg = ac3.get_qr_code()
     ac2.secure_join(qr_code)
-
-    ac3.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("ac3 sends a message to the group")
     assert len(ac3_chat.get_contacts()) == 3
@@ -477,3 +491,76 @@ def test_gossip_verification(acfactory) -> None:
     # Securejoin propagates verification.
     carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
     assert carol_contact_alice_snapshot.is_verified
+
+
+def test_securejoin_after_contact_resetup(acfactory) -> None:
+    """
+    Regression test for a bug that prevented joining verified group with a QR code
+    if the group is already created and contains
+    a contact with inconsistent (Autocrypt and verified keys exist but don't match) key state.
+    """
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    # ac3 creates protected group with ac1.
+    ac3_chat = ac3.create_group("Verified group", protect=True)
+
+    # ac1 joins ac3 group.
+    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac1.secure_join(ac3_qr_code)
+    ac1.wait_for_securejoin_joiner_success()
+
+    # ac1 waits for member added message and creates a QR code.
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+
+    # ac2 verifies ac1
+    qr_code, _svg = ac1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    # ac1 is verified for ac2.
+    ac2_contact_ac1 = ac2.create_contact(ac1.get_config("addr"), "")
+    assert ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 resetups the account.
+    ac1 = acfactory.resetup_account(ac1)
+
+    # ac1 sends a message to ac2.
+    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
+    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
+    ac1_chat_ac2.send_text("Hello!")
+
+    # ac2 receives a message.
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+
+    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
+    assert not ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 goes offline.
+    ac1.remove()
+
+    # Scanning a QR code results in creating an unprotected group with an inviter.
+    # In this case inviter is ac1 which has an inconsistent key state.
+    # Normally inviter becomes verified as a result of Securejoin protocol
+    # and then the group chat becomes verified when "Member added" is received,
+    # but in this case ac1 is offline and this Securejoin process will never finish.
+    logging.info("ac2 scans ac1 QR code, this is not expected to finish")
+    ac2.secure_join(ac1_qr_code)
+
+    logging.info("ac2 scans ac3 QR code")
+    ac2.secure_join(ac3_qr_code)
+
+    logging.info("ac2 waits for joiner success")
+    ac2.wait_for_securejoin_joiner_success()
+
+    # Wait for member added.
+    logging.info("ac2 waits for member added message")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.is_info
+    ac2_chat = snapshot.chat
+    assert ac2_chat.get_basic_snapshot().is_protected
+    assert len(ac2_chat.get_contacts()) == 3
+
+    # ac1 is still "not verified" for ac2 due to inconsistent state.
+    assert not ac2_contact_ac1.get_snapshot().is_verified

deltachat-rpc-client/tests/test_something.py

@@ -140,12 +140,9 @@ def test_chat(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
     assert snapshot.chat_id == chat_id
@@ -224,12 +221,9 @@ def test_message(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
 
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
@@ -331,7 +325,7 @@ def test_wait_next_messages(acfactory) -> None:
         next_messages_task = executor.submit(bot.wait_next_messages)
 
         bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
         alice_chat_bot = alice_contact_bot.create_chat()
         alice_chat_bot.send_text("Hello!")
 
@@ -341,7 +335,7 @@ def test_wait_next_messages(acfactory) -> None:
         assert snapshot.text == "Hello!"
 
 
-def test_import_export(acfactory, tmp_path) -> None:
+def test_import_export_backup(acfactory, tmp_path) -> None:
     alice = acfactory.new_configured_account()
     alice.export_backup(tmp_path)
 
@@ -352,6 +346,31 @@ def test_import_export(acfactory, tmp_path) -> None:
     assert alice2.manager.get_system_info()
 
 
+def test_import_export_keys(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello Bob!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Bob!"
+
+    # Alice resetups account, but keeps the key.
+    alice_keys_path = tmp_path / "alice_keys"
+    alice_keys_path.mkdir()
+    alice.export_self_keys(alice_keys_path)
+    alice = acfactory.resetup_account(alice)
+    alice.import_self_keys(alice_keys_path)
+
+    snapshot.chat.accept()
+    snapshot.chat.send_text("Hello Alice!")
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Alice!"
+    assert snapshot.show_padlock
+
+
 def test_openrpc_command_line() -> None:
     """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
     out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
@@ -377,3 +396,46 @@ def test_provider_info(rpc) -> None:
     rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
+
+
+def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+
+    # Bob creates chat manually so chat with Alice is accepted.
+    alice_chat_bob = alice_contact_bob.create_chat()
+
+    # Alice sends a message to Bob.
+    alice_chat_bob.send_text("Hello Bob!")
+    event = bob.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = bob.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+
+    # Bob sends a message to Alice.
+    bob_chat_alice = snapshot.chat
+    bob_chat_alice.accept()
+    bob_chat_alice.send_text("Hello Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock
+
+    # Alice reads Bob's message.
+    message.mark_seen()
+    while True:
+        event = bob.wait_for_event()
+        if event.kind == EventType.MSG_READ:
+            break
+
+    # Bob sends a message to Alice, it should also be encrypted.
+    bob_chat_alice.send_text("Hi Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.131.9"
+version = "1.135.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -15,9 +15,9 @@ deltachat = { path = "..", default-features = false }
 
 anyhow = "1"
 env_logger = { version = "0.10.0" }
-futures-lite = "2.0.0"
+futures-lite = "2.2.0"
 log = "0.4"
-serde_json = "1.0.105"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "1.33.0", features = ["io-std"] }
 tokio-util = "0.7.9"

deny.toml

@@ -33,7 +33,6 @@ skip = [
      { name = "ed25519-dalek", version = "1.0.1" },
      { name = "ed25519", version = "1.5.3" },
      { name = "event-listener", version = "2.5.3" },
-     { name = "fd-lock", version = "3.0.13" },
      { name = "getrandom", version = "<0.2" },
      { name = "h2", version = "0.3.22" },
      { name = "http-body", version = "0.4.5" },
@@ -53,7 +52,6 @@ skip = [
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },
-     { name = "socket2", version = "0.4.9" },
      { name = "spin", version = "<0.9.6" },
      { name = "spki", version = "0.6.0" },
      { name = "syn", version = "1.0.109" },
@@ -101,6 +99,4 @@ license-files = [
 github = [
        "async-email",
        "deltachat",
-       "djc",
-       "n0-computer", # iroh
 ]

fuzz/Cargo.lock

@@ -190,11 +190,11 @@ dependencies = [
 
 [[package]]
 name = "async-imap"
-version = "0.9.1"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b538b767cbf9c162a6c5795d4b932bd2c20ba10b5a91a94d2b2b6886c1dce6a8"
+checksum = "a9d69fc1499878158750f644c4eb46aff55bb9d32d77e3dc4aecf8308d5c3ba6"
 dependencies = [
- "async-channel 1.8.0",
+ "async-channel 2.1.1",
  "base64 0.21.0",
  "bytes",
  "chrono",
@@ -922,7 +922,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.131.9"
+version = "1.133.0"
 dependencies = [
  "anyhow",
  "async-channel 2.1.1",
@@ -963,6 +963,7 @@ dependencies = [
  "pin-project",
  "qrcodegen",
  "quick-xml",
+ "quoted_printable 0.5.0",
  "rand 0.8.5",
  "ratelimit",
  "regex",
@@ -1859,9 +1860,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.17"
+version = "0.3.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f"
+checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9"
 dependencies = [
  "bytes",
  "fnv",
@@ -1869,7 +1870,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http",
- "indexmap 1.9.2",
+ "indexmap",
  "slab",
  "tokio",
  "tokio-util",
@@ -2146,16 +2147,6 @@ dependencies = [
  "nom",
 ]
 
-[[package]]
-name = "indexmap"
-version = "1.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
-dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
-]
-
 [[package]]
 name = "indexmap"
 version = "2.1.0"
@@ -2215,7 +2206,8 @@ checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e"
 [[package]]
 name = "iroh"
 version = "0.4.2"
-source = "git+https://github.com/n0-computer/iroh?branch=maint-0.4#9881b7886235035a1124e4371f7a4cd59379e51b"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85075391dcb8491a4939266334b28601052d418b37d20b33c58ffb5776adc912"
 dependencies = [
  "abao",
  "anyhow",
@@ -2420,7 +2412,7 @@ checksum = "8cae768a50835557749599277fc59f7c728118724eb34185e8feb633ef266a32"
 dependencies = [
  "charset",
  "data-encoding",
- "quoted_printable",
+ "quoted_printable 0.4.6",
 ]
 
 [[package]]
@@ -2431,7 +2423,7 @@ checksum = "6b56570f5f8c0047260d1c8b5b331f62eb9c660b9dd4071a8c46f8c7d3f280aa"
 dependencies = [
  "charset",
  "data-encoding",
- "quoted_printable",
+ "quoted_printable 0.4.6",
 ]
 
 [[package]]
@@ -3258,6 +3250,12 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20f14e071918cbeefc5edc986a7aa92c425dae244e003a35e1cdddb5ca39b5cb"
 
+[[package]]
+name = "quoted_printable"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79ec282e887b434b68c18fe5c121d38e72a5cf35119b59e54ec5b992ea9c8eb0"
+
 [[package]]
 name = "rand"
 version = "0.7.3"
@@ -3411,9 +3409,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
 
 [[package]]
 name = "reqwest"
-version = "0.11.20"
+version = "0.11.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
+checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41"
 dependencies = [
  "base64 0.21.0",
  "bytes",
@@ -3436,6 +3434,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded",
+ "system-configuration",
  "tokio",
  "tokio-native-tls",
  "tower-service",
@@ -4156,9 +4155,9 @@ dependencies = [
 
 [[package]]
 name = "system-configuration"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
 dependencies = [
  "bitflags 1.3.2",
  "core-foundation",
@@ -4429,7 +4428,7 @@ version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03"
 dependencies = [
- "indexmap 2.1.0",
+ "indexmap",
  "serde",
  "serde_spanned",
  "toml_datetime",

node/constants.js

@@ -29,9 +29,11 @@ module.exports = {
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
+  DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
+  DC_EVENT_CONFIG_SYNCED: 2111,
   DC_EVENT_CONNECTIVITY_CHANGED: 2100,
   DC_EVENT_CONTACTS_CHANGED: 2030,
   DC_EVENT_DELETED_BLOB_FILE: 151,
@@ -79,6 +81,7 @@ module.exports = {
   DC_INFO_EPHEMERAL_TIMER_CHANGED: 10,
   DC_INFO_GROUP_IMAGE_CHANGED: 3,
   DC_INFO_GROUP_NAME_CHANGED: 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL: 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED: 8,
   DC_INFO_LOCATION_ONLY: 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP: 4,
@@ -225,11 +228,13 @@ module.exports = {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU: 124,
   DC_STR_IMAGE: 9,
   DC_STR_INCOMING_MESSAGES: 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL: 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY: 111,
   DC_STR_LOCATION: 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER: 137,
   DC_STR_LOCATION_ENABLED_BY_YOU: 136,
   DC_STR_MESSAGES: 114,
+  DC_STR_MESSAGE_ADD_MEMBER: 173,
   DC_STR_MSGACTIONBYME: 63,
   DC_STR_MSGACTIONBYUSER: 62,
   DC_STR_MSGADDMEMBER: 17,

node/events.js

@@ -34,6 +34,8 @@ module.exports = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
-  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED'
+  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE'
 }

node/lib/constants.ts

@@ -29,9 +29,11 @@ export enum C {
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
+  DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
+  DC_EVENT_CONFIG_SYNCED = 2111,
   DC_EVENT_CONNECTIVITY_CHANGED = 2100,
   DC_EVENT_CONTACTS_CHANGED = 2030,
   DC_EVENT_DELETED_BLOB_FILE = 151,
@@ -79,6 +81,7 @@ export enum C {
   DC_INFO_EPHEMERAL_TIMER_CHANGED = 10,
   DC_INFO_GROUP_IMAGE_CHANGED = 3,
   DC_INFO_GROUP_NAME_CHANGED = 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL = 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED = 8,
   DC_INFO_LOCATION_ONLY = 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP = 4,
@@ -225,11 +228,13 @@ export enum C {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU = 124,
   DC_STR_IMAGE = 9,
   DC_STR_INCOMING_MESSAGES = 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL = 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY = 111,
   DC_STR_LOCATION = 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER = 137,
   DC_STR_LOCATION_ENABLED_BY_YOU = 136,
   DC_STR_MESSAGES = 114,
+  DC_STR_MESSAGE_ADD_MEMBER = 173,
   DC_STR_MSGACTIONBYME = 63,
   DC_STR_MSGACTIONBYUSER = 62,
   DC_STR_MSGADDMEMBER = 17,
@@ -319,6 +324,8 @@ export const EventId2EventName: { [key: number]: string } = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
 }

node/lib/deltachat.ts

@@ -178,7 +178,7 @@ export class AccountManager extends EventEmitter {
   static newTemporary() {
     let directory = null
     while (true) {
-      const randomString = Math.random().toString(36).substr(2, 5)
+      const randomString = Math.random().toString(36).substring(2, 5)
       directory = join(tmpdir(), 'deltachat-' + randomString)
       if (!existsSync(directory)) break
     }

node/test/test.mjs

@@ -1,13 +1,17 @@
 // @ts-check
-import DeltaChat from '../dist'
+import { DeltaChat } from '../dist/index.js'
 
 import { deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
-import { EventId2EventName, C } from '../dist/constants'
+import { EventId2EventName, C } from '../dist/constants.js'
 import { join } from 'path'
 import { statSync } from 'fs'
-import { Context } from '../dist/context'
+import { Context } from '../dist/context.js'
+import {fileURLToPath} from 'url';
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+
 chai.use(chaiAsPromised)
 chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 
@@ -246,6 +250,7 @@ describe('Basic offline Tests', function () {
       'journal_mode',
       'key_gen_type',
       'last_housekeeping',
+      'last_cant_decrypt_outgoing_msgs',
       'level',
       'mdns_enabled',
       'media_quality',

package.json

@@ -8,9 +8,8 @@
   "devDependencies": {
     "@types/debug": "^4.1.7",
     "@types/node": "^20.8.10",
-    "chai": "^4.2.0",
+    "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
-    "esm": "^3.2.25",
     "mocha": "^8.2.1",
     "node-gyp": "^10.0.0",
     "prebuildify": "^5.0.1",
@@ -53,8 +52,8 @@
     "prebuildify": "cd node && prebuildify -t 18.0.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
-    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
+    "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.131.9"
+  "version": "1.135.0"
 }

python/src/deltachat/account.py

@@ -1,6 +1,5 @@
 """Account class implementation."""
 
-
 import os
 from array import array
 from contextlib import contextmanager
@@ -478,6 +477,16 @@ class Account:
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
+    def resend_messages(self, messages: List[Message]) -> None:
+        """Resend list of messages.
+
+        :param messages: list of :class:`deltachat.message.Message` object.
+        :returns: None
+        """
+        msg_ids = [msg.id for msg in messages]
+        if lib.dc_resend_msgs(self._dc_context, msg_ids, len(msg_ids)) != 1:
+            raise ValueError(f"could not resend messages {msg_ids}")
+
     def delete_messages(self, messages: List[Message]) -> None:
         """delete messages (local and remote).
 

python/src/deltachat/testplugin.py

@@ -10,6 +10,7 @@ import time
 import weakref
 import random
 from queue import Queue
+from threading import Event
 from typing import Callable, Dict, List, Optional, Set
 
 import pytest
@@ -590,6 +591,27 @@ class ACFactory:
         ac2.create_chat(ac1)
         return ac1.create_chat(ac2)
 
+    def get_protected_chat(self, ac1: Account, ac2: Account):
+        class SetupPlugin:
+            def __init__(self) -> None:
+                self.member_added = Event()
+
+            @account_hookimpl
+            def ac_member_added(self, chat: deltachat.Chat, contact, actor, message):
+                self.member_added.set()
+
+        setupplugin = SetupPlugin()
+        ac1.add_account_plugin(setupplugin)
+        chat = ac1.create_group_chat("Protected Group", verified=True)
+        qr = chat.get_join_qr()
+        ac2.qr_join_chat(qr)
+        setupplugin.member_added.wait()
+        msg = ac2.wait_next_incoming_message()
+        assert msg.text == "Messages are guaranteed to be end-to-end encrypted from now on."
+        msg = ac2.wait_next_incoming_message()
+        assert "Member Me " in msg.text and " added by " in msg.text
+        return chat
+
     def introduce_each_other(self, accounts, sending=True):
         to_wait = []
         for i, acc in enumerate(accounts):

python/tests/test_1_online.py

@@ -382,6 +382,21 @@ def test_webxdc_download_on_demand(acfactory, data, lp):
     assert msgs_changed_event.data2 == 0
 
 
+def test_enable_mvbox_move(acfactory, lp):
+    (ac1,) = acfactory.get_online_accounts(1)
+
+    lp.sec("ac2: start without mvbox thread")
+    ac2 = acfactory.new_online_configuring_account(mvbox_move=False)
+    acfactory.bring_accounts_online()
+
+    lp.sec("ac2: configuring mvbox")
+    ac2.set_config("mvbox_move", "1")
+
+    lp.sec("ac1: send message and wait for ac2 to receive it")
+    acfactory.get_accepted_chat(ac1, ac2).send_text("message1")
+    assert ac2._evtracker.wait_next_incoming_message().text == "message1"
+
+
 def test_mvbox_sentbox_threads(acfactory, lp):
     lp.sec("ac1: start with mvbox thread")
     ac1 = acfactory.new_online_configuring_account(mvbox_move=True, sentbox_watch=True)
@@ -498,6 +513,26 @@ def test_forward_messages(acfactory, lp):
     assert not chat3.get_messages()
 
 
+def test_forward_encrypted_to_unencrypted(acfactory, lp):
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+    chat = acfactory.get_protected_chat(ac1, ac2)
+
+    lp.sec("ac1: send encrypted message to ac2")
+    txt = "This should be encrypted"
+    chat.send_text(txt)
+    msg = ac2.wait_next_incoming_message()
+    assert msg.text == txt
+    assert msg.is_encrypted()
+
+    lp.sec("ac2: forward message to ac3 unencrypted")
+    unencrypted_chat = ac2.create_chat(ac3)
+    msg_id = msg.id
+    msg2 = unencrypted_chat.send_msg(msg)
+    assert msg2 == msg
+    assert msg.id != msg_id
+    assert not msg.is_encrypted()
+
+
 def test_forward_own_message(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     chat = acfactory.get_accepted_chat(ac1, ac2)
@@ -523,6 +558,27 @@ def test_forward_own_message(acfactory, lp):
     assert msg_in.is_forwarded()
 
 
+def test_resend_message(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat1 = ac1.create_chat(ac2)
+
+    lp.sec("ac1: send message to ac2")
+    chat1.send_text("message")
+
+    lp.sec("ac2: receive message")
+    msg_in = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in.text == "message"
+    chat2 = msg_in.chat
+    chat2_msg_cnt = len(chat2.get_messages())
+
+    lp.sec("ac1: resend message")
+    ac1.resend_messages([msg_in])
+
+    lp.sec("ac2: check that message is deleted")
+    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+    assert len(chat2.get_messages()) == chat2_msg_cnt
+
+
 def test_long_group_name(acfactory, lp):
     """See bug https://github.com/deltachat/deltachat-core-rust/issues/3650 "Space added before long
     group names after MIME serialization/deserialization".
@@ -1531,10 +1587,11 @@ def test_reactions_for_a_reordering_move(acfactory, lp):
 
 
 def test_import_export_online_all(acfactory, tmp_path, data, lp):
-    (ac1,) = acfactory.get_online_accounts(1)
+    (ac1, some1) = acfactory.get_online_accounts(2)
 
     lp.sec("create some chat content")
-    chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
+    some1_addr = some1.get_config("addr")
+    chat1 = ac1.create_contact(some1_addr, name="some1").create_chat()
     chat1.send_text("msg1")
     assert len(ac1.get_contacts(query="some1")) == 1
 
@@ -1551,7 +1608,7 @@ def test_import_export_online_all(acfactory, tmp_path, data, lp):
         contacts = ac.get_contacts(query="some1")
         assert len(contacts) == 1
         contact2 = contacts[0]
-        assert contact2.addr == "some1@example.org"
+        assert contact2.addr == some1_addr
         chat2 = contact2.create_chat()
         messages = chat2.get_messages()
         assert len(messages) == 3
@@ -1979,6 +2036,32 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
 
 
+def test_all_work_done(acfactory, lp):
+    """
+    Tests that calling start_io() immediately followed by maybe_network()
+    and then waiting for all_work_done() reliably fetches the messages
+    delivered while account was offline.
+    In other words, connectivity should not change to a state
+    where all_work_done() returns true until IMAP connection goes idle.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    ac1.stop_io()
+    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+    ac1.direct_imap.select_config_folder("inbox")
+    with ac1.direct_imap.idle() as idle1:
+        ac2.create_chat(ac1).send_text("Hi")
+        idle1.wait_for_new_message()
+
+    ac1.start_io()
+    ac1.maybe_network()
+    ac1._evtracker.wait_for_all_work_done()
+    msgs = ac1.create_chat(ac2).get_messages()
+    assert len(msgs) == 1
+    assert msgs[0].text == "Hi"
+
+
 def test_fetch_deleted_msg(acfactory, lp):
     """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
     hundreds of times, because uid_next was not updated.

release-date.in

@@ -1 +1 @@
-2023-12-02
+2024-02-13

scripts/build-python-docs.sh

@@ -9,5 +9,4 @@ python3 -m venv venv
 venv/bin/pip install ./python
 venv/bin/pip install ./deltachat-rpc-client
 venv/bin/pip install sphinx breathe sphinx_rtd_theme
-venv/bin/pip install ./deltachat-rpc-client
 venv/bin/sphinx-build -b html -a python/doc/ dist/html

scripts/create-provider-data-rs.py

@@ -220,9 +220,9 @@ if __name__ == "__main__":
 
     process_dir(Path(sys.argv[1]))
 
-    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| HashMap::from([\n"
+    out_all += "pub(crate) static PROVIDER_DATA: [(&str, &Provider); " + str(len(domains_set)) + "] = [\n";
     out_all += out_domains
-    out_all += "]));\n\n"
+    out_all += "];\n\n"
 
     out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| HashMap::from([\n"
     out_all += out_ids

scripts/set_core_version.py

@@ -102,7 +102,7 @@ def main():
                 found = True
         if not found:
             raise SystemExit(
-                f"{changelog_name} contains no entry for version: {newversion}"
+                f"CHANGELOG.md contains no entry for version: {newversion}"
             )
 
     for toml_filename in toml_list:

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=18f714cf73d0bdfb8b013fa344494ab80c92b477
+REV=2f3db24107e4802c2df0aa0a40f0e144006c0a9b
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

src/accounts.rs

@@ -5,6 +5,7 @@ use std::future::Future;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
+use futures::future::join_all;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
@@ -291,6 +292,42 @@ impl Accounts {
         }
     }
 
+    /// Performs a background fetch for all accounts in parallel.
+    ///
+    /// This is an auxiliary function and not part of public API.
+    /// Use [Accounts::background_fetch] instead.
+    async fn background_fetch_without_timeout(&self) {
+        async fn background_fetch_and_log_error(account: Context) {
+            if let Err(error) = account.background_fetch().await {
+                warn!(account, "{error:#}");
+            }
+        }
+
+        join_all(
+            self.accounts
+                .values()
+                .cloned()
+                .map(background_fetch_and_log_error),
+        )
+        .await;
+    }
+
+    /// Performs a background fetch for all accounts in parallel with a timeout.
+    ///
+    /// The `AccountsBackgroundFetchDone` event is emitted at the end,
+    /// process all events until you get this one and you can safely return to the background
+    /// without forgetting to create notifications caused by timing race conditions.
+    pub async fn background_fetch(&self, timeout: std::time::Duration) {
+        if let Err(_err) =
+            tokio::time::timeout(timeout, self.background_fetch_without_timeout()).await
+        {
+            self.emit_event(EventType::Warning(
+                "Background fetch timed out.".to_string(),
+            ));
+        }
+        self.emit_event(EventType::AccountsBackgroundFetchDone);
+    }
+
     /// Emits a single event.
     pub fn emit_event(&self, event: EventType) {
         self.events.emit(Event { id: 0, typ: event })
@@ -424,11 +461,13 @@ impl Config {
     /// Takes a mutable reference because the saved file is a part of the `Config` state. This
     /// protects from parallel calls resulting to a wrong file contents.
     async fn sync(&mut self) -> Result<()> {
+        #[cfg(not(target_os = "ios"))]
         ensure!(!self
             .lock_task
             .as_ref()
             .context("Config is read-only")?
             .is_finished());
+
         let tmp_path = self.file.with_extension("toml.tmp");
         let mut file = fs::File::create(&tmp_path)
             .await
@@ -544,8 +583,12 @@ impl Config {
             }
             if self.inner.selected_account == id {
                 // reset selected account
-                self.inner.selected_account =
-                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                self.inner.selected_account = self
+                    .inner
+                    .accounts
+                    .first()
+                    .map(|e| e.id)
+                    .unwrap_or_default();
             }
         }
 

src/authres.rs

@@ -216,7 +216,7 @@ async fn update_authservid_candidates(
     if old_ids != new_ids {
         let new_config = new_ids.into_iter().collect::<Vec<_>>().join(" ");
         context
-            .set_config(Config::AuthservIdCandidates, Some(&new_config))
+            .set_config_internal(Config::AuthservIdCandidates, Some(&new_config))
             .await?;
         // Updating the authservid candidates may mean that we now consider
         // emails as "failed" which "passed" previously, so we need to

src/chat.rs

@@ -19,8 +19,8 @@ use crate::chatlist::Chatlist;
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_LAST_SPECIAL,
-    DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS,
+    self, Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK,
+    DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS,
 };
 use crate::contact::{self, Contact, ContactAddress, ContactId, Origin};
 use crate::context::Context;
@@ -209,6 +209,30 @@ impl ChatId {
         self == DC_CHAT_ID_ALLDONE_HINT
     }
 
+    /// Returns [`ChatId`] of a chat that `msg` belongs to.
+    ///
+    /// Checks that `msg` is assigned to the right chat.
+    pub(crate) fn lookup_by_message(msg: &Message) -> Option<Self> {
+        if msg.chat_id == DC_CHAT_ID_TRASH {
+            return None;
+        }
+        if msg.download_state != DownloadState::Done
+            // TODO (2023-09-12): Added for backward compatibility with versions that did not have
+            // `DownloadState::Undecipherable`. Remove eventually with the comment in
+            // `MimeMessage::from_bytes()`.
+            || msg
+                .error
+                .as_ref()
+                .filter(|e| e.starts_with("Decrypting failed:"))
+                .is_some()
+        {
+            // If `msg` is not fully downloaded or undecipherable, it may have been assigned to the
+            // wrong chat (they often get assigned to the 1:1 chat with the sender).
+            return None;
+        }
+        Some(msg.chat_id)
+    }
+
     /// Returns the [`ChatId`] for the 1:1 chat with `contact_id`
     /// if it exists and is not blocked.
     ///
@@ -327,7 +351,7 @@ impl ChatId {
 
         info!(
             context,
-            "Created group/mailinglist '{}' grpid={} as {}, blocked={}.",
+            "Created group/mailinglist '{}' grpid={} as {}, blocked={}, protected={create_protected}.",
             &grpname,
             grpid,
             chat_id,
@@ -374,6 +398,7 @@ impl ChatId {
 
     pub(crate) async fn block_ex(self, context: &Context, sync: sync::Sync) -> Result<()> {
         let chat = Chat::load_from_db(context, self).await?;
+        let mut delete = false;
 
         match chat.typ {
             Chattype::Broadcast => {
@@ -392,7 +417,7 @@ impl ChatId {
             }
             Chattype::Group => {
                 info!(context, "Can't block groups yet, deleting the chat.");
-                self.delete(context).await?;
+                delete = true;
             }
             Chattype::Mailinglist => {
                 if self.set_blocked(context, Blocked::Yes).await? {
@@ -408,6 +433,9 @@ impl ChatId {
                 .log_err(context)
                 .ok();
         }
+        if delete {
+            self.delete(context).await?;
+        }
         Ok(())
     }
 
@@ -502,15 +530,7 @@ impl ChatId {
 
         match protect {
             ProtectionStatus::Protected => match chat.typ {
-                Chattype::Single | Chattype::Group | Chattype::Broadcast => {
-                    let contact_ids = get_chat_contacts(context, self).await?;
-                    for contact_id in contact_ids {
-                        let contact = Contact::get_by_id(context, contact_id).await?;
-                        if !contact.is_verified(context).await? {
-                            bail!("{} is not verified.", contact.get_display_name());
-                        }
-                    }
-                }
+                Chattype::Single | Chattype::Group | Chattype::Broadcast => {}
                 Chattype::Mailinglist => bail!("Cannot protect mailing lists"),
             },
             ProtectionStatus::Unprotected | ProtectionStatus::ProtectionBroken => {}
@@ -558,7 +578,7 @@ impl ChatId {
     ///
     /// `timestamp_sort` is used as the timestamp of the added message
     /// and should be the timestamp of the change happening.
-    pub(crate) async fn set_protection(
+    async fn set_protection_for_timestamp_sort(
         self,
         context: &Context,
         protect: ProtectionStatus,
@@ -580,6 +600,24 @@ impl ChatId {
         }
     }
 
+    /// Sets protection and sends or adds a message.
+    ///
+    /// `timestamp_sent` is the "sent" timestamp of a message caused the protection state change.
+    pub(crate) async fn set_protection(
+        self,
+        context: &Context,
+        protect: ProtectionStatus,
+        timestamp_sent: i64,
+        contact_id: Option<ContactId>,
+    ) -> Result<()> {
+        let sort_to_bottom = true;
+        let ts = self
+            .calc_sort_timestamp(context, timestamp_sent, sort_to_bottom, false)
+            .await?;
+        self.set_protection_for_timestamp_sort(context, protect, ts, contact_id)
+            .await
+    }
+
     /// Sets the 1:1 chat with the given address to ProtectionStatus::Protected,
     /// and posts a `SystemMessage::ChatProtectionEnabled` into it.
     ///
@@ -587,6 +625,7 @@ impl ChatId {
     pub(crate) async fn set_protection_for_contact(
         context: &Context,
         contact_id: ContactId,
+        timestamp: i64,
     ) -> Result<()> {
         let chat_id = ChatId::create_for_contact_with_blocked(context, contact_id, Blocked::Yes)
             .await
@@ -595,7 +634,7 @@ impl ChatId {
             .set_protection(
                 context,
                 ProtectionStatus::Protected,
-                smeared_time(context),
+                timestamp,
                 Some(contact_id),
             )
             .await?;
@@ -744,7 +783,9 @@ impl ChatId {
 
         context.emit_msgs_changed_without_ids();
 
-        context.set_config(Config::LastHousekeeping, None).await?;
+        context
+            .set_config_internal(Config::LastHousekeeping, None)
+            .await?;
         context.scheduler.interrupt_inbox().await;
 
         if chat.is_self_talk() {
@@ -1132,47 +1173,46 @@ impl ChatId {
         Ok(self.get_param(context).await?.exists(Param::Devicetalk))
     }
 
-    async fn parent_query<T, F>(self, context: &Context, fields: &str, f: F) -> Result<Option<T>>
+    async fn parent_query<T, F>(
+        self,
+        context: &Context,
+        fields: &str,
+        state_out_min: MessageState,
+        f: F,
+    ) -> Result<Option<T>>
     where
         F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
         T: Send + 'static,
     {
         let sql = &context.sql;
-        // Do not reply to not fully downloaded messages. Such a message could be a group chat
-        // message that we assigned to 1:1 chat.
         let query = format!(
             "SELECT {fields} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden AND download_state={} \
+             FROM msgs \
+             WHERE chat_id=? \
+             AND ((state BETWEEN {} AND {}) OR (state >= {})) \
+             AND NOT hidden \
+             AND download_state={} \
              ORDER BY timestamp DESC, id DESC \
              LIMIT 1;",
-             DownloadState::Done as u32,
+            MessageState::InFresh as u32,
+            MessageState::InSeen as u32,
+            state_out_min as u32,
+            // Do not reply to not fully downloaded messages. Such a message could be a group chat
+            // message that we assigned to 1:1 chat.
+            DownloadState::Done as u32,
         );
-        let row = sql
-            .query_row_optional(
-                &query,
-                (
-                    self,
-                    MessageState::OutPreparing,
-                    MessageState::OutDraft,
-                    // We don't filter `OutPending` and `OutFailed` messages because the new message
-                    // for which `parent_query()` is done may assume that it will be received in a
-                    // context affected by those messages, e.g. they could add new members to a
-                    // group and the new message will contain them in "To:". Anyway recipients must
-                    // be prepared to orphaned references.
-                ),
-                f,
-            )
-            .await?;
-        Ok(row)
+        sql.query_row_optional(&query, (self,), f).await
     }
 
     async fn get_parent_mime_headers(
         self,
         context: &Context,
+        state_out_min: MessageState,
     ) -> Result<Option<(String, String, String)>> {
         self.parent_query(
             context,
             "rfc724_mid, mime_in_reply_to, mime_references",
+            state_out_min,
             |row: &rusqlite::Row| {
                 let rfc724_mid: String = row.get(0)?;
                 let mime_in_reply_to: String = row.get(1)?;
@@ -1299,6 +1339,64 @@ impl ChatId {
             .unwrap_or_default();
         Ok(protection_status)
     }
+
+    /// Returns the sort timestamp for a new message in the chat.
+    ///
+    /// `message_timestamp` should be either the message "sent" timestamp or a timestamp of the
+    /// corresponding event in case of a system message (usually the current system time).
+    /// `always_sort_to_bottom` makes this ajust the returned timestamp up so that the message goes
+    /// to the chat bottom.
+    /// `incoming` -- whether the message is incoming.
+    pub(crate) async fn calc_sort_timestamp(
+        self,
+        context: &Context,
+        message_timestamp: i64,
+        always_sort_to_bottom: bool,
+        incoming: bool,
+    ) -> Result<i64> {
+        let mut sort_timestamp = cmp::min(message_timestamp, smeared_time(context));
+
+        let last_msg_time: Option<i64> = if always_sort_to_bottom {
+            // get newest message for this chat
+
+            // Let hidden messages also be ordered with protection messages because hidden messages
+            // also can be or not be verified, so let's preserve this information -- even it's not
+            // used currently, it can be useful in the future versions.
+            context
+                .sql
+                .query_get_value(
+                    "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state!=?",
+                    (self, MessageState::OutDraft),
+                )
+                .await?
+        } else if incoming {
+            // get newest non fresh message for this chat.
+
+            // If a user hasn't been online for some time, the Inbox is fetched first and then the
+            // Sentbox. In order for Inbox and Sent messages to be allowed to mingle, outgoing
+            // messages are purely sorted by their sent timestamp. NB: The Inbox must be fetched
+            // first otherwise Inbox messages would be always below old Sentbox messages. We could
+            // take in the query below only incoming messages, but then new incoming messages would
+            // mingle with just sent outgoing ones and apear somewhere in the middle of the chat.
+            context
+                .sql
+                .query_get_value(
+                    "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND hidden=0 AND state>?",
+                    (self, MessageState::InFresh),
+                )
+                .await?
+        } else {
+            None
+        };
+
+        if let Some(last_msg_time) = last_msg_time {
+            if last_msg_time > sort_timestamp {
+                sort_timestamp = last_msg_time;
+            }
+        }
+
+        Ok(sort_timestamp)
+    }
 }
 
 impl std::fmt::Display for ChatId {
@@ -1749,7 +1847,15 @@ impl Chat {
         // we do not set In-Reply-To/References in this case.
         if !self.is_self_talk() {
             if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                self.id.get_parent_mime_headers(context).await?
+                // We don't filter `OutPending` and `OutFailed` messages because the new message for
+                // which `parent_query()` is done may assume that it will be received in a context
+                // affected by those messages, e.g. they could add new members to a group and the
+                // new message will contain them in "To:". Anyway recipients must be prepared to
+                // orphaned references.
+                self
+                    .id
+                    .get_parent_mime_headers(context, MessageState::OutPending)
+                    .await?
             {
                 // "In-Reply-To:" is not changed if it is set manually.
                 // This does not affect "References:" header, it will contain "default parent" (the
@@ -1967,10 +2073,26 @@ impl Chat {
                 Ok(r)
             }
             Chattype::Broadcast | Chattype::Group | Chattype::Mailinglist => {
-                if self.grpid.is_empty() {
-                    return Ok(None);
+                if !self.grpid.is_empty() {
+                    return Ok(Some(SyncId::Grpid(self.grpid.clone())));
                 }
-                Ok(Some(SyncId::Grpid(self.grpid.clone())))
+
+                let Some((parent_rfc724_mid, parent_in_reply_to, _)) = self
+                    .id
+                    .get_parent_mime_headers(context, MessageState::OutDelivered)
+                    .await?
+                else {
+                    warn!(
+                        context,
+                        "Chat::get_sync_id({}): No good message identifying the chat found.",
+                        self.id
+                    );
+                    return Ok(None);
+                };
+                Ok(Some(SyncId::Msgids(vec![
+                    parent_in_reply_to,
+                    parent_rfc724_mid,
+                ])))
             }
         }
     }
@@ -1984,7 +2106,7 @@ impl Chat {
     }
 }
 
-async fn sync(context: &Context, id: SyncId, action: SyncAction) -> Result<()> {
+pub(crate) async fn sync(context: &Context, id: SyncId, action: SyncAction) -> Result<()> {
     context
         .add_sync_item(SyncData::AlterChat { id, action })
         .await?;
@@ -2537,22 +2659,30 @@ pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) ->
         return send_msg_inner(context, chat_id, msg).await;
     }
 
+    if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing {
+        msg.param.remove(Param::GuaranteeE2ee);
+        msg.param.remove(Param::ForcePlaintext);
+        msg.update_param(context).await?;
+    }
     send_msg_inner(context, chat_id, msg).await
 }
 
 /// Tries to send a message synchronously.
 ///
-/// Creates a new message in `smtp` table, then drectly opens an SMTP connection and sends the
-/// message. If this fails, the message remains in the database to be sent later.
+/// Creates jobs in the `smtp` table, then drectly opens an SMTP connection and sends the
+/// message. If this fails, the jobs remain in the database for later sending.
 pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
-    if let Some(rowid) = prepare_send_msg(context, chat_id, msg).await? {
-        let mut smtp = crate::smtp::Smtp::new();
+    let rowids = prepare_send_msg(context, chat_id, msg).await?;
+    if rowids.is_empty() {
+        return Ok(msg.id);
+    }
+    let mut smtp = crate::smtp::Smtp::new();
+    for rowid in rowids {
         send_msg_to_smtp(context, &mut smtp, rowid)
             .await
             .context("failed to send message, queued for later sending")?;
-
-        context.emit_msgs_changed(msg.chat_id, msg.id);
     }
+    context.emit_msgs_changed(msg.chat_id, msg.id);
     Ok(msg.id)
 }
 
@@ -2562,7 +2692,7 @@ async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -
         msg.text = strip_rtlo_characters(&msg.text);
     }
 
-    if prepare_send_msg(context, chat_id, msg).await?.is_some() {
+    if !prepare_send_msg(context, chat_id, msg).await?.is_empty() {
         context.emit_msgs_changed(msg.chat_id, msg.id);
 
         if msg.param.exists(Param::SetLatitude) {
@@ -2575,12 +2705,12 @@ async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -
     Ok(msg.id)
 }
 
-/// Returns rowid from `smtp` table.
+/// Returns row ids of the `smtp` table.
 async fn prepare_send_msg(
     context: &Context,
     chat_id: ChatId,
     msg: &mut Message,
-) -> Result<Option<i64>> {
+) -> Result<Vec<i64>> {
     // prepare_msg() leaves the message state to OutPreparing, we
     // only have to change the state to OutPending in this case.
     // Otherwise we still have to prepare the message, which will set
@@ -2596,20 +2726,16 @@ async fn prepare_send_msg(
         );
         message::update_msg_state(context, msg.id, MessageState::OutPending).await?;
     }
-    let row_id = create_send_msg_job(context, msg).await?;
-    Ok(row_id)
+    create_send_msg_jobs(context, msg).await
 }
 
-/// Constructs a job for sending a message and inserts into `smtp` table.
+/// Constructs jobs for sending a message and inserts them into the `smtp` table.
 ///
-/// Returns rowid if job was created or `None` if SMTP job is not needed, e.g. when sending to a
+/// Returns row ids if jobs were created or an empty `Vec` otherwise, e.g. when sending to a
 /// group with only self and no BCC-to-self configured.
 ///
-/// The caller has to interrupt SMTP loop or otherwise process a new row.
-pub(crate) async fn create_send_msg_job(
-    context: &Context,
-    msg: &mut Message,
-) -> Result<Option<i64>> {
+/// The caller has to interrupt SMTP loop or otherwise process new rows.
+pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result<Vec<i64>> {
     let needs_encryption = msg.param.get_bool(Param::GuaranteeE2ee).unwrap_or_default();
 
     let attach_selfavatar = match shall_attach_selfavatar(context, msg.chat_id).await {
@@ -2627,10 +2753,18 @@ pub(crate) async fn create_send_msg_job(
     let from = context.get_primary_self_addr().await?;
     let lowercase_from = from.to_lowercase();
 
-    // Send BCC to self if it is enabled and we are not going to
-    // delete it immediately.
+    // Send BCC to self if it is enabled.
+    //
+    // Previous versions of Delta Chat did not send BCC self
+    // if DeleteServerAfter was set to immediately delete messages
+    // from the server. This is not the case anymore
+    // because BCC-self messages are also used to detect
+    // that message was sent if SMTP server is slow to respond
+    // and connection is frequently lost
+    // before receiving status line.
+    //
+    // `from` must be the last addr, see `receive_imf_inner()` why.
     if context.get_config_bool(Config::BccSelf).await?
-        && context.get_config_delete_server_after().await? != Some(0)
         && !recipients
             .iter()
             .any(|x| x.to_lowercase() == lowercase_from)
@@ -2646,7 +2780,7 @@ pub(crate) async fn create_send_msg_job(
         );
         msg.id.set_delivered(context).await?;
         msg.state = MessageState::OutDelivered;
-        return Ok(None);
+        return Ok(Vec::new());
     }
 
     let rendered_msg = match mimefactory.render(context).await {
@@ -2672,12 +2806,14 @@ pub(crate) async fn create_send_msg_job(
         );
     }
 
+    let now = time();
+
     if rendered_msg.is_gossiped {
-        msg.chat_id.set_gossiped_timestamp(context, time()).await?;
+        msg.chat_id.set_gossiped_timestamp(context, now).await?;
     }
 
     if let Some(last_added_location_id) = rendered_msg.last_added_location_id {
-        if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, time()).await {
+        if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, now).await {
             error!(context, "Failed to set kml sent_timestamp: {err:#}.");
         }
         if !msg.hidden {
@@ -2696,7 +2832,7 @@ pub(crate) async fn create_send_msg_job(
     }
 
     if attach_selfavatar {
-        if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
+        if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, now).await {
             error!(context, "Failed to set selfavatar timestamp: {err:#}.");
         }
     }
@@ -2706,27 +2842,32 @@ pub(crate) async fn create_send_msg_job(
         msg.update_param(context).await?;
     }
 
-    ensure!(!recipients.is_empty(), "no recipients for smtp job set");
-
-    let recipients = recipients.join(" ");
-
     msg.subject = rendered_msg.subject.clone();
     msg.update_subject(context).await?;
-
-    let row_id = context
-        .sql
-        .insert(
-            "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id)
-             VALUES           (?1,         ?2,         ?3,   ?4)",
-            (
-                &rendered_msg.rfc724_mid,
-                recipients,
-                &rendered_msg.message,
-                msg.id,
-            ),
-        )
-        .await?;
-    Ok(Some(row_id))
+    let chunk_size = context
+        .get_configured_provider()
+        .await?
+        .and_then(|provider| provider.opt.max_smtp_rcpt_to)
+        .map_or(constants::DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
+    let trans_fn = |t: &mut rusqlite::Transaction| {
+        let mut row_ids = Vec::<i64>::new();
+        for recipients_chunk in recipients.chunks(chunk_size) {
+            let recipients_chunk = recipients_chunk.join(" ");
+            let row_id = t.execute(
+                "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
+                VALUES            (?1,         ?2,         ?3,   ?4)",
+                (
+                    &rendered_msg.rfc724_mid,
+                    recipients_chunk,
+                    &rendered_msg.message,
+                    msg.id,
+                ),
+            )?;
+            row_ids.push(row_id.try_into()?);
+        }
+        Ok(row_ids)
+    };
+    context.sql.transaction(trans_fn).await
 }
 
 /// Sends a text message to the given chat.
@@ -3215,7 +3356,7 @@ pub async fn create_group_chat(
 
     if protect == ProtectionStatus::Protected {
         chat_id
-            .set_protection(context, protect, timestamp, None)
+            .set_protection_for_timestamp_sort(context, protect, timestamp, None)
             .await?;
     }
 
@@ -3446,7 +3587,7 @@ pub(crate) async fn add_contact_to_chat_ex(
         if chat.is_protected() && !contact.is_verified(context).await? {
             error!(
                 context,
-                "Only bidirectional verified contacts can be added to protected chats."
+                "Cannot add non-bidirectionally verified contact {contact_id} to protected chat {chat_id}."
             );
             return Ok(false);
         }
@@ -3882,7 +4023,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
                 .prepare_msg_raw(context, &mut msg, None, curr_timestamp)
                 .await?;
             curr_timestamp += 1;
-            if create_send_msg_job(context, &mut msg).await?.is_some() {
+            if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
                 context.scheduler.interrupt_smtp().await;
             }
         }
@@ -3939,7 +4080,8 @@ pub async fn resend_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             chat_id: msg.chat_id,
             msg_id: msg.id,
         });
-        if create_send_msg_job(context, &mut msg).await?.is_some() {
+        msg.timestamp_sort = create_smeared_timestamp(context);
+        if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
             context.scheduler.interrupt_smtp().await;
         }
     }
@@ -4128,7 +4270,9 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
             (),
         )
         .await?;
-    context.set_config(Config::QuotaExceeding, None).await?;
+    context
+        .set_config_internal(Config::QuotaExceeding, None)
+        .await?;
     Ok(())
 }
 
@@ -4250,8 +4394,8 @@ async fn set_contacts_by_addrs(context: &Context, id: ChatId, addrs: &[String])
 pub(crate) enum SyncId {
     ContactAddr(String),
     Grpid(String),
-    // NOTE: Ad-hoc groups lack an identifier that can be used across devices so
-    // block/mute/etc. actions on them are not synchronized to other devices.
+    /// "Message-ID"-s, from oldest to latest. Used for ad-hoc groups.
+    Msgids(Vec<String>),
 }
 
 /// An action synchronised to other devices.
@@ -4274,12 +4418,13 @@ impl Context {
     pub(crate) async fn sync_alter_chat(&self, id: &SyncId, action: &SyncAction) -> Result<()> {
         let chat_id = match id {
             SyncId::ContactAddr(addr) => {
-                let Some(contact_id) =
-                    Contact::lookup_id_by_addr_ex(self, addr, Origin::Unknown, None).await?
-                else {
-                    warn!(self, "sync_alter_chat: No contact for addr '{addr}'.");
+                if let SyncAction::Rename(to) = action {
+                    Contact::create_ex(self, Nosync, to, addr).await?;
                     return Ok(());
-                };
+                }
+                let contact_id = Contact::lookup_id_by_addr_ex(self, addr, Origin::Unknown, None)
+                    .await?
+                    .with_context(|| format!("No contact for addr '{addr}'"))?;
                 match action {
                     SyncAction::Block => {
                         return contact::set_blocked(self, Nosync, contact_id, true).await
@@ -4289,22 +4434,26 @@ impl Context {
                     }
                     _ => (),
                 }
-                let Some(chat_id) = ChatId::lookup_by_contact(self, contact_id).await? else {
-                    warn!(self, "sync_alter_chat: No chat for addr '{addr}'.");
-                    return Ok(());
-                };
-                chat_id
+                ChatId::lookup_by_contact(self, contact_id)
+                    .await?
+                    .with_context(|| format!("No chat for addr '{addr}'"))?
             }
             SyncId::Grpid(grpid) => {
                 if let SyncAction::CreateBroadcast(name) = action {
                     create_broadcast_list_ex(self, Nosync, grpid.clone(), name.clone()).await?;
                     return Ok(());
                 }
-                let Some((chat_id, ..)) = get_chat_id_by_grpid(self, grpid).await? else {
-                    warn!(self, "sync_alter_chat: No chat for grpid '{grpid}'.");
-                    return Ok(());
-                };
-                chat_id
+                get_chat_id_by_grpid(self, grpid)
+                    .await?
+                    .with_context(|| format!("No chat for grpid '{grpid}'"))?
+                    .0
+            }
+            SyncId::Msgids(msgids) => {
+                let msg = message::get_latest_by_rfc724_mids(self, msgids)
+                    .await?
+                    .with_context(|| format!("No message found for Message-IDs {msgids:?}"))?;
+                ChatId::lookup_by_message(&msg)
+                    .with_context(|| format!("No chat found for Message-IDs {msgids:?}"))?
             }
         };
         match action {
@@ -5783,7 +5932,7 @@ mod tests {
         // Alice has an SMTP-server replacing the `Message-ID:`-header (as done eg. by outlook.com).
         let sent_msg = alice.pop_sent_msg().await;
         let msg = sent_msg.payload();
-        assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 1);
+        assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 2);
         assert_eq!(msg.match_indices("References: <Gr.").count(), 1);
         let msg = msg.replace("Message-ID: <Gr.", "Message-ID: <XXX");
         assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 0);
@@ -6360,6 +6509,7 @@ mod tests {
         // Bob receives all messages
         let bob = TestContext::new_bob().await;
         let msg = bob.recv_msg(&sent1).await;
+        let sent1_ts_sent = msg.timestamp_sent;
         assert_eq!(msg.get_text(), "alice->bob");
         assert_eq!(get_chat_contacts(&bob, msg.chat_id).await?.len(), 2);
         assert_eq!(get_chat_msgs(&bob, msg.chat_id).await?.len(), 1);
@@ -6382,6 +6532,7 @@ mod tests {
         assert_eq!(get_chat_msgs(&claire, msg.chat_id).await?.len(), 2);
         let msg_from = Contact::get_by_id(&claire, msg.get_from_id()).await?;
         assert_eq!(msg_from.get_addr(), "alice@example.org");
+        assert!(sent1_ts_sent < msg.timestamp_sent);
 
         Ok(())
     }
@@ -6949,6 +7100,51 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync_adhoc_grp() -> Result<()> {
+        let alice0 = &TestContext::new_alice().await;
+        let alice1 = &TestContext::new_alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let mut chat_ids = Vec::new();
+        for a in [alice0, alice1] {
+            let msg = receive_imf(
+                a,
+                b"Subject: =?utf-8?q?Message_from_alice=40example=2Eorg?=\r\n\
+                    From: alice@example.org\r\n\
+                    To: <bob@example.net>, <fiona@example.org> \r\n\
+                    Date: Mon, 2 Dec 2023 16:59:39 +0000\r\n\
+                    Message-ID: <Mr.alices_original_mail@example.org>\r\n\
+                    Chat-Version: 1.0\r\n\
+                    \r\n\
+                    hi\r\n",
+                false,
+            )
+            .await?
+            .unwrap();
+            chat_ids.push(msg.chat_id);
+        }
+        let chat1 = Chat::load_from_db(alice1, chat_ids[1]).await?;
+        assert_eq!(chat1.typ, Chattype::Group);
+        assert!(chat1.grpid.is_empty());
+
+        // Test synchronisation on chat blocking because it causes chat deletion currently and thus
+        // requires generating a sync message in advance.
+        chat_ids[0].block(alice0).await?;
+        sync(alice0, alice1).await;
+        assert!(Chat::load_from_db(alice1, chat_ids[1]).await.is_err());
+        assert!(
+            !alice1
+                .sql
+                .exists("SELECT COUNT(*) FROM chats WHERE id=?", (chat_ids[1],))
+                .await?
+        );
+
+        Ok(())
+    }
+
     /// Tests syncing of chat visibility on a self-chat. This way we test:
     /// - Self-chat synchronisation.
     /// - That sync messages don't unarchive the self-chat.

src/config.rs

@@ -10,7 +10,7 @@ use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::constants::DC_VERSION_STR;
+use crate::constants::{self, DC_VERSION_STR};
 use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::events::EventType;
@@ -18,7 +18,7 @@ use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 use crate::sync::{self, Sync::*, SyncData};
-use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
+use crate::tools::{get_abs_path, improve_single_line_input};
 
 /// The available configuration keys.
 #[derive(
@@ -291,6 +291,9 @@ pub enum Config {
     /// Timestamp of the last time housekeeping was run
     LastHousekeeping,
 
+    /// Timestamp of the last `CantDecryptOutgoingMsgs` notification.
+    LastCantDecryptOutgoingMsgs,
+
     /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
@@ -343,6 +346,14 @@ pub enum Config {
     /// until `chat_id.accept()` is called.
     #[strum(props(default = "0"))]
     VerifiedOneOnOneChats,
+
+    /// Row ID of the key in the `keypairs` table
+    /// used for signatures, encryption to self and included in `Autocrypt` header.
+    KeyId,
+
+    /// This key is sent to the self_reporting bot so that the bot can recognize the user
+    /// without storing the email address
+    SelfReportingId,
 }
 
 impl Config {
@@ -356,11 +367,23 @@ impl Config {
     /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
     /// mustn't be controlled by other devices.
     pub(crate) fn is_synced(&self) -> bool {
+        // We don't restart IO from the synchronisation code, so this is to be on the safe side.
+        if self.needs_io_restart() {
+            return false;
+        }
         matches!(
             self,
             Self::Displayname | Self::MdnsEnabled | Self::ShowEmails
         )
     }
+
+    /// Whether the config option needs an IO scheduler restart to take effect.
+    pub(crate) fn needs_io_restart(&self) -> bool {
+        matches!(
+            self,
+            Config::MvboxMove | Config::OnlyFetchMvbox | Config::SentboxWatch
+        )
+    }
 }
 
 impl Context {
@@ -480,10 +503,50 @@ impl Context {
         }
     }
 
-    /// Set the given config key.
-    /// If `None` is passed as a value the value is cleared and set to the default if there is one.
+    fn check_config(key: Config, value: Option<&str>) -> Result<()> {
+        match key {
+            Config::Socks5Enabled
+            | Config::BccSelf
+            | Config::E2eeEnabled
+            | Config::MdnsEnabled
+            | Config::SentboxWatch
+            | Config::MvboxMove
+            | Config::OnlyFetchMvbox
+            | Config::FetchExistingMsgs
+            | Config::DeleteToTrash
+            | Config::SaveMimeHeaders
+            | Config::Configured
+            | Config::Bot
+            | Config::NotifyAboutWrongPw
+            | Config::SyncMsgs
+            | Config::SignUnencrypted
+            | Config::DisableIdle => {
+                ensure!(
+                    matches!(value, None | Some("0") | Some("1")),
+                    "Boolean value must be either 0 or 1"
+                );
+            }
+            _ => (),
+        }
+        Ok(())
+    }
+
+    /// Set the given config key and make it effective.
+    /// This may restart the IO scheduler. If `None` is passed as a value the value is cleared and
+    /// set to the default if there is one.
     pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
-        self.set_config_ex(key.is_synced().into(), key, value).await
+        Self::check_config(key, value)?;
+
+        let _pause = match key.needs_io_restart() {
+            true => self.scheduler.pause(self.clone()).await?,
+            _ => Default::default(),
+        };
+        self.set_config_internal(key, value).await?;
+        Ok(())
+    }
+
+    pub(crate) async fn set_config_internal(&self, key: Config, value: Option<&str>) -> Result<()> {
+        self.set_config_ex(Sync, key, value).await
     }
 
     pub(crate) async fn set_config_ex(
@@ -492,7 +555,10 @@ impl Context {
         key: Config,
         mut value: Option<&str>,
     ) -> Result<()> {
+        Self::check_config(key, value)?;
+        let sync = sync == Sync && key.is_synced();
         let better_value;
+
         match key {
             Config::Selfavatar => {
                 self.sql
@@ -525,39 +591,25 @@ impl Context {
                 }
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
-            Config::Socks5Enabled
-            | Config::BccSelf
-            | Config::E2eeEnabled
-            | Config::MdnsEnabled
-            | Config::SentboxWatch
-            | Config::MvboxMove
-            | Config::OnlyFetchMvbox
-            | Config::FetchExistingMsgs
-            | Config::DeleteToTrash
-            | Config::SaveMimeHeaders
-            | Config::Configured
-            | Config::Bot
-            | Config::NotifyAboutWrongPw
-            | Config::SyncMsgs
-            | Config::SignUnencrypted
-            | Config::DisableIdle => {
-                ensure!(
-                    matches!(value, None | Some("0") | Some("1")),
-                    "Boolean value must be either 0 or 1"
-                );
-                self.sql.set_raw_config(key.as_ref(), value).await?;
-            }
             Config::Addr => {
                 self.sql
                     .set_raw_config(key.as_ref(), value.map(|s| s.to_lowercase()).as_deref())
                     .await?;
             }
+            Config::MvboxMove => {
+                self.sql.set_raw_config(key.as_ref(), value).await?;
+                self.sql
+                    .set_raw_config(constants::DC_FOLDERS_CONFIGURED_KEY, None)
+                    .await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
         }
-
-        if sync != Sync {
+        if key.is_synced() {
+            self.emit_event(EventType::ConfigSynced { key });
+        }
+        if !sync {
             return Ok(());
         }
         let Some(val) = value else {
@@ -584,8 +636,7 @@ impl Context {
 
     /// Set the given config to a boolean value.
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { Some("0") })
-            .await?;
+        self.set_config(key, from_bool(value)).await?;
         Ok(())
     }
 
@@ -605,6 +656,11 @@ impl Context {
     }
 }
 
+/// Returns a value for use in `Context::set_config_*()` for the given `bool`.
+pub(crate) fn from_bool(val: bool) -> Option<&'static str> {
+    Some(if val { "1" } else { "0" })
+}
+
 // Separate impl block for self address handling
 impl Context {
     /// Determine whether the specified addr maps to the/a self addr.
@@ -627,32 +683,19 @@ impl Context {
     ///
     /// This should only be used by test code and during configure.
     pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
-        let old_addr = self.get_config(Config::ConfiguredAddr).await?;
-
         // add old primary address (if exists) to secondary addresses
         let mut secondary_addrs = self.get_all_self_addrs().await?;
         // never store a primary address also as a secondary
         secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
-        self.set_config(
+        self.set_config_internal(
             Config::SecondaryAddrs,
             Some(secondary_addrs.join(" ").as_str()),
         )
         .await?;
 
-        self.set_config(Config::ConfiguredAddr, Some(primary_new))
+        self.set_config_internal(Config::ConfiguredAddr, Some(primary_new))
             .await?;
 
-        if let Some(old_addr) = old_addr {
-            let old_addr = EmailAddress::new(&old_addr)?;
-            let old_keypair = crate::key::load_keypair(self, &old_addr).await?;
-
-            if let Some(mut old_keypair) = old_keypair {
-                old_keypair.addr = EmailAddress::new(primary_new)?;
-                crate::key::store_self_keypair(self, &old_keypair, crate::key::KeyPairUse::Default)
-                    .await?;
-            }
-        }
-
         Ok(())
     }
 
@@ -875,10 +918,10 @@ mod tests {
         // Reset to default. Test that it's not synced because defaults may differ across client
         // versions.
         alice0.set_config(Config::MdnsEnabled, None).await?;
-        assert!(alice0.get_config_bool(Config::MdnsEnabled).await?);
+        assert_eq!(alice0.get_config_bool(Config::MdnsEnabled).await?, true);
         alice0.set_config_bool(Config::MdnsEnabled, false).await?;
         sync(&alice0, &alice1).await;
-        assert!(!alice1.get_config_bool(Config::MdnsEnabled).await?);
+        assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
 
         let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
         alice0
@@ -909,4 +952,59 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_event_config_synced() -> Result<()> {
+        let alice0 = TestContext::new_alice().await;
+        let alice1 = TestContext::new_alice().await;
+        for a in [&alice0, &alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        alice0
+            .set_config(Config::Displayname, Some("Alice Sync"))
+            .await?;
+        alice0
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::Displayname
+                    }
+                )
+            })
+            .await;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config(Config::Displayname).await?,
+            Some("Alice Sync".to_string())
+        );
+        alice1
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::Displayname
+                    }
+                )
+            })
+            .await;
+
+        alice0.set_config(Config::Displayname, None).await?;
+        alice0
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::Displayname
+                    }
+                )
+            })
+            .await;
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -22,7 +22,7 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use server_params::{expand_param_vector, ServerParams};
 use tokio::task;
 
-use crate::config::Config;
+use crate::config::{self, Config};
 use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::imap::Imap;
@@ -112,12 +112,13 @@ impl Context {
         let mut param = LoginParam::load_candidate_params(self).await?;
         let old_addr = self.get_config(Config::ConfiguredAddr).await?;
         let success = configure(self, &mut param).await;
-        self.set_config(Config::NotifyAboutWrongPw, None).await?;
+        self.set_config_internal(Config::NotifyAboutWrongPw, None)
+            .await?;
 
         on_configure_completed(self, param, old_addr).await?;
 
         success?;
-        self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+        self.set_config_internal(Config::NotifyAboutWrongPw, Some("1"))
             .await?;
         Ok(())
     }
@@ -473,7 +474,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     // the trailing underscore is correct
     param.save_as_configured_params(ctx).await?;
-    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
+    ctx.set_config_internal(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
     progress!(ctx, 920);
@@ -481,7 +482,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
-    ctx.set_config_bool(Config::FetchedExistingMsgs, false)
+    ctx.set_config_internal(Config::FetchedExistingMsgs, config::from_bool(false))
         .await?;
     ctx.scheduler.interrupt_inbox().await;
 

src/configure/auto_mozilla.rs

@@ -1,6 +1,7 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: <https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
+//! RFC draft: <https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html>
+//! Archived original documentation: <https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
 use std::io::BufRead;
 use std::str::FromStr;
 

src/configure/server_params.rs

@@ -137,20 +137,11 @@ impl ServerParams {
     }
 
     fn expand_strict_tls(self) -> Vec<ServerParams> {
-        if self.strict_tls.is_none() {
-            vec![
-                Self {
-                    strict_tls: Some(true), // Strict.
-                    ..self.clone()
-                },
-                Self {
-                    strict_tls: None, // Automatic.
-                    ..self
-                },
-            ]
-        } else {
-            vec![self]
-        }
+        vec![Self {
+            // Strict if not set by the user or provider database.
+            strict_tls: Some(self.strict_tls.unwrap_or(true)),
+            ..self
+        }]
     }
 }
 
@@ -162,31 +153,10 @@ pub(crate) fn expand_param_vector(
     domain: &str,
 ) -> Vec<ServerParams> {
     v.into_iter()
-        .map(|params| {
-            if params.socket == Socket::Plain {
-                ServerParams {
-                    // Avoid expanding plaintext configuration into configuration with and without
-                    // `strict_tls` if `strict_tls` is set to `None` as `strict_tls` is not used for
-                    // plaintext connections. Always setting it to "enabled", just in case.
-                    strict_tls: Some(true),
-                    ..params
-                }
-            } else {
-                params
-            }
-        })
         // The order of expansion is important.
         //
         // Ports are expanded the last, so they are changed the first.  Username is only changed if
         // default value (address with domain) didn't work for all available hosts and ports.
-        //
-        // Strict TLS must be expanded first, so we try all configurations with strict TLS first
-        // and only then try again without strict TLS. Otherwise we may lock to wrong hostname
-        // without strict TLS when another hostname with strict TLS is available.  For example, if
-        // both smtp.example.net and mail.example.net are running an SMTP server, but both use a
-        // certificate that is only valid for mail.example.net, we want to skip smtp.example.net
-        // and use mail.example.net with strict TLS instead of using smtp.example.net without
-        // strict TLS.
         .flat_map(|params| params.expand_strict_tls().into_iter())
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
@@ -257,22 +227,6 @@ mod tests {
                     username: "foobar".to_string(),
                     strict_tls: Some(true)
                 },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: None,
-                },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Starttls,
-                    username: "foobar".to_string(),
-                    strict_tls: None
-                }
             ],
         );
 
@@ -284,7 +238,7 @@ mod tests {
                 port: 123,
                 socket: Socket::Plain,
                 username: "foobar".to_string(),
-                strict_tls: None,
+                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",

src/constants.rs

@@ -206,9 +206,19 @@ pub(crate) const WORSE_AVATAR_SIZE: u32 = 128;
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;
 pub const WORSE_IMAGE_SIZE: u32 = 640;
 
+// Key for the folder configuration version (see below).
+pub(crate) const DC_FOLDERS_CONFIGURED_KEY: &str = "folders_configured";
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
 
+// If more recipients are needed in SMTP's `RCPT TO:` header, the recipient list is split into
+// chunks. This does not affect MIME's `To:` header. Can be overwritten by setting
+// `max_smtp_rcpt_to` in the provider db.
+pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
+/// How far the last quota check needs to be in the past to be checked by the background function (in seconds).
+pub(crate) const DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT: u64 = 12 * 60 * 60; // 12 hours
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/contact.rs

@@ -26,13 +26,14 @@ use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY}
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{load_self_public_key, DcKey};
+use crate::log::LogExt;
 use crate::login_param::LoginParam;
 use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::sql::{self, params_iter};
-use crate::sync::{self, Sync::*, SyncData};
+use crate::sync::{self, Sync::*};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
     EmailAddress,
@@ -476,6 +477,15 @@ impl Contact {
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
     pub async fn create(context: &Context, name: &str, addr: &str) -> Result<ContactId> {
+        Self::create_ex(context, Sync, name, addr).await
+    }
+
+    pub(crate) async fn create_ex(
+        context: &Context,
+        sync: sync::Sync,
+        name: &str,
+        addr: &str,
+    ) -> Result<ContactId> {
         let name = improve_single_line_input(name);
 
         let (name, addr) = sanitize_name_and_addr(&name, addr);
@@ -496,6 +506,16 @@ impl Contact {
             set_blocked(context, Nosync, contact_id, false).await?;
         }
 
+        if sync.into() {
+            chat::sync(
+                context,
+                chat::SyncId::ContactAddr(addr.to_string()),
+                chat::SyncAction::Rename(name.to_string()),
+            )
+            .await
+            .log_err(context)
+            .ok();
+        }
         Ok(contact_id)
     }
 
@@ -772,7 +792,7 @@ impl Contact {
 
                 sth_modified = Modifier::Created;
                 row_id = u32::try_from(transaction.last_insert_rowid())?;
-                info!(context, "added contact id={} addr={}", row_id, &addr);
+                info!(context, "Added contact id={row_id} addr={addr}.");
             }
             Ok(row_id)
         }).await?;
@@ -1268,13 +1288,30 @@ impl Contact {
             return Ok(true);
         }
 
-        if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
-            if peerstate.is_using_verified_key() {
-                return Ok(true);
-            }
+        let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? else {
+            return Ok(false);
+        };
+
+        let forward_verified = peerstate.is_using_verified_key();
+        let backward_verified = peerstate.is_backward_verified(context).await?;
+        Ok(forward_verified && backward_verified)
+    }
+
+    /// Returns true if we have a verified key for the contact
+    /// and it is the same as Autocrypt key.
+    /// This is enough to send messages to the contact in verified chat
+    /// and verify received messages, but not enough to display green checkmark
+    /// or add the contact to verified groups.
+    pub async fn is_forward_verified(&self, context: &Context) -> Result<bool> {
+        if self.id == ContactId::SELF {
+            return Ok(true);
         }
 
-        Ok(false)
+        let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? else {
+            return Ok(false);
+        };
+
+        Ok(peerstate.is_using_verified_key())
     }
 
     /// Returns the `ContactId` that verified the contact.
@@ -1480,13 +1517,14 @@ WHERE type=? AND id IN (
                 true => chat::SyncAction::Block,
                 false => chat::SyncAction::Unblock,
             };
-            context
-                .add_sync_item(SyncData::AlterChat {
-                    id: chat::SyncId::ContactAddr(contact.addr.clone()),
-                    action,
-                })
-                .await?;
-            context.send_sync_msg().await?;
+            chat::sync(
+                context,
+                chat::SyncId::ContactAddr(contact.addr.clone()),
+                action,
+            )
+            .await
+            .log_err(context)
+            .ok();
         }
     }
 
@@ -1512,7 +1550,7 @@ pub(crate) async fn set_profile_image(
             if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context
-                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .set_config_internal(Config::Selfavatar, Some(profile_image))
                         .await?;
                 } else {
                     info!(context, "Do not use unencrypted selfavatar.");
@@ -1525,7 +1563,9 @@ pub(crate) async fn set_profile_image(
         AvatarAction::Delete => {
             if contact_id == ContactId::SELF {
                 if was_encrypted {
-                    context.set_config(Config::Selfavatar, None).await?;
+                    context
+                        .set_config_internal(Config::Selfavatar, None)
+                        .await?;
                 } else {
                     info!(context, "Do not use unencrypted selfavatar deletion.");
                 }
@@ -1557,7 +1597,7 @@ pub(crate) async fn set_status(
     if contact_id == ContactId::SELF {
         if encrypted && has_chat_version {
             context
-                .set_config(Config::Selfstatus, Some(&status))
+                .set_config_internal(Config::Selfstatus, Some(&status))
                 .await?;
         }
     } else {
@@ -1692,6 +1732,12 @@ impl RecentlySeenLoop {
     async fn run(context: Context, interrupt: Receiver<RecentlySeenInterrupt>) {
         type MyHeapElem = (Reverse<i64>, ContactId);
 
+        let now = SystemTime::now();
+        let now_ts = now
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs() as i64;
+
         // Priority contains all recently seen sorted by the timestamp
         // when they become not recently seen.
         //
@@ -1702,7 +1748,7 @@ impl RecentlySeenLoop {
             .query_map(
                 "SELECT id, last_seen FROM contacts
                  WHERE last_seen > ?",
-                (time() - SEEN_RECENTLY_SECONDS,),
+                (now_ts - SEEN_RECENTLY_SECONDS,),
                 |row| {
                     let contact_id: ContactId = row.get("id")?;
                     let last_seen: i64 = row.get("last_seen")?;
@@ -1717,8 +1763,6 @@ impl RecentlySeenLoop {
             .unwrap_or_default();
 
         loop {
-            let now = SystemTime::now();
-
             let (until, contact_id) =
                 if let Some((Reverse(timestamp), contact_id)) = unseen_queue.peek() {
                     (
@@ -1885,12 +1929,12 @@ mod tests {
         // Search by name.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         // Search by address.
         let contacts = Contact::get_all(&context.ctx, 0, Some("user")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         let contacts = Contact::get_all(&context.ctx, 0, Some("alice")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1917,7 +1961,7 @@ mod tests {
         // Search by display name (same as manually set name).
         let contacts = Contact::get_all(&context.ctx, 0, Some("someone")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         Ok(())
     }
@@ -2789,4 +2833,33 @@ Hi."#;
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync_create() -> Result<()> {
+        let alice0 = &TestContext::new_alice().await;
+        let alice1 = &TestContext::new_alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        Contact::create(alice0, "Bob", "bob@example.net").await?;
+        test_utils::sync(alice0, alice1).await;
+        let a1b_contact_id =
+            Contact::lookup_id_by_addr(alice1, "bob@example.net", Origin::ManuallyCreated)
+                .await?
+                .unwrap();
+        let a1b_contact = Contact::get_by_id(alice1, a1b_contact_id).await?;
+        assert_eq!(a1b_contact.name, "Bob");
+
+        Contact::create(alice0, "Bob Renamed", "bob@example.net").await?;
+        test_utils::sync(alice0, alice1).await;
+        let id = Contact::lookup_id_by_addr(alice1, "bob@example.net", Origin::ManuallyCreated)
+            .await?
+            .unwrap();
+        assert_eq!(id, a1b_contact_id);
+        let a1b_contact = Contact::get_by_id(alice1, a1b_contact_id).await?;
+        assert_eq!(a1b_contact.name, "Bob Renamed");
+
+        Ok(())
+    }
 }

src/context.rs

@@ -6,28 +6,34 @@ use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
-use std::time::{Duration, Instant, SystemTime};
+use std::time::Duration;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
+use pgp::SignedPublicKey;
 use ratelimit::Ratelimit;
 use tokio::sync::{Mutex, Notify, RwLock};
 
-use crate::chat::{get_chat_cnt, ChatId};
+use crate::aheader::EncryptPreference;
+use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::DC_VERSION_STR;
+use crate::constants::{
+    self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR,
+};
 use crate::contact::Contact;
 use crate::debug_logging::DebugLogging;
 use crate::events::{Event, EventEmitter, EventType, Events};
-use crate::key::{load_self_public_key, DcKey as _};
+use crate::imap::{FolderMeaning, Imap, ServerMetadata};
+use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _};
 use crate::login_param::LoginParam;
-use crate::message::{self, MessageState, MsgId};
+use crate::message::{self, Message, MessageState, MsgId, Viewtype};
+use crate::peerstate::Peerstate;
 use crate::quota::QuotaInfo;
-use crate::scheduler::SchedulerState;
+use crate::scheduler::{convert_folder_meaning, SchedulerState};
 use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
-use crate::tools::{duration_to_str, time};
+use crate::tools::{self, create_id, duration_to_str, time, time_elapsed};
 
 /// Builder for the [`Context`].
 ///
@@ -224,7 +230,10 @@ pub struct InnerContext {
     /// <https://datatracker.ietf.org/doc/html/rfc2971>
     pub(crate) server_id: RwLock<Option<HashMap<String, String>>>,
 
-    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
+    /// IMAP METADATA.
+    pub(crate) metadata: RwLock<Option<ServerMetadata>>,
+
+    pub(crate) last_full_folder_scan: Mutex<Option<tools::Time>>,
 
     /// ID for this `Context` in the current process.
     ///
@@ -232,7 +241,7 @@ pub struct InnerContext {
     /// be identified by this ID.
     pub(crate) id: u32,
 
-    creation_time: SystemTime,
+    creation_time: tools::Time,
 
     /// The text of the last error logged and emitted as an event.
     /// If the ui wants to display an error after a failure,
@@ -253,7 +262,7 @@ enum RunningState {
     Running { cancel_sender: Sender<()> },
 
     /// Cancel signal has been sent, waiting for ongoing process to be freed.
-    ShallStop { request: Instant },
+    ShallStop { request: tools::Time },
 
     /// There is no ongoing process, a new one can be allocated.
     Stopped,
@@ -384,7 +393,8 @@ impl Context {
             resync_request: AtomicBool::new(false),
             new_msgs_notify,
             server_id: RwLock::new(None),
-            creation_time: std::time::SystemTime::now(),
+            metadata: RwLock::new(None),
+            creation_time: tools::Time::now(),
             last_full_folder_scan: Mutex::new(None),
             last_error: std::sync::RwLock::new("".to_string()),
             debug_logging: std::sync::RwLock::new(None),
@@ -398,7 +408,7 @@ impl Context {
     }
 
     /// Starts the IO scheduler.
-    pub async fn start_io(&mut self) {
+    pub async fn start_io(&self) {
         if !self.is_configured().await.unwrap_or_default() {
             warn!(self, "can not start io on a context that is not configured");
             return;
@@ -436,6 +446,58 @@ impl Context {
         self.scheduler.maybe_network().await;
     }
 
+    /// Does a background fetch
+    /// pauses the scheduler and does one imap fetch, then unpauses and returns
+    pub async fn background_fetch(&self) -> Result<()> {
+        if !(self.is_configured().await?) {
+            return Ok(());
+        }
+
+        let address = self.get_primary_self_addr().await?;
+        let time_start = tools::Time::now();
+        info!(self, "background_fetch started fetching {address}");
+
+        let _pause_guard = self.scheduler.pause(self.clone()).await?;
+
+        // connection
+        let mut connection = Imap::new_configured(self, channel::bounded(1).1).await?;
+        connection.prepare(self).await?;
+
+        // fetch imap folders
+        for folder_meaning in [FolderMeaning::Inbox, FolderMeaning::Mvbox] {
+            let (_, watch_folder) = convert_folder_meaning(self, folder_meaning).await?;
+            connection
+                .fetch_move_delete(self, &watch_folder, folder_meaning)
+                .await?;
+        }
+
+        // update quota (to send warning if full) - but only check it once in a while
+        let quota_needs_update = {
+            let quota = self.quota.read().await;
+            quota
+                .as_ref()
+                .filter(|quota| {
+                    time_elapsed(&quota.modified)
+                        > Duration::from_secs(DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT)
+                })
+                .is_none()
+        };
+
+        if quota_needs_update {
+            if let Err(err) = self.update_recent_quota(&mut connection).await {
+                warn!(self, "Failed to update quota: {err:#}.");
+            }
+        }
+
+        info!(
+            self,
+            "background_fetch done for {address} took {:?}",
+            time_elapsed(&time_start),
+        );
+
+        Ok(())
+    }
+
     pub(crate) async fn schedule_resync(&self) -> Result<()> {
         self.resync_request.store(true, Ordering::Relaxed);
         self.scheduler.interrupt_inbox().await;
@@ -532,7 +594,7 @@ impl Context {
     pub(crate) async fn free_ongoing(&self) {
         let mut s = self.running_state.write().await;
         if let RunningState::ShallStop { request } = *s {
-            info!(self, "Ongoing stopped in {:?}", request.elapsed());
+            info!(self, "Ongoing stopped in {:?}", time_elapsed(&request));
         }
         *s = RunningState::Stopped;
     }
@@ -547,7 +609,7 @@ impl Context {
                 }
                 info!(self, "Signaling the ongoing process to stop ASAP.",);
                 *s = RunningState::ShallStop {
-                    request: Instant::now(),
+                    request: tools::Time::now(),
                 };
             }
             RunningState::ShallStop { .. } | RunningState::Stopped => {
@@ -613,7 +675,7 @@ impl Context {
         let only_fetch_mvbox = self.get_config_int(Config::OnlyFetchMvbox).await?;
         let folders_configured = self
             .sql
-            .get_raw_config_int("folders_configured")
+            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
             .await?
             .unwrap_or_default();
 
@@ -669,6 +731,16 @@ impl Context {
             res.insert("imap_server_id", format!("{server_id:?}"));
         }
 
+        if let Some(metadata) = &*self.metadata.read().await {
+            if let Some(comment) = &metadata.comment {
+                res.insert("imap_server_comment", format!("{comment:?}"));
+            }
+
+            if let Some(admin) = &metadata.admin {
+                res.insert("imap_server_admin", format!("{admin:?}"));
+            }
+        }
+
         res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
@@ -695,7 +767,10 @@ impl Context {
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
         res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
-        res.insert("folders_configured", folders_configured.to_string());
+        res.insert(
+            constants::DC_FOLDERS_CONFIGURED_KEY,
+            folders_configured.to_string(),
+        );
         res.insert("configured_inbox_folder", configured_inbox_folder);
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
@@ -746,6 +821,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "last_cant_decrypt_outgoing_msgs",
+            self.get_config_int(Config::LastCantDecryptOutgoingMsgs)
+                .await?
+                .to_string(),
+        );
         res.insert(
             "scan_all_folders_debounce_secs",
             self.get_config_int(Config::ScanAllFoldersDebounceSecs)
@@ -789,12 +870,97 @@ impl Context {
                 .to_string(),
         );
 
-        let elapsed = self.creation_time.elapsed();
-        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
+        let elapsed = time_elapsed(&self.creation_time);
+        res.insert("uptime", duration_to_str(elapsed));
 
         Ok(res)
     }
 
+    async fn get_self_report(&self) -> Result<String> {
+        let mut res = String::new();
+        res += &format!("core_version {}\n", get_version_str());
+
+        let num_msgs: u32 = self
+            .sql
+            .query_get_value(
+                "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?",
+                (DC_CHAT_ID_TRASH,),
+            )
+            .await?
+            .unwrap_or_default();
+        res += &format!("num_msgs {}\n", num_msgs);
+
+        let num_chats: u32 = self
+            .sql
+            .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ())
+            .await?
+            .unwrap_or_default();
+        res += &format!("num_chats {}\n", num_chats);
+
+        let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len();
+        res += &format!("db_size_bytes {}\n", db_size);
+
+        let secret_key = &load_self_secret_key(self).await?.primary_key;
+        let key_created = secret_key.created_at().timestamp();
+        res += &format!("key_created {}\n", key_created);
+
+        let self_reporting_id = match self.get_config(Config::SelfReportingId).await? {
+            Some(id) => id,
+            None => {
+                let id = create_id();
+                self.set_config(Config::SelfReportingId, Some(&id)).await?;
+                id
+            }
+        };
+        res += &format!("self_reporting_id {}", self_reporting_id);
+
+        Ok(res)
+    }
+
+    /// Drafts a message with statistics about the usage of Delta Chat.
+    /// The user can inspect the message if they want, and then hit "Send".
+    ///
+    /// On the other end, a bot will receive the message and make it available
+    /// to Delta Chat's developers.
+    pub async fn draft_self_report(&self) -> Result<ChatId> {
+        const SELF_REPORTING_BOT: &str = "self_reporting@testrun.org";
+
+        let contact_id = Contact::create(self, "Statistics bot", SELF_REPORTING_BOT).await?;
+        let chat_id = ChatId::create_for_contact(self, contact_id).await?;
+
+        // We're including the bot's public key in Delta Chat
+        // so that the first message to the bot can directly be encrypted:
+        let public_key = SignedPublicKey::from_base64(
+            "xjMEZbfBlBYJKwYBBAHaRw8BAQdABpLWS2PUIGGo4pslVt4R8sylP5wZihmhf1DTDr3oCM\
+	        PNHDxzZWxmX3JlcG9ydGluZ0B0ZXN0cnVuLm9yZz7CiwQQFggAMwIZAQUCZbfBlAIbAwQLCQgHBhUI\
+	        CQoLAgMWAgEWIQTS2i16sHeYTckGn284K3M5Z4oohAAKCRA4K3M5Z4oohD8dAQCQV7CoH6UP4PD+Nq\
+	        I4kW5tbbqdh2AnDROg60qotmLExAEAxDfd3QHAK9f8b9qQUbLmHIztCLxhEuVbWPBEYeVW0gvOOARl\
+	        t8GUEgorBgEEAZdVAQUBAQdAMBUhYoAAcI625vGZqnM5maPX4sGJ7qvJxPAFILPy6AcDAQgHwngEGB\
+	        YIACAFAmW3wZQCGwwWIQTS2i16sHeYTckGn284K3M5Z4oohAAKCRA4K3M5Z4oohPwCAQCvzk1ObIkj\
+	        2GqsuIfaULlgdnfdZY8LNary425CEfHZDQD5AblXVrlMO1frdlc/Vo9z3pEeCrfYdD7ITD3/OeVoiQ\
+	        4=",
+        )?;
+        let mut peerstate = Peerstate::from_public_key(
+            SELF_REPORTING_BOT,
+            0,
+            EncryptPreference::Mutual,
+            &public_key,
+        );
+        let fingerprint = public_key.fingerprint();
+        peerstate.set_verified(public_key, fingerprint, "".to_string())?;
+        peerstate.save_to_db(&self.sql).await?;
+        chat_id
+            .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id))
+            .await?;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = self.get_self_report().await?;
+
+        chat_id.set_draft(self, Some(&mut msg)).await?;
+
+        Ok(chat_id)
+    }
+
     /// Get a list of fresh, unmuted messages in unblocked chats.
     ///
     /// The list starts with the most recent message
@@ -1051,7 +1217,7 @@ pub fn get_version_str() -> &'static str {
 
 #[cfg(test)]
 mod tests {
-    use std::time::Duration;
+    use std::time::{Duration, SystemTime};
 
     use anyhow::Context as _;
     use strum::IntoEnumIterator;
@@ -1065,8 +1231,9 @@ mod tests {
     use crate::constants::Chattype;
     use crate::contact::ContactId;
     use crate::message::{Message, Viewtype};
+    use crate::mimeparser::SystemMessage;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{get_chat_msg, TestContext};
     use crate::tools::create_outgoing_rfc724_mid;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1305,6 +1472,7 @@ mod tests {
             "mail_security",
             "notify_about_wrong_pw",
             "save_mime_headers",
+            "self_reporting_id",
             "selfstatus",
             "send_server",
             "send_user",
@@ -1318,6 +1486,7 @@ mod tests {
             "socks5_port",
             "socks5_user",
             "socks5_password",
+            "key_id",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();
@@ -1369,7 +1538,7 @@ mod tests {
         assert_eq!(res.len(), 2);
 
         // Message added later is returned first.
-        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.first(), Some(&msg2.id));
         assert_eq!(res.get(1), Some(&msg1.id));
 
         // Global search with longer text does not find any message.
@@ -1586,7 +1755,7 @@ mod tests {
 
         let bob_next_msg_ids = bob.get_next_msgs().await?;
         assert_eq!(bob_next_msg_ids.len(), 1);
-        assert_eq!(bob_next_msg_ids.get(0), Some(&received_msg.id));
+        assert_eq!(bob_next_msg_ids.first(), Some(&received_msg.id));
 
         bob.set_config_u32(Config::LastMsgId, received_msg.id.to_u32())
             .await?;
@@ -1595,7 +1764,7 @@ mod tests {
         // Next messages include self-sent messages.
         let alice_next_msg_ids = alice.get_next_msgs().await?;
         assert_eq!(alice_next_msg_ids.len(), 1);
-        assert_eq!(alice_next_msg_ids.get(0), Some(&sent_msg.sender_msg_id));
+        assert_eq!(alice_next_msg_ids.first(), Some(&sent_msg.sender_msg_id));
 
         alice
             .set_config_u32(Config::LastMsgId, sent_msg.sender_msg_id.to_u32())
@@ -1604,4 +1773,24 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_draft_self_report() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let chat_id = alice.draft_self_report().await?;
+        let msg = get_chat_msg(&alice, chat_id, 0, 1).await;
+        assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled);
+
+        let chat = Chat::load_from_db(&alice, chat_id).await?;
+        assert!(chat.is_protected());
+
+        let mut draft = chat_id.get_draft(&alice).await?.unwrap();
+        assert!(draft.text.starts_with("core_version"));
+
+        // Test that sending into the protected chat works:
+        let _sent = alice.send_msg(chat_id, &mut draft).await;
+
+        Ok(())
+    }
 }

src/decrypt.rs

@@ -23,32 +23,14 @@ use crate::pgp;
 ///
 /// If the message is wrongly signed, HashSet will be empty.
 pub fn try_decrypt(
-    context: &Context,
     mail: &ParsedMail<'_>,
     private_keyring: &[SignedSecretKey],
     public_keyring_for_validate: &[SignedPublicKey],
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let encrypted_data_part = match {
-        let mime = get_autocrypt_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected Autocrypt-mime message.");
-        }
-        mime
-    }
-    .or_else(|| {
-        let mime = get_mixed_up_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected mixed-up mime message.");
-        }
-        mime
-    })
-    .or_else(|| {
-        let mime = get_attachment_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected attached Autocrypt-mime message.");
-        }
-        mime
-    }) {
+    let encrypted_data_part = match get_autocrypt_mime(mail)
+        .or_else(|| get_mixed_up_mime(mail))
+        .or_else(|| get_attachment_mime(mail))
+    {
         None => return Ok(None),
         Some(res) => res,
     };

src/dehtml.rs

@@ -181,7 +181,12 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
     if dehtml.get_add_text() == AddText::YesPreserveLineEnds
         || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
-        let last_added = escaper::decode_html_buf_sloppy(event as &[_]).unwrap_or_default();
+        let event = event as &[_];
+        let event_str = std::str::from_utf8(event).unwrap_or_default();
+        let mut last_added = escaper::decode_html_buf_sloppy(event).unwrap_or_default();
+        if event_str.starts_with(&last_added) {
+            last_added = event_str.to_string();
+        }
 
         if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             // Replace all line ends with spaces.
@@ -527,6 +532,6 @@ mod tests {
     fn test_spaces() {
         let input = include_str!("../test-data/spaces.html");
         let txt = dehtml(input).unwrap();
-        assert_eq!(txt.text, "Welcome back to Strolling!\n\nHey there,\n\nWelcome back! Use this link to securely sign in to your Strolling account:\n\nSign in to Strolling\n\nFor your security, the link will expire in 24 hours time.\n\nSee you soon!\n\nYou can also copy\n\nhttps://strolling.rosano.ca/members/?token=XXX\n\nIf you did not make this request, you can safely ignore this email.\n\nThis message was sent from [strolling.rosano.ca](https://strolling.rosano.ca/) to [alice@example.org](mailto:alice@example.org)");
+        assert_eq!(txt.text, "Welcome back to Strolling!\n\nHey there,\n\nWelcome back! Use this link to securely sign in to your Strolling account:\n\nSign in to Strolling\n\nFor your security, the link will expire in 24 hours time.\n\nSee you soon!\n\nYou can also copy & paste this URL into your browser:\n\nhttps://strolling.rosano.ca/members/?token=XXX&action=signin&r=https%3A%2F%2Fstrolling.rosano.ca%2F\n\nIf you did not make this request, you can safely ignore this email.\n\nThis message was sent from [strolling.rosano.ca](https://strolling.rosano.ca/) to [alice@example.org](mailto:alice@example.org)");
     }
 }

src/download.rs

@@ -136,39 +136,36 @@ pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Im
     let row = context
         .sql
         .query_row_optional(
-            "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
+            "SELECT uid, folder, uidvalidity FROM imap WHERE rfc724_mid=? AND target!=''",
             (&msg.rfc724_mid,),
             |row| {
                 let server_uid: u32 = row.get(0)?;
                 let server_folder: String = row.get(1)?;
-                Ok((server_uid, server_folder))
+                let uidvalidity: u32 = row.get(2)?;
+                Ok((server_uid, server_folder, uidvalidity))
             },
         )
         .await?;
 
-    if let Some((server_uid, server_folder)) = row {
-        match imap
-            .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
-            .await
-        {
-            ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                msg.id
-                    .update_download_state(context, DownloadState::Failure)
-                    .await?;
-                Err(anyhow!("Call download_full() again to try over."))
-            }
-            ImapActionResult::Success => {
-                // update_download_state() not needed as receive_imf() already
-                // set the state and emitted the event.
-                Ok(())
-            }
-        }
-    } else {
+    let Some((server_uid, server_folder, uidvalidity)) = row else {
         // No IMAP record found, we don't know the UID and folder.
-        msg.id
-            .update_download_state(context, DownloadState::Failure)
-            .await?;
-        Err(anyhow!("Call download_full() again to try over."))
+        return Err(anyhow!("Call download_full() again to try over."));
+    };
+
+    match imap
+        .fetch_single_msg(
+            context,
+            &server_folder,
+            uidvalidity,
+            server_uid,
+            msg.rfc724_mid.clone(),
+        )
+        .await
+    {
+        ImapActionResult::RetryLater | ImapActionResult::Failed => {
+            Err(anyhow!("Call download_full() again to try over."))
+        }
+        ImapActionResult::Success => Ok(()),
     }
 }
 
@@ -181,6 +178,7 @@ impl Imap {
         &mut self,
         context: &Context,
         folder: &str,
+        uidvalidity: u32,
         uid: u32,
         rfc724_mid: String,
     ) -> ImapActionResult {
@@ -197,7 +195,15 @@ impl Imap {
         let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
         uid_message_ids.insert(uid, rfc724_mid);
         let (last_uid, _received) = match self
-            .fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, false)
+            .fetch_many_msgs(
+                context,
+                folder,
+                uidvalidity,
+                vec![uid],
+                &uid_message_ids,
+                false,
+                false,
+            )
             .await
         {
             Ok(res) => res,
@@ -257,7 +263,7 @@ mod tests {
     use crate::chat::{get_chat_msgs, send_msg};
     use crate::ephemeral::Timer;
     use crate::message::Viewtype;
-    use crate::receive_imf::receive_imf_inner;
+    use crate::receive_imf::receive_imf_from_inbox;
     use crate::test_utils::TestContext;
 
     #[test]
@@ -338,7 +344,7 @@ mod tests {
              Date: Sun, 22 Mar 2020 22:37:57 +0000\
              Content-Type: text/plain";
 
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &t,
             "Mr.12345678901@example.com",
             header.as_bytes(),
@@ -354,7 +360,7 @@ mod tests {
             .get_text()
             .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
 
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &t,
             "Mr.12345678901@example.com",
             format!("{header}\n\n100k text...").as_bytes(),
@@ -383,7 +389,7 @@ mod tests {
             .await?;
 
         // download message from bob partially, this must not change the ephemeral timer
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &t,
             "first@example.org",
             b"From: Bob <bob@example.org>\n\
@@ -426,7 +432,7 @@ mod tests {
         let sent2_rfc724_mid = sent2.load_from_db().await.rfc724_mid;
 
         // not downloading the status update results in an placeholder
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &bob,
             &sent2_rfc724_mid,
             sent2.payload().as_bytes(),
@@ -442,7 +448,7 @@ mod tests {
 
         // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat
         // (usually status updates are too small for not being downloaded directly)
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &bob,
             &sent2_rfc724_mid,
             sent2.payload().as_bytes(),
@@ -493,7 +499,7 @@ mod tests {
             ";
 
         // not downloading the mdn results in an placeholder
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &bob,
             "bar@example.org",
             raw,
@@ -509,7 +515,7 @@ mod tests {
 
         // downloading the mdn afterwards expands to nothing and deletes the placeholder directly
         // (usually mdn are too small for not being downloaded directly)
-        receive_imf_inner(&bob, "bar@example.org", raw, false, None, false).await?;
+        receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None, false).await?;
         assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0);
         assert!(Message::load_from_db(&bob, msg.id)
             .await?

src/e2ee.rs

@@ -52,7 +52,7 @@ impl EncryptHelper {
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
-        peerstates: &[(Option<Peerstate>, &str)],
+        peerstates: &[(Option<Peerstate>, String)],
     ) -> Result<bool> {
         let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
             1
@@ -62,21 +62,19 @@ impl EncryptHelper {
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    info!(
-                        context,
-                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
-                    );
+                    let prefer_encrypt = peerstate.prefer_encrypt;
+                    info!(context, "Peerstate for {addr:?} is {prefer_encrypt}.");
                     match peerstate.prefer_encrypt {
                         EncryptPreference::NoPreference | EncryptPreference::Reset => {}
                         EncryptPreference::Mutual => prefer_encrypt_count += 1,
                     };
                 }
                 None => {
-                    let msg = format!("peerstate for {addr:?} missing, cannot encrypt");
+                    let msg = format!("Peerstate for {addr:?} missing, cannot encrypt");
                     if e2ee_guaranteed {
-                        return Err(format_err!("{}", msg));
+                        return Err(format_err!("{msg}"));
                     } else {
-                        info!(context, "{}", msg);
+                        info!(context, "{msg}.");
                         return Ok(false);
                     }
                 }
@@ -96,7 +94,7 @@ impl EncryptHelper {
         context: &Context,
         verified: bool,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate>, &str)>,
+        peerstates: Vec<(Option<Peerstate>, String)>,
     ) -> Result<String> {
         let mut keyring: Vec<SignedPublicKey> = Vec::new();
 
@@ -119,7 +117,7 @@ impl EncryptHelper {
         // Encrypt to secondary verified keys
         // if we also encrypt to the introducer ("verifier") of the key.
         if verified {
-            for (peerstate, _addr) in peerstates {
+            for (peerstate, _addr) in &peerstates {
                 if let Some(peerstate) = peerstate {
                     if let (Some(key), Some(verifier)) = (
                         peerstate.secondary_verified_key.as_ref(),
@@ -171,11 +169,10 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat;
     use crate::key::DcKey;
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
-    use crate::test_utils::{bob_keypair, TestContext};
+    use crate::test_utils::{bob_keypair, TestContext, TestContextManager};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -219,37 +216,35 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
 
         let chat_alice = alice.create_chat(&bob).await.id;
         let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
         // Bob receives unencrypted message from Alice
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
 
-        // Parsing a message is enough to update peerstate
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
-        // Bob sends encrypted message to Alice
+        // Bob sends empty encrypted message to Alice
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        let sent = bob.pop_sent_msg().await;
+        let sent = bob.send_msg(chat_bob, &mut msg).await;
 
-        // Alice receives encrypted message from Bob
-        let msg = alice.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        // Alice receives an empty encrypted message from Bob.
+        // This is also a regression test for previously existing bug
+        // that resulted in no padlock on encrypted empty messages.
+        let msg = alice.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
 
         let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
             .await?
@@ -261,12 +256,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends encrypted message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -275,12 +268,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -290,12 +281,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -304,7 +293,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         Ok(())
     }
 
-    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, &'static str)> {
+    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, String)> {
         let addr = "bob@foo.bar";
         let pub_key = bob_keypair().public;
         let peerstate = Peerstate {
@@ -323,9 +312,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
-        vec![(Some(peerstate), addr)]
+        vec![(Some(peerstate), addr.to_string())]
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -350,7 +340,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with missing peerstate
-        let ps = vec![(None, "bob@foo.bar")];
+        let ps = vec![(None, "bob@foo.bar".to_string())];
         assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
         assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }

src/ephemeral.rs

@@ -590,7 +590,11 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()
         match context.get_config_delete_server_after().await? {
             None => (0, 0),
             Some(delete_server_after) => (
-                now - delete_server_after,
+                match delete_server_after {
+                    // Guarantee immediate deletion.
+                    0 => i64::MAX,
+                    _ => now - delete_server_after,
+                },
                 now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
             ),
         };
@@ -1127,6 +1131,7 @@ mod tests {
             (1030, now - 19 * HOUR, 0),
             (2000, now - 18 * HOUR, now - HOUR),
             (2020, now - 17 * HOUR, now + HOUR),
+            (3000, now + HOUR, 0),
         ] {
             let message_id = id.to_string();
             t.sql
@@ -1212,6 +1217,10 @@ mod tests {
             0
         );
 
+        t.set_config(Config::DeleteServerAfter, Some("1")).await?;
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 3000).await?;
+
         Ok(())
     }
 

src/events/payload.rs

@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
 use crate::chat::ChatId;
+use crate::config::Config;
 use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
@@ -261,8 +262,17 @@ pub enum EventType {
     ConnectivityChanged,
 
     /// The user's avatar changed.
+    /// Deprecated by `ConfigSynced`.
     SelfavatarChanged,
 
+    /// A multi-device synced config value changed. Maybe the app needs to refresh smth. For
+    /// uniformity this is emitted on the source device too. The value isn't here, otherwise it
+    /// would be logged which might not be good for privacy.
+    ConfigSynced {
+        /// Configuration key.
+        key: Config,
+    },
+
     /// Webxdc status update received.
     WebxdcStatusUpdate {
         /// Message ID.
@@ -277,4 +287,11 @@ pub enum EventType {
         /// ID of the deleted message.
         msg_id: MsgId,
     },
+
+    /// Tells that the Background fetch was completed (or timed out).
+    /// This event acts as a marker, when you reach this event you can be sure
+    /// that all events emitted during the background fetch were processed.
+    ///
+    /// This event is only emitted by the account manager
+    AccountsBackgroundFetchDone,
 }

src/headerdef.rs

@@ -2,7 +2,7 @@
 
 use mailparse::{MailHeader, MailHeaderMap};
 
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, IntoStaticStr)]
+#[derive(Debug, Display, Clone, PartialEq, Eq, IntoStaticStr)]
 #[strum(serialize_all = "kebab_case")]
 #[allow(missing_docs)]
 pub enum HeaderDef {
@@ -38,6 +38,9 @@ pub enum HeaderDef {
     /// Mailing list ID defined in [RFC 2919](https://tools.ietf.org/html/rfc2919).
     ListId,
     ListPost,
+
+    /// List-Help header defined in [RFC 2369](https://datatracker.ietf.org/doc/html/rfc2369).
+    ListHelp,
     References,
 
     /// In-Reply-To header containing Message-ID of the parent message.

src/imap.rs

@@ -22,9 +22,7 @@ use tokio::sync::RwLock;
 
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
-use crate::constants::{
-    Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION,
-};
+use crate::constants::{self, Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT};
 use crate::contact::{normalize_name, Contact, ContactAddress, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::events::EventType;
@@ -73,6 +71,7 @@ pub enum ImapActionResult {
 ///   not necessarily sent by Delta Chat.
 const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
                               MESSAGE-ID \
+                              DATE \
                               X-MICROSOFT-ORIGINAL-MESSAGE-ID \
                               FROM \
                               IN-REPLY-TO REFERENCES \
@@ -94,6 +93,20 @@ pub struct Imap {
     login_failed_once: bool,
 
     pub(crate) connectivity: ConnectivityStore,
+
+    /// Rate limit for IMAP connection usage attempts.
+    ///
+    /// Rate limit is checked before connecting
+    /// and updated right before login attempt.
+    /// It does not limit the number of connection attempts
+    /// if the network is bad as only successful connections are counted.
+    ///
+    /// Main purpose of this rate limit is
+    /// to prevent busy loop in case
+    /// connection gets dropped over and over due to IMAP bug,
+    /// e.g. the server returning invalid response to SELECT command
+    /// immediately after logging in or returning an error in response to LOGIN command
+    /// due to internal server error.
     ratelimit: RwLock<Ratelimit>,
 }
 
@@ -103,6 +116,17 @@ struct OAuth2 {
     access_token: String,
 }
 
+#[derive(Debug)]
+pub(crate) struct ServerMetadata {
+    /// IMAP METADATA `/shared/comment` as defined in
+    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1>.
+    pub comment: Option<String>,
+
+    /// IMAP METADATA `/shared/admin` as defined in
+    /// <https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2>.
+    pub admin: Option<String>,
+}
+
 impl async_imap::Authenticator for OAuth2 {
     type Response = String;
 
@@ -256,7 +280,7 @@ impl Imap {
             session: None,
             login_failed_once: false,
             connectivity: Default::default(),
-            // 1 connection per minute + a burst of 2.
+            // 1 login per minute + a burst of 2.
             ratelimit: RwLock::new(Ratelimit::new(Duration::new(120, 0), 2.0)),
         };
 
@@ -306,6 +330,12 @@ impl Imap {
         }
 
         self.connectivity.set_connecting(context).await;
+
+        // Check rate limit before trying to connect
+        // to avoid connecting and not using the connection
+        // in case we are currently ratelimited.
+        // Otherwise connection may become unusable due to NAT forgetting about it
+        // before we attempt to actually login.
         let ratelimit_duration = self.ratelimit.read().await.until_can_send();
         if !ratelimit_duration.is_zero() {
             warn!(
@@ -316,10 +346,7 @@ impl Imap {
             tokio::time::sleep(ratelimit_duration).await;
         }
 
-        let oauth2 = self.config.lp.oauth2;
-
         info!(context, "Connecting to IMAP server");
-        self.ratelimit.write().await.send();
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
         {
@@ -369,11 +396,13 @@ impl Imap {
                 Client::connect_secure(context, imap_server, imap_port, config.strict_tls).await
             }
         };
-
         let client = connection_res?;
+        self.ratelimit.write().await.send();
+
         let config = &self.config;
         let imap_user: &str = config.lp.user.as_ref();
         let imap_pw: &str = config.lp.password.as_ref();
+        let oauth2 = self.config.lp.oauth2;
 
         let login_res = if oauth2 {
             info!(context, "Logging into IMAP server with OAuth 2");
@@ -420,7 +449,10 @@ impl Imap {
                     && err.to_string().to_lowercase().contains("authentication")
                     && context.get_config_bool(Config::NotifyAboutWrongPw).await?
                 {
-                    if let Err(e) = context.set_config(Config::NotifyAboutWrongPw, None).await {
+                    if let Err(e) = context
+                        .set_config_internal(Config::NotifyAboutWrongPw, None)
+                        .await
+                    {
                         warn!(context, "{:#}", e);
                     }
                     drop(lock);
@@ -750,6 +782,7 @@ impl Imap {
         let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1);
         let mut uid_message_ids = BTreeMap::new();
         let mut largest_uid_skipped = None;
+        let delete_target = context.get_delete_msgs_target().await?;
 
         // Store the info about IMAP messages in the database.
         for (uid, ref fetch_response) in msgs {
@@ -775,8 +808,24 @@ impl Imap {
             // Such move to the same folder results in the messages
             // getting a new UID, so the messages will be detected as new
             // in the `INBOX.DeltaChat` folder again.
+            let _target;
             let target = if let Some(message_id) = &message_id {
-                if context
+                let is_dup = if let Some((_, ts_sent_old)) =
+                    message::rfc724_mid_exists(context, message_id).await?
+                {
+                    let is_chat_msg = headers.get_header_value(HeaderDef::ChatVersion).is_some();
+                    let ts_sent = headers
+                        .get_header_value(HeaderDef::Date)
+                        .and_then(|v| mailparse::dateparse(&v).ok())
+                        .unwrap_or_default();
+                    is_dup_msg(is_chat_msg, ts_sent, ts_sent_old)
+                } else {
+                    false
+                };
+                if is_dup {
+                    info!(context, "Deleting duplicate message {message_id}.");
+                    &delete_target
+                } else if context
                     .sql
                     .exists(
                         "SELECT COUNT (*) FROM imap WHERE rfc724_mid=?",
@@ -788,9 +837,10 @@ impl Imap {
                         context,
                         "Not moving the message {} that we have seen before.", &message_id
                     );
-                    folder.to_string()
+                    folder
                 } else {
-                    target_folder(context, folder, folder_meaning, &headers).await?
+                    _target = target_folder(context, folder, folder_meaning, &headers).await?;
+                    &_target
                 }
             } else {
                 // Do not move the messages without Message-ID.
@@ -800,7 +850,7 @@ impl Imap {
                     context,
                     "Not moving the message that does not have a Message-ID."
                 );
-                folder.to_string()
+                folder
             };
 
             // Generate a fake Message-ID to identify the message in the database
@@ -815,7 +865,7 @@ impl Imap {
                        ON CONFLICT(folder, uid, uidvalidity)
                        DO UPDATE SET rfc724_mid=excluded.rfc724_mid,
                                      target=excluded.target",
-                    (&message_id, &folder, uid, uid_validity, &target),
+                    (&message_id, &folder, uid, uid_validity, target),
                 )
                 .await?;
 
@@ -868,6 +918,7 @@ impl Imap {
                     .fetch_many_msgs(
                         context,
                         folder,
+                        uid_validity,
                         uids_fetch_in_batch.split_off(0),
                         &uid_message_ids,
                         fetch_partially,
@@ -975,7 +1026,7 @@ impl Imap {
         self.prepare(context).await?;
 
         let all_folders = self
-            .list_folders(context)
+            .list_folders()
             .await
             .context("listing folders for resync")?;
         for folder in all_folders {
@@ -1412,10 +1463,12 @@ impl Imap {
     /// Returns the last UID fetched successfully and the info about each downloaded message.
     /// If the message is incorrect or there is a failure to write a message to the database,
     /// it is skipped and the error is logged.
+    #[allow(clippy::too_many_arguments)]
     pub(crate) async fn fetch_many_msgs(
         &mut self,
         context: &Context,
         folder: &str,
+        uidvalidity: u32,
         request_uids: Vec<u32>,
         uid_message_ids: &BTreeMap<u32, String>,
         fetch_partially: bool,
@@ -1549,6 +1602,9 @@ impl Imap {
                 );
                 match receive_imf_inner(
                     context,
+                    folder,
+                    uidvalidity,
+                    request_uid,
                     rfc724_mid,
                     body,
                     is_seen,
@@ -1592,6 +1648,50 @@ impl Imap {
 
         Ok((last_uid, received_msgs))
     }
+
+    /// Retrieves server metadata if it is supported.
+    ///
+    /// We get [`/shared/comment`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.1)
+    /// and [`/shared/admin`](https://www.rfc-editor.org/rfc/rfc5464#section-6.2.2)
+    /// metadata.
+    pub(crate) async fn fetch_metadata(&mut self, context: &Context) -> Result<()> {
+        let session = self.session.as_mut().context("no session")?;
+        if !session.can_metadata() {
+            return Ok(());
+        }
+
+        let mut lock = context.metadata.write().await;
+        if (*lock).is_some() {
+            return Ok(());
+        }
+
+        info!(
+            context,
+            "Server supports metadata, retrieving server comment and admin contact."
+        );
+
+        let mut comment = None;
+        let mut admin = None;
+
+        let mailbox = "";
+        let options = "";
+        let metadata = session
+            .get_metadata(mailbox, options, "(/shared/comment /shared/admin)")
+            .await?;
+        for m in metadata {
+            match m.entry.as_ref() {
+                "/shared/comment" => {
+                    comment = m.value;
+                }
+                "/shared/admin" => {
+                    admin = m.value;
+                }
+                _ => {}
+            }
+        }
+        *lock = Some(ServerMetadata { comment, admin });
+        Ok(())
+    }
 }
 
 impl Session {
@@ -1665,18 +1765,24 @@ impl Imap {
         context: &Context,
         create_mvbox: bool,
     ) -> Result<()> {
-        let folders_configured = context.sql.get_raw_config_int("folders_configured").await?;
-        if folders_configured.unwrap_or_default() >= DC_FOLDERS_CONFIGURED_VERSION {
+        let folders_configured = context
+            .sql
+            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
+            .await?;
+        if folders_configured.unwrap_or_default() >= constants::DC_FOLDERS_CONFIGURED_VERSION {
             return Ok(());
         }
-
+        if let Err(err) = self.connect(context).await {
+            self.connectivity.set_err(context, &err).await;
+            return Err(err);
+        }
         self.configure_folders(context, create_mvbox).await
     }
 
     /// Attempts to configure mvbox.
     ///
     /// Tries to find any folder in the given list of `folders`. If none is found, tries to create
-    /// any of them in the same order. This method does not use LIST command to ensure that
+    /// `folders[0]`. This method does not use LIST command to ensure that
     /// configuration works even if mailbox lookup is forbidden via Access Control List (see
     /// <https://datatracker.ietf.org/doc/html/rfc4314>).
     ///
@@ -1705,24 +1811,28 @@ impl Imap {
                     "MVBOX-folder {:?} successfully selected, using it.", &folder
                 );
                 session.close().await?;
+                // Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
+                // emails moved before that wouldn't be fetched but considered "old" instead.
+                self.select_with_uidvalidity(context, folder).await?;
                 return Ok(Some(folder));
             }
         }
 
-        if create_mvbox {
-            for folder in folders {
-                match session.create(&folder).await {
-                    Ok(_) => {
-                        info!(context, "MVBOX-folder {} created.", &folder);
-                        return Ok(Some(folder));
-                    }
-                    Err(err) => {
-                        warn!(context, "Cannot create MVBOX-folder {:?}: {}", &folder, err);
-                    }
-                }
+        if !create_mvbox {
+            return Ok(None);
+        }
+        let Some(folder) = folders.first() else {
+            return Ok(None);
+        };
+        match self.select_with_uidvalidity(context, folder).await {
+            Ok(_) => {
+                info!(context, "MVBOX-folder {} created.", folder);
+                return Ok(Some(folder));
+            }
+            Err(err) => {
+                warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
             }
         }
-
         Ok(None)
     }
 
@@ -1774,20 +1884,23 @@ impl Imap {
             .context("failed to configure mvbox")?;
 
         context
-            .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
+            .set_config_internal(Config::ConfiguredInboxFolder, Some("INBOX"))
             .await?;
         if let Some(mvbox_folder) = mvbox_folder {
             info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
             context
-                .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
+                .set_config_internal(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
                 .await?;
         }
         for (config, name) in folder_configs {
-            context.set_config(config, Some(&name)).await?;
+            context.set_config_internal(config, Some(&name)).await?;
         }
         context
             .sql
-            .set_raw_config_int("folders_configured", DC_FOLDERS_CONFIGURED_VERSION)
+            .set_raw_config_int(
+                constants::DC_FOLDERS_CONFIGURED_KEY,
+                constants::DC_FOLDERS_CONFIGURED_VERSION,
+            )
             .await?;
 
         info!(context, "FINISHED configuring IMAP-folders.");
@@ -1805,13 +1918,14 @@ impl Session {
         use async_imap::imap_proto::ResponseCode;
         use UnsolicitedResponse::*;
 
+        let folder = self.selected_folder.as_deref().unwrap_or_default();
         let mut unsolicited_exists = false;
         while let Ok(response) = self.unsolicited_responses.try_recv() {
             match response {
                 Exists(_) => {
                     info!(
                         context,
-                        "Need to fetch again, got unsolicited EXISTS {:?}", response
+                        "Need to refetch {folder:?}, got unsolicited EXISTS {response:?}"
                     );
                     unsolicited_exists = true;
                 }
@@ -1830,7 +1944,7 @@ impl Session {
                     ) => {}
 
                 _ => {
-                    info!(context, "got unsolicited response {:?}", response)
+                    info!(context, "{folder:?}: got unsolicited response {response:?}")
                 }
             }
         }
@@ -2228,6 +2342,15 @@ pub(crate) async fn prefetch_should_download(
     Ok(should_download)
 }
 
+/// Returns whether a message is a duplicate (resent message).
+pub(crate) fn is_dup_msg(is_chat_msg: bool, ts_sent: i64, ts_sent_old: i64) -> bool {
+    // If the existing message has timestamp_sent == 0, that means we don't know its actual sent
+    // timestamp, so don't delete the new message. E.g. outgoing messages have zero timestamp_sent
+    // because they are stored to the db before sending. Also consider as duplicates only messages
+    // with greater timestamp to avoid deleting both messages in a multi-device setting.
+    is_chat_msg && ts_sent_old != 0 && ts_sent > ts_sent_old
+}
+
 /// Marks messages in `msgs` table as seen, searching for them by UID.
 ///
 /// Returns updated chat ID if any message was marked as seen.

src/imap/capabilities.rs

@@ -21,6 +21,10 @@ pub(crate) struct Capabilities {
     /// <https://tools.ietf.org/html/rfc7162>
     pub can_condstore: bool,
 
+    /// True if the server has METADATA capability as defined in
+    /// <https://tools.ietf.org/html/rfc5464>
+    pub can_metadata: bool,
+
     /// Server ID if the server supports ID capability.
     pub server_id: Option<HashMap<String, String>>,
 }

src/imap/client.rs

@@ -59,6 +59,7 @@ async fn determine_capabilities(
         can_move: caps.has_str("MOVE"),
         can_check_quota: caps.has_str("QUOTA"),
         can_condstore: caps.has_str("CONDSTORE"),
+        can_metadata: caps.has_str("METADATA"),
         server_id,
     };
     Ok(capabilities)

src/imap/idle.rs

@@ -1,4 +1,4 @@
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use anyhow::{bail, Context as _, Result};
 use async_channel::Receiver;
@@ -11,8 +11,15 @@ use crate::config::Config;
 use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
 use crate::log::LogExt;
+use crate::tools::{self, time_elapsed};
 
-const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
+/// Timeout after which IDLE is finished
+/// if there are no responses from the server.
+///
+/// If `* OK Still here` keepalives are sent more frequently
+/// than this duration, timeout should never be triggered.
+/// For example, Dovecot sends keepalives every 2 minutes by default.
+const IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
 
 impl Session {
     pub async fn idle(
@@ -100,7 +107,7 @@ impl Imap {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
-        let fake_idle_start_time = SystemTime::now();
+        let fake_idle_start_time = tools::Time::now();
 
         // Do not poll, just wait for an interrupt when no folder is passed in.
         let watch_folder = if let Some(watch_folder) = watch_folder {
@@ -190,11 +197,7 @@ impl Imap {
         info!(
             context,
             "IMAP-fake-IDLE done after {:.4}s",
-            SystemTime::now()
-                .duration_since(fake_idle_start_time)
-                .unwrap_or_default()
-                .as_millis() as f64
-                / 1000.,
+            time_elapsed(&fake_idle_start_time).as_millis() as f64 / 1000.,
         );
     }
 }

src/imap/scan_folders.rs

@@ -1,12 +1,13 @@
-use std::{collections::BTreeMap, time::Instant};
+use std::collections::BTreeMap;
 
 use anyhow::{Context as _, Result};
-use futures::stream::StreamExt;
+use futures::TryStreamExt;
 
 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
 use crate::imap::Imap;
 use crate::log::LogExt;
+use crate::tools::{self, time_elapsed};
 use crate::{context::Context, imap::FolderMeaning};
 
 impl Imap {
@@ -15,7 +16,7 @@ impl Imap {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
-            let elapsed_secs = last_scan.elapsed().as_secs();
+            let elapsed_secs = time_elapsed(&last_scan).as_secs();
             let debounce_secs = context
                 .get_config_u64(Config::ScanAllFoldersDebounceSecs)
                 .await?;
@@ -27,7 +28,7 @@ impl Imap {
         info!(context, "Starting full folder scan");
 
         self.prepare(context).await?;
-        let folders = self.list_folders(context).await?;
+        let folders = self.list_folders().await?;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
@@ -89,30 +90,24 @@ impl Imap {
             Config::ConfiguredTrashFolder,
         ] {
             context
-                .set_config(conf, folder_configs.get(&conf).map(|s| s.as_str()))
+                .set_config_internal(conf, folder_configs.get(&conf).map(|s| s.as_str()))
                 .await?;
         }
 
-        last_scan.replace(Instant::now());
+        last_scan.replace(tools::Time::now());
         Ok(true)
     }
 
     /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(
-        self: &mut Imap,
-        context: &Context,
-    ) -> Result<Vec<async_imap::types::Name>> {
+    pub async fn list_folders(self: &mut Imap) -> Result<Vec<async_imap::types::Name>> {
         let session = self.session.as_mut();
         let session = session.context("No IMAP connection")?;
         let list = session
             .list(Some(""), Some("*"))
             .await?
-            .filter_map(|f| async {
-                f.context("list_folders() can't get folder")
-                    .log_err(context)
-                    .ok()
-            });
-        Ok(list.collect().await)
+            .try_collect()
+            .await?;
+        Ok(list)
     }
 }
 

src/imap/select_folder.rs

@@ -112,11 +112,16 @@ impl ImapSession {
             Err(err) => match err {
                 Error::NoFolder(..) => {
                     info!(context, "Failed to select folder {} because it does not exist, trying to create it.", folder);
-                    self.create(folder).await.with_context(|| {
-                        format!("Couldn't select folder ('{err}'), then create() failed")
-                    })?;
-
-                    Ok(self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {folder}"))?)
+                    let create_res = self.create(folder).await;
+                    if let Err(ref err) = create_res {
+                        info!(context, "Couldn't select folder, then create() failed: {err:#}.");
+                        // Need to recheck, could have been created in parallel.
+                    }
+                    let select_res = self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {folder}"));
+                    if select_res.is_err() {
+                        create_res?;
+                    }
+                    select_res
                 }
                 _ => Err(err).with_context(|| format!("failed to select folder {folder} with error other than NO, not trying to create it")),
             },

src/imap/session.rs

@@ -64,4 +64,8 @@ impl Session {
     pub fn can_condstore(&self) -> bool {
         self.capabilities.can_condstore
     }
+
+    pub fn can_metadata(&self) -> bool {
+        self.capabilities.can_metadata
+    }
 }

src/imex.rs

@@ -375,7 +375,15 @@ async fn imex_inner(
     path: &Path,
     passphrase: Option<String>,
 ) -> Result<()> {
-    info!(context, "Import/export dir: {}", path.display());
+    info!(
+        context,
+        "{} path: {}",
+        match what {
+            ImexMode::ExportSelfKeys | ImexMode::ExportBackup => "Export",
+            ImexMode::ImportSelfKeys | ImexMode::ImportBackup => "Import",
+        },
+        path.display()
+    );
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
@@ -526,7 +534,7 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
     let _d1 = DeleteOnDrop(temp_db_path.clone());
     let _d2 = DeleteOnDrop(temp_path.clone());
 
-    export_database(context, &temp_db_path, passphrase)
+    export_database(context, &temp_db_path, passphrase, now)
         .await
         .context("could not export database")?;
 
@@ -670,7 +678,7 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let keys = context
         .sql
         .query_map(
-            "SELECT id, public_key, private_key, is_default FROM keypairs;",
+            "SELECT id, public_key, private_key, id=(SELECT value FROM config WHERE keyname='key_id') FROM keypairs;",
             (),
             |row| {
                 let id = row.get(0)?;
@@ -762,19 +770,27 @@ where
 /// overwritten.
 ///
 /// This also verifies that IO is not running during the export.
-async fn export_database(context: &Context, dest: &Path, passphrase: String) -> Result<()> {
+async fn export_database(
+    context: &Context,
+    dest: &Path,
+    passphrase: String,
+    timestamp: i64,
+) -> Result<()> {
     ensure!(
         !context.scheduler.is_running().await,
         "cannot export backup, IO is running"
     );
-    let now = time().try_into().context("32-bit UNIX time overflow")?;
+    let timestamp = timestamp.try_into().context("32-bit UNIX time overflow")?;
 
     // TODO: Maybe introduce camino crate for UTF-8 paths where we need them.
     let dest = dest
         .to_str()
         .with_context(|| format!("path {} is not valid unicode", dest.display()))?;
 
-    context.sql.set_raw_config_int("backup_time", now).await?;
+    context
+        .sql
+        .set_raw_config_int("backup_time", timestamp)
+        .await?;
     sql::housekeeping(context).await.log_err(context).ok();
     context
         .sql

src/imex/transfer.rs

@@ -52,6 +52,7 @@ use crate::context::Context;
 use crate::message::{Message, Viewtype};
 use crate::qr::{self, Qr};
 use crate::stock_str::backup_transfer_msg_body;
+use crate::tools::time;
 use crate::{e2ee, EventType};
 
 use super::{export_database, DBFILE_BACKUP_NAME};
@@ -158,7 +159,7 @@ impl BackupProvider {
         // Generate the token up front: we also use it to encrypt the database.
         let token = AuthToken::generate();
         context.emit_event(SendProgress::Started.into());
-        export_database(context, dbfile, token.to_string())
+        export_database(context, dbfile, token.to_string(), time())
             .await
             .context("Database export failed")?;
         context.emit_event(SendProgress::DatabaseExported.into());
@@ -638,7 +639,7 @@ mod tests {
         let self_chat = ctx1.get_self_chat().await;
         let msgs = get_chat_msgs(&ctx1, self_chat.id).await.unwrap();
         assert_eq!(msgs.len(), 2);
-        let msgid = match msgs.get(0).unwrap() {
+        let msgid = match msgs.first().unwrap() {
             ChatItem::Message { msg_id } => msg_id,
             _ => panic!("wrong chat item"),
         };

src/key.rs

@@ -18,7 +18,7 @@ use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::log::LogExt;
 use crate::pgp::KeyPair;
-use crate::tools::{time, EmailAddress};
+use crate::tools::{self, time_elapsed, EmailAddress};
 
 /// Convenience trait for working with keys.
 ///
@@ -82,10 +82,9 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
     match context
         .sql
         .query_row_optional(
-            r#"SELECT public_key
-               FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1"#,
+            "SELECT public_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
             (),
             |row| {
                 let bytes: Vec<u8> = row.get(0)?;
@@ -106,10 +105,9 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
     match context
         .sql
         .query_row_optional(
-            r#"SELECT private_key
-               FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1"#,
+            "SELECT private_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
             (),
             |row| {
                 let bytes: Vec<u8> = row.get(0)?;
@@ -132,8 +130,7 @@ pub(crate) async fn load_self_secret_keyring(context: &Context) -> Result<Vec<Si
         .query_map(
             r#"SELECT private_key
                FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               ORDER BY is_default DESC"#,
+               ORDER BY id=(SELECT value FROM config WHERE keyname='key_id') DESC"#,
             (),
             |row| row.get::<_, Vec<u8>>(0),
             |keys| keys.collect::<Result<Vec<_>, _>>().map_err(Into::into),
@@ -207,7 +204,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     match load_keypair(context, &addr).await? {
         Some(key_pair) => Ok(key_pair),
         None => {
-            let start = std::time::SystemTime::now();
+            let start = tools::Time::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await?)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
@@ -219,7 +216,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().unwrap_or_default().as_secs()
+                time_elapsed(&start).as_secs(),
             );
             Ok(keypair)
         }
@@ -233,13 +230,10 @@ pub(crate) async fn load_keypair(
     let res = context
         .sql
         .query_row_optional(
-            r#"
-        SELECT public_key, private_key
-          FROM keypairs
-         WHERE addr=?1
-           AND is_default=1;
-        "#,
-            (addr,),
+            "SELECT public_key, private_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
+            (),
             |row| {
                 let pub_bytes: Vec<u8> = row.get(0)?;
                 let sec_bytes: Vec<u8> = row.get(1)?;
@@ -288,42 +282,50 @@ pub async fn store_self_keypair(
     keypair: &KeyPair,
     default: KeyPairUse,
 ) -> Result<()> {
-    context
+    let mut config_cache_lock = context.sql.config_cache.write().await;
+    let new_key_id = context
         .sql
         .transaction(|transaction| {
             let public_key = DcKey::to_bytes(&keypair.public);
             let secret_key = DcKey::to_bytes(&keypair.secret);
-            transaction
-                .execute(
-                    "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-                    (&public_key, &secret_key),
-                )
-                .context("failed to remove old use of key")?;
-            if default == KeyPairUse::Default {
-                transaction
-                    .execute("UPDATE keypairs SET is_default=0;", ())
-                    .context("failed to clear default")?;
-            }
+
             let is_default = match default {
-                KeyPairUse::Default => i32::from(true),
-                KeyPairUse::ReadOnly => i32::from(false),
+                KeyPairUse::Default => true,
+                KeyPairUse::ReadOnly => false,
             };
 
+            // `addr` and `is_default` written for compatibility with older versions,
+            // until new cores are rolled out everywhere.
+            // otherwise "add second device" or "backup" may break.
+            // moreover, this allows downgrades to the previous version.
+            // writing of `addr` and `is_default` can be removed ~ 2024-08
             let addr = keypair.addr.to_string();
-            let t = time();
-
             transaction
                 .execute(
-                    "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
-                VALUES (?,?,?,?,?);",
-                    (addr, is_default, &public_key, &secret_key, t),
+                    "INSERT OR REPLACE INTO keypairs (public_key, private_key, addr, is_default)
+                     VALUES (?,?,?,?)",
+                    (&public_key, &secret_key, addr, is_default),
                 )
-                .context("failed to insert keypair")?;
+                .context("Failed to insert keypair")?;
 
-            Ok(())
+            if is_default {
+                let new_key_id = transaction.last_insert_rowid();
+                transaction.execute(
+                    "INSERT OR REPLACE INTO config (keyname, value) VALUES ('key_id', ?)",
+                    (new_key_id,),
+                )?;
+                Ok(Some(new_key_id))
+            } else {
+                Ok(None)
+            }
         })
         .await?;
 
+    if let Some(new_key_id) = new_key_id {
+        // Update config cache if transaction succeeded and changed current default key.
+        config_cache_lock.insert("key_id".to_string(), Some(new_key_id.to_string()));
+    }
+
     Ok(())
 }
 

src/location.rs

@@ -139,8 +139,9 @@ impl Kml {
                 match chrono::NaiveDateTime::parse_from_str(&val, "%Y-%m-%dT%H:%M:%SZ") {
                     Ok(res) => {
                         self.curr.timestamp = res.timestamp();
-                        if self.curr.timestamp > time() {
-                            self.curr.timestamp = time();
+                        let now = time();
+                        if self.curr.timestamp > now {
+                            self.curr.timestamp = now;
                         }
                     }
                     Err(_err) => {
@@ -333,12 +334,13 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         return Ok(true);
     }
     let mut continue_streaming = false;
+    let now = time();
 
     let chats = context
         .sql
         .query_map(
             "SELECT id FROM chats WHERE locations_send_until>?;",
-            (time(),),
+            (now,),
             |row| row.get::<_, i32>(0),
             |chats| {
                 chats
@@ -356,7 +358,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                     latitude,
                     longitude,
                     accuracy,
-                    time(),
+                    now,
                     chat_id,
                     ContactId::SELF,
                 )).await.context("Failed to store location")?;
@@ -954,7 +956,7 @@ Content-Disposition: attachment; filename="location.kml"
         assert!(msg.chat_id == bob_chat_id);
         assert_eq!(msg.msg_ids.len(), 1);
 
-        let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.get(0).unwrap()).await?;
+        let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.first().unwrap()).await?;
         assert_eq!(bob_msg.chat_id, bob_chat_id);
         assert_eq!(bob_msg.viewtype, Viewtype::Image);
 

src/login_param.rs

@@ -14,8 +14,22 @@ use crate::socks::Socks5Config;
 #[repr(u32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
-    /// Same as AcceptInvalidCertificates unless overridden by
-    /// `strict_tls` setting in provider database.
+    /// Same as AcceptInvalidCertificates if stored in the database
+    /// as `configured_{imap,smtp}_certificate_checks`.
+    ///
+    /// Previous Delta Chat versions stored this in `configured_*`
+    /// if Automatic configuration
+    /// was selected, configuration with strict TLS checks failed
+    /// and configuration without strict TLS checks succeeded.
+    ///
+    /// Currently Delta Chat stores only
+    /// `Strict` or `AcceptInvalidCertificates` variants
+    /// in `configured_*` settings.
+    ///
+    /// `Automatic` in `{imap,smtp}_certificate_checks`
+    /// means that provider database setting should be taken.
+    /// If there is no provider database setting for certificate checks,
+    /// `Automatic` is the same as `Strict`.
     Automatic = 0,
 
     Strict = 1,

src/message.rs

@@ -757,7 +757,7 @@ impl Message {
         self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
-    /// Returns true if message is Auto-Submitted.
+    /// Returns true if message is auto-generated.
     pub fn is_bot(&self) -> bool {
         self.param.get_bool(Param::Bot).unwrap_or_default()
     }
@@ -1130,7 +1130,7 @@ impl Message {
     /// `References` header is not taken into account.
     pub async fn parent(&self, context: &Context) -> Result<Option<Message>> {
         if let Some(in_reply_to) = &self.in_reply_to {
-            if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? {
+            if let Some((msg_id, _ts_sent)) = rfc724_mid_exists(context, in_reply_to).await? {
                 let msg = Message::load_from_db(context, msg_id).await?;
                 return if msg.chat_id.is_trash() {
                     // If message is already moved to trash chat, pretend it does not exist.
@@ -1522,7 +1522,9 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
 
     if !msg_ids.is_empty() {
         // Run housekeeping to delete unused blobs.
-        context.set_config(Config::LastHousekeeping, None).await?;
+        context
+            .set_config_internal(Config::LastHousekeeping, None)
+            .await?;
     }
 
     // Interrupt Inbox loop to start message deletion and run housekeeping.
@@ -1550,7 +1552,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
     let old_last_msg_id = MsgId::new(context.get_config_u32(Config::LastMsgId).await?);
     let last_msg_id = msg_ids.iter().fold(&old_last_msg_id, std::cmp::max);
     context
-        .set_config_u32(Config::LastMsgId, last_msg_id.to_u32())
+        .set_config_internal(Config::LastMsgId, Some(&last_msg_id.to_u32().to_string()))
         .await?;
 
     let msgs = context
@@ -1663,9 +1665,17 @@ pub(crate) async fn update_msg_state(
     msg_id: MsgId,
     state: MessageState,
 ) -> Result<()> {
+    ensure!(state != MessageState::OutFailed, "use set_msg_failed()!");
+    let error_subst = match state >= MessageState::OutPending {
+        true => ", error=''",
+        false => "",
+    };
     context
         .sql
-        .execute("UPDATE msgs SET state=? WHERE id=?;", (state, msg_id))
+        .execute(
+            &format!("UPDATE msgs SET state=?1 {error_subst} WHERE id=?2 AND (?1!=?3 OR state<?3)"),
+            (state, msg_id, MessageState::OutDelivered),
+        )
         .await?;
     Ok(())
 }
@@ -1808,18 +1818,23 @@ pub async fn estimate_deletion_cnt(
     Ok(cnt)
 }
 
+/// See [`rfc724_mid_exists_and()`].
 pub(crate) async fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<Option<MsgId>> {
+) -> Result<Option<(MsgId, i64)>> {
     rfc724_mid_exists_and(context, rfc724_mid, "1").await
 }
 
+/// Returns [MsgId] and "sent" timestamp of the message with given `rfc724_mid` (Message-ID header)
+/// if it exists in the db.
+///
+/// @param cond SQL subexpression for filtering messages.
 pub(crate) async fn rfc724_mid_exists_and(
     context: &Context,
     rfc724_mid: &str,
     cond: &str,
-) -> Result<Option<MsgId>> {
+) -> Result<Option<(MsgId, i64)>> {
     let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>');
     if rfc724_mid.is_empty() {
         warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
@@ -1829,12 +1844,13 @@ pub(crate) async fn rfc724_mid_exists_and(
     let res = context
         .sql
         .query_row_optional(
-            &("SELECT id FROM msgs WHERE rfc724_mid=? AND ".to_string() + cond),
+            &("SELECT id, timestamp_sent FROM msgs WHERE rfc724_mid=? AND ".to_string() + cond),
             (rfc724_mid,),
             |row| {
                 let msg_id: MsgId = row.get(0)?;
+                let timestamp_sent: i64 = row.get(1)?;
 
-                Ok(msg_id)
+                Ok((msg_id, timestamp_sent))
             },
         )
         .await?;
@@ -1842,6 +1858,24 @@ pub(crate) async fn rfc724_mid_exists_and(
     Ok(res)
 }
 
+/// Given a list of Message-IDs, returns the latest message found in the database.
+///
+/// Only messages that are not in the trash chat are considered.
+pub(crate) async fn get_latest_by_rfc724_mids(
+    context: &Context,
+    mids: &[String],
+) -> Result<Option<Message>> {
+    for id in mids.iter().rev() {
+        if let Some((msg_id, _)) = rfc724_mid_exists(context, id).await? {
+            let msg = Message::load_from_db(context, msg_id).await?;
+            if msg.chat_id != DC_CHAT_ID_TRASH {
+                return Ok(Some(msg));
+            }
+        }
+    }
+    Ok(None)
+}
+
 /// How a message is primarily displayed.
 #[derive(
     Debug,

src/mimefactory.rs

@@ -1,5 +1,6 @@
 //! # MIME message production.
 
+use std::collections::HashSet;
 use std::convert::TryInto;
 
 use anyhow::{bail, ensure, Context as _, Result};
@@ -123,7 +124,8 @@ struct MessageHeaders {
     /// Headers that MUST NOT go into IMF header section.
     ///
     /// These are large headers which may hit the header section size limit on the server, such as
-    /// Chat-User-Avatar with a base64-encoded image inside.
+    /// Chat-User-Avatar with a base64-encoded image inside. Also there are headers duplicated here
+    /// that servers mess up with in the IMF header section, like Message-ID.
     pub hidden: Vec<Header>,
 }
 
@@ -275,7 +277,7 @@ impl<'a> MimeFactory<'a> {
     async fn peerstates_for_recipients(
         &self,
         context: &Context,
-    ) -> Result<Vec<(Option<Peerstate>, &str)>> {
+    ) -> Result<Vec<(Option<Peerstate>, String)>> {
         let self_addr = context.get_primary_self_addr().await?;
 
         let mut res = Vec::new();
@@ -284,7 +286,7 @@ impl<'a> MimeFactory<'a> {
             .iter()
             .filter(|(_, addr)| addr != &self_addr)
         {
-            res.push((Peerstate::from_addr(context, addr).await?, addr.as_str()));
+            res.push((Peerstate::from_addr(context, addr).await?, addr.clone()));
         }
 
         Ok(res)
@@ -349,7 +351,7 @@ impl<'a> MimeFactory<'a> {
                         .unwrap_or_default()
                 }
             }
-            Loaded::Mdn { .. } => true,
+            Loaded::Mdn { .. } => false,
         }
     }
 
@@ -517,6 +519,7 @@ impl<'a> MimeFactory<'a> {
         // <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.1.1>.
         let from_header = Header::new_with_value("From".into(), vec![from]).unwrap();
         headers.unprotected.push(from_header.clone());
+        headers.protected.push(from_header);
 
         if let Some(sender_displayname) = &self.sender_displayname {
             let sender =
@@ -558,24 +561,9 @@ impl<'a> MimeFactory<'a> {
             Loaded::Mdn { .. } => create_outgoing_rfc724_mid(None, &self.from_addr),
         };
         let rfc724_mid_headervalue = render_rfc724_mid(&rfc724_mid);
-
-        // Amazon's SMTP servers change the `Message-ID`, just as Outlook's SMTP servers do.
-        // Outlook's servers add an `X-Microsoft-Original-Message-ID` header with the original `Message-ID`,
-        // and when downloading messages we look for this header in order to correctly identify
-        // messages.
-        // Amazon's servers do not add such a header, so we just add it ourselves.
-        if let Some(server) = context.get_config(Config::ConfiguredSendServer).await? {
-            if server.ends_with(".amazonaws.com") {
-                headers.unprotected.push(Header::new(
-                    "X-Microsoft-Original-Message-ID".into(),
-                    rfc724_mid_headervalue.clone(),
-                ))
-            }
-        }
-
-        headers
-            .unprotected
-            .push(Header::new("Message-ID".into(), rfc724_mid_headervalue));
+        let rfc724_mid_header = Header::new("Message-ID".into(), rfc724_mid_headervalue);
+        headers.unprotected.push(rfc724_mid_header.clone());
+        headers.hidden.push(rfc724_mid_header);
 
         // Reply headers as in <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.2>.
         if !self.in_reply_to.is_empty() {
@@ -704,8 +692,6 @@ impl<'a> MimeFactory<'a> {
             )
         };
         let outer_message = if is_encrypted {
-            headers.protected.push(from_header);
-
             // Store protected headers in the inner message.
             let message = headers
                 .protected
@@ -782,30 +768,53 @@ impl<'a> MimeFactory<'a> {
                         .build(),
                 )
                 .header(("Subject".to_string(), "...".to_string()))
-        } else {
-            let message = if headers.hidden.is_empty() {
-                message
-            } else {
-                // Store hidden headers in the inner unencrypted message.
-                let message = headers
-                    .hidden
-                    .into_iter()
-                    .fold(message, |message, header| message.header(header));
+        } else if matches!(self.loaded, Loaded::Mdn { .. }) {
+            // Never add outer multipart/mixed wrapper to MDN
+            // as multipart/report Content-Type is used to recognize MDNs
+            // by Delta Chat receiver and Chatmail servers
+            // allowing them to be unencrypted and not contain Autocrypt header
+            // without resetting Autocrypt encryption or triggering Chatmail filter
+            // that normally only allows encrypted mails.
 
-                PartBuilder::new()
-                    .message_type(MimeMultipartType::Mixed)
-                    .child(message.build())
-            };
+            // Hidden headers are dropped.
 
             // Store protected headers in the outer message.
             let message = headers
                 .protected
+                .iter()
+                .fold(message, |message, header| message.header(header.clone()));
+
+            let protected: HashSet<Header> = HashSet::from_iter(headers.protected.into_iter());
+            for h in headers.unprotected.split_off(0) {
+                if !protected.contains(&h) {
+                    headers.unprotected.push(h);
+                }
+            }
+
+            message
+        } else {
+            // Store hidden headers in the inner unencrypted message.
+            let message = headers
+                .hidden
                 .into_iter()
                 .fold(message, |message, header| message.header(header));
+            let message = PartBuilder::new()
+                .message_type(MimeMultipartType::Mixed)
+                .child(message.build());
 
-            if self.should_skip_autocrypt()
-                || !context.get_config_bool(Config::SignUnencrypted).await?
-            {
+            // Store protected headers in the outer message.
+            let message = headers
+                .protected
+                .iter()
+                .fold(message, |message, header| message.header(header.clone()));
+
+            if skip_autocrypt || !context.get_config_bool(Config::SignUnencrypted).await? {
+                let protected: HashSet<Header> = HashSet::from_iter(headers.protected.into_iter());
+                for h in headers.unprotected.split_off(0) {
+                    if !protected.contains(&h) {
+                        headers.unprotected.push(h);
+                    }
+                }
                 message
             } else {
                 let message = message.header(get_content_type_directives_header());
@@ -908,6 +917,16 @@ impl<'a> MimeFactory<'a> {
         Ok(Some(part))
     }
 
+    fn add_message_text(&self, part: PartBuilder, mut text: String) -> PartBuilder {
+        // This is needed to protect from ESPs (such as gmx.at) doing their own Quoted-Printable
+        // encoding and thus breaking messages and signatures. It's unlikely that the reader uses a
+        // MUA not supporting Quoted-Printable encoding. And RFC 2646 "4.6" also recommends it for
+        // encrypted messages.
+        let part = part.header(("Content-Transfer-Encoding", "quoted-printable"));
+        text = quoted_printable::encode_to_str(text);
+        part.body(text)
+    }
+
     #[allow(clippy::cognitive_complexity)]
     async fn render_message(
         &mut self,
@@ -988,24 +1007,12 @@ impl<'a> MimeFactory<'a> {
                     {
                         info!(
                             context,
-                            "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>",
-                            "vg-member-added",
+                            "Sending secure-join message {:?}.", "vg-member-added",
                         );
                         headers.protected.push(Header::new(
                             "Secure-Join".to_string(),
                             "vg-member-added".to_string(),
                         ));
-                        // FIXME: Old clients require Secure-Join-Fingerprint header. Remove this
-                        // eventually.
-                        let fingerprint = Peerstate::from_addr(context, email_to_add)
-                            .await?
-                            .context("No peerstate found in db")?
-                            .public_key_fingerprint
-                            .context("No public key fingerprint in db for the member to add")?;
-                        headers.protected.push(Header::new(
-                            "Secure-Join-Fingerprint".into(),
-                            fingerprint.hex(),
-                        ));
                     }
                 }
                 SystemMessage::GroupNameChanged => {
@@ -1071,10 +1078,7 @@ impl<'a> MimeFactory<'a> {
                 let msg = &self.msg;
                 let step = msg.param.get(Param::Arg).unwrap_or_default();
                 if !step.is_empty() {
-                    info!(
-                        context,
-                        "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>", step,
-                    );
+                    info!(context, "Sending secure-join message {step:?}.");
                     headers
                         .protected
                         .push(Header::new("Secure-Join".into(), step.into()));
@@ -1220,13 +1224,11 @@ impl<'a> MimeFactory<'a> {
             footer
         );
 
-        // Message is sent as text/plain, with charset = utf-8
-        let mut main_part = PartBuilder::new()
-            .header((
-                "Content-Type".to_string(),
-                "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
-            ))
-            .body(message_text);
+        let mut main_part = PartBuilder::new().header((
+            "Content-Type",
+            "text/plain; charset=utf-8; format=flowed; delsp=no",
+        ));
+        main_part = self.add_message_text(main_part, message_text);
 
         if is_reaction {
             main_part = main_part.header(("Content-Disposition", "reaction"));
@@ -1353,15 +1355,12 @@ impl<'a> MimeFactory<'a> {
         };
         let p2 = stock_str::read_rcpt_mail_body(context, &p1).await;
         let message_text = format!("{}\r\n", format_flowed(&p2));
-        message = message.child(
-            PartBuilder::new()
-                .header((
-                    "Content-Type".to_string(),
-                    "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
-                ))
-                .body(message_text)
-                .build(),
-        );
+        let text_part = PartBuilder::new().header((
+            "Content-Type".to_string(),
+            "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+        ));
+        let text_part = self.add_message_text(text_part, message_text);
+        message = message.child(text_part.build());
 
         // second body part: machine-readable, always REQUIRED by RFC 6522
         let message_text2 = format!(
@@ -1577,6 +1576,7 @@ mod tests {
     use crate::mimeparser::MimeMessage;
     use crate::receive_imf::receive_imf;
     use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
+
     #[test]
     fn test_render_email_address() {
         let display_name = "ä space";
@@ -1832,6 +1832,37 @@ mod tests {
         assert_eq!("Re: Hello, Bob", mf.subject_str(&t).await.unwrap());
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mdn_create_encrypted() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+        bob.set_config_bool(Config::MdnsEnabled, true).await?;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
+
+        let rcvd = bob.recv_msg(&sent).await;
+        message::markseen_msgs(&bob, vec![rcvd.id]).await?;
+        let mimefactory = MimeFactory::from_mdn(&bob, &rcvd, vec![]).await?;
+        let rendered_msg = mimefactory.render(&bob).await?;
+
+        assert!(!rendered_msg.is_encrypted);
+
+        let rcvd = tcm.send_recv(&alice, &bob, "Heyho").await;
+        message::markseen_msgs(&bob, vec![rcvd.id]).await?;
+
+        let mimefactory = MimeFactory::from_mdn(&bob, &rcvd, vec![]).await?;
+        let rendered_msg = mimefactory.render(&bob).await?;
+
+        // When encrypted, the MDN should be encrypted as well
+        assert!(rendered_msg.is_encrypted);
+
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_subject_in_group() -> Result<()> {
         async fn send_msg_get_subject(
@@ -2163,33 +2194,39 @@ mod tests {
         let body = payload.next().unwrap();
 
         assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(outer.match_indices("Message-ID:").count(), 1);
         assert_eq!(outer.match_indices("Subject:").count(), 1);
         assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
         assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
 
         assert_eq!(inner.match_indices("text/plain").count(), 1);
+        assert_eq!(inner.match_indices("Message-ID:").count(), 1);
         assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 1);
         assert_eq!(inner.match_indices("Subject:").count(), 0);
+        assert_eq!(inner.match_indices("quoted-printable").count(), 1);
 
         assert_eq!(body.match_indices("this is the text!").count(), 1);
 
         // if another message is sent, that one must not contain the avatar
-        // and no artificial multipart/mixed nesting
         let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(2, "\r\n\r\n");
+        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
         let outer = payload.next().unwrap();
+        let inner = payload.next().unwrap();
         let body = payload.next().unwrap();
 
-        assert_eq!(outer.match_indices("text/plain").count(), 1);
+        assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(outer.match_indices("Message-ID:").count(), 1);
         assert_eq!(outer.match_indices("Subject:").count(), 1);
         assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
-        assert_eq!(outer.match_indices("multipart/mixed").count(), 0);
         assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
 
+        assert_eq!(inner.match_indices("text/plain").count(), 1);
+        assert_eq!(inner.match_indices("Message-ID:").count(), 1);
+        assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 0);
+        assert_eq!(inner.match_indices("Subject:").count(), 0);
+        assert_eq!(inner.match_indices("quoted-printable").count(), 1);
+
         assert_eq!(body.match_indices("this is the text!").count(), 1);
-        assert_eq!(body.match_indices("text/plain").count(), 0);
-        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
-        assert_eq!(body.match_indices("Subject:").count(), 0);
 
         Ok(())
     }
@@ -2220,6 +2257,8 @@ mod tests {
 
         let part = payload.next().unwrap();
         assert_eq!(part.match_indices("multipart/signed").count(), 1);
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
         assert_eq!(part.match_indices("Subject:").count(), 0);
         assert_eq!(part.match_indices("Autocrypt:").count(), 1);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
@@ -2230,14 +2269,19 @@ mod tests {
                 .count(),
             1
         );
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 0);
         assert_eq!(part.match_indices("Subject:").count(), 1);
         assert_eq!(part.match_indices("Autocrypt:").count(), 0);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
         let part = payload.next().unwrap();
         assert_eq!(part.match_indices("text/plain").count(), 1);
+        assert_eq!(part.match_indices("From:").count(), 0);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1);
         assert_eq!(part.match_indices("Subject:").count(), 0);
+        assert_eq!(part.match_indices("quoted-printable").count(), 1);
 
         let body = payload.next().unwrap();
         assert_eq!(body.match_indices("this is the text!").count(), 1);
@@ -2256,28 +2300,39 @@ mod tests {
             .is_some());
 
         // if another message is sent, that one must not contain the avatar
-        // and no artificial multipart/mixed nesting
         let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
+        let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n");
 
         let part = payload.next().unwrap();
         assert_eq!(part.match_indices("multipart/signed").count(), 1);
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
         assert_eq!(part.match_indices("Subject:").count(), 0);
         assert_eq!(part.match_indices("Autocrypt:").count(), 1);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
         let part = payload.next().unwrap();
-        assert_eq!(part.match_indices("text/plain").count(), 1);
+        assert_eq!(
+            part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+                .count(),
+            1
+        );
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 0);
         assert_eq!(part.match_indices("Subject:").count(), 1);
         assert_eq!(part.match_indices("Autocrypt:").count(), 0);
-        assert_eq!(part.match_indices("multipart/mixed").count(), 0);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
+        let part = payload.next().unwrap();
+        assert_eq!(part.match_indices("text/plain").count(), 1);
+        assert_eq!(body.match_indices("From:").count(), 0);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
+        assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+        assert_eq!(part.match_indices("Subject:").count(), 0);
+        assert_eq!(part.match_indices("quoted-printable").count(), 1);
+
         let body = payload.next().unwrap();
         assert_eq!(body.match_indices("this is the text!").count(), 1);
-        assert_eq!(body.match_indices("text/plain").count(), 0);
-        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
-        assert_eq!(body.match_indices("Subject:").count(), 0);
 
         bob.recv_msg(&sent_msg).await;
         let alice_contact = Contact::get_by_id(&bob.ctx, alice_id).await.unwrap();

src/mimeparser.rs

@@ -1,5 +1,6 @@
 //! # MIME message parsing module.
 
+use std::cmp::min;
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::path::Path;
@@ -37,8 +38,8 @@ use crate::simplify::{simplify, SimplifiedText};
 use crate::stock_str;
 use crate::sync::SyncItems;
 use crate::tools::{
-    create_smeared_timestamp, get_filemeta, parse_receive_headers, strip_rtlo_characters,
-    truncate_by_lines,
+    create_smeared_timestamp, get_filemeta, parse_receive_headers, smeared_time,
+    strip_rtlo_characters, truncate_by_lines,
 };
 use crate::{location, tools};
 
@@ -68,6 +69,8 @@ pub(crate) struct MimeMessage {
     /// Whether the From address was repeated in the signed part
     /// (and we know that the signer intended to send from this address)
     pub from_is_signed: bool,
+    /// Whether the message is incoming or outgoing (self-sent).
+    pub incoming: bool,
     /// The List-Post address is only set for mailing lists. Users can send
     /// messages to this address to post them to the list.
     pub list_post: Option<String>,
@@ -81,9 +84,10 @@ pub(crate) struct MimeMessage {
     /// If a message is not encrypted or the signature is not valid,
     /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
-    /// The set of mail recipient addresses for which gossip headers were applied, regardless of
-    /// whether they modified any peerstates.
-    pub gossiped_addr: HashSet<String>,
+    /// The mail recipient addresses for which gossip headers were applied
+    /// and their respective gossiped keys,
+    /// regardless of whether they modified any peerstates.
+    pub gossiped_keys: HashMap<String, SignedPublicKey>,
 
     /// True if the message is a forwarded message.
     pub is_forwarded: bool,
@@ -116,8 +120,14 @@ pub(crate) struct MimeMessage {
     /// Hop info for debugging.
     pub(crate) hop_info: String,
 
-    /// Whether the contact sending this should be marked as bot.
-    pub(crate) is_bot: bool,
+    /// Whether the contact sending this should be marked as bot or non-bot.
+    pub(crate) is_bot: Option<bool>,
+
+    /// When the message was received, in secs since epoch.
+    pub(crate) timestamp_rcvd: i64,
+    /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized
+    /// clocks, but not too much.
+    pub(crate) timestamp_sent: i64,
 }
 
 #[derive(Debug, PartialEq)]
@@ -169,6 +179,10 @@ pub enum SystemMessage {
     /// "%1$s sent a message from another device."
     ChatProtectionDisabled = 12,
 
+    /// Message can't be sent because of `Invalid unencrypted mail to <>`
+    /// which is sent by chatmail servers.
+    InvalidUnencryptedMail = 13,
+
     /// Self-sent-message that contains only json used for multi-device-sync;
     /// if possible, we attach that to other messages as for locations.
     MultiDeviceSync = 20,
@@ -196,11 +210,12 @@ impl MimeMessage {
     ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
-        let message_time = mail
+        let timestamp_rcvd = smeared_time(context);
+        let timestamp_sent = mail
             .headers
             .get_header_value(HeaderDef::Date)
             .and_then(|v| mailparse::dateparse(&v).ok())
-            .unwrap_or_default();
+            .map_or(timestamp_rcvd, |value| min(value, timestamp_rcvd + 60));
         let mut hop_info = parse_receive_headers(&mail.get_headers());
 
         let mut headers = Default::default();
@@ -267,18 +282,18 @@ impl MimeMessage {
         let private_keyring = load_self_secret_keyring(context).await?;
 
         let mut decryption_info =
-            prepare_decryption(context, &mail, &from.addr, message_time).await?;
+            prepare_decryption(context, &mail, &from.addr, timestamp_sent).await?;
 
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
-        let mut gossiped_addr = Default::default();
+        let mut gossiped_keys = Default::default();
         let mut from_is_signed = false;
         hop_info += "\n\n";
         hop_info += &decryption_info.dkim_results.to_string();
 
         let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref());
         let (mail, mut signatures, encrypted) = match tokio::task::block_in_place(|| {
-            try_decrypt(context, &mail, &private_keyring, &public_keyring)
+            try_decrypt(&mail, &private_keyring, &public_keyring)
         }) {
             Ok(Some((raw, signatures))) => {
                 mail_raw = raw;
@@ -311,9 +326,9 @@ impl MimeMessage {
                 // but only if the mail was correctly signed. Probably it's ok to not require
                 // encryption here, but let's follow the standard.
                 let gossip_headers = mail.headers.get_all_values("Autocrypt-Gossip");
-                gossiped_addr = update_gossip_peerstates(
+                gossiped_keys = update_gossip_peerstates(
                     context,
-                    message_time,
+                    timestamp_sent,
                     &from.addr,
                     &recipients,
                     gossip_headers,
@@ -364,12 +379,12 @@ impl MimeMessage {
 
             // If it is not a read receipt, degrade encryption.
             if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) {
-                if message_time > peerstate.last_seen_autocrypt
+                if timestamp_sent > peerstate.last_seen_autocrypt
                     && mail.ctype.mimetype != "multipart/report"
                 // Disallowing keychanges is disabled for now:
                 // && decryption_info.dkim_results.allow_keychange
                 {
-                    peerstate.degrade_encryption(message_time);
+                    peerstate.degrade_encryption(timestamp_sent);
                 }
             }
         }
@@ -383,9 +398,7 @@ impl MimeMessage {
             }
         }
 
-        // Auto-submitted is also set by holiday-notices so we also check `chat-version`
-        let is_bot = headers.contains_key("auto-submitted") && headers.contains_key("chat-version");
-
+        let incoming = !context.is_self_addr(&from.addr).await?;
         let mut parser = MimeMessage {
             parts: Vec::new(),
             headers,
@@ -393,13 +406,14 @@ impl MimeMessage {
             list_post,
             from,
             from_is_signed,
+            incoming,
             chat_disposition_notification_to,
             decryption_info,
             decrypting_failed: mail.is_err(),
 
             // only non-empty if it was a valid autocrypt message
             signatures,
-            gossiped_addr,
+            gossiped_keys,
             is_forwarded: false,
             mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
@@ -414,7 +428,9 @@ impl MimeMessage {
             is_mime_modified: false,
             decoded_data: Vec::new(),
             hop_info,
-            is_bot,
+            is_bot: None,
+            timestamp_rcvd,
+            timestamp_sent,
         };
 
         match partial {
@@ -445,6 +461,13 @@ impl MimeMessage {
             },
         };
 
+        if parser.mdn_reports.is_empty() {
+            // "Auto-Submitted" is also set by holiday-notices so we also check "chat-version".
+            let is_bot = parser.headers.get("auto-submitted")
+                == Some(&"auto-generated".to_string())
+                && parser.headers.contains_key("chat-version");
+            parser.is_bot = Some(is_bot);
+        }
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
@@ -461,20 +484,6 @@ impl MimeMessage {
             parser.decoded_data = mail_raw;
         }
 
-        crate::peerstate::maybe_do_aeap_transition(context, &mut parser).await?;
-        if let Some(peerstate) = &parser.decryption_info.peerstate {
-            peerstate
-                .handle_fingerprint_change(context, message_time)
-                .await?;
-            // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
-            // of an error. Otherwise we don't save peerstate until get here to reduce the number of
-            // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
-            // successfully.
-            if peerstate.prefer_encrypt != EncryptPreference::Mutual {
-                peerstate.save_to_db(&context.sql).await?;
-            }
-        }
-
         Ok(parser)
     }
 
@@ -694,10 +703,10 @@ impl MimeMessage {
                 }
             }
 
-            self.parts.push(part);
+            self.do_add_single_part(part);
         }
 
-        if self.headers.contains_key("auto-submitted") {
+        if self.is_bot == Some(true) {
             for part in &mut self.parts {
                 part.param.set(Param::Bot, "1");
             }
@@ -913,7 +922,7 @@ impl MimeMessage {
                 skip the rest.  (see
                 <https://k9mail.app/2016/11/24/OpenPGP-Considerations-Part-I.html>
                 for background information why we use encrypted+signed) */
-                if let Some(first) = mail.subparts.get(0) {
+                if let Some(first) = mail.subparts.first() {
                     any_part_added = self
                         .parse_mime_recursive(context, first, is_related)
                         .await?;
@@ -969,10 +978,13 @@ impl MimeMessage {
                             }
                         }
                         Some(_) => {
-                            if let Some(first) = mail.subparts.get(0) {
-                                any_part_added = self
-                                    .parse_mime_recursive(context, first, is_related)
-                                    .await?;
+                            for cur_data in &mail.subparts {
+                                if self
+                                    .parse_mime_recursive(context, cur_data, is_related)
+                                    .await?
+                                {
+                                    any_part_added = true;
+                                }
                             }
                         }
                     }
@@ -1364,6 +1376,15 @@ impl MimeMessage {
         self.get_mailinglist_header().is_some()
     }
 
+    /// Detects Schleuder mailing list by List-Help header.
+    pub(crate) fn is_schleuder_message(&self) -> bool {
+        if let Some(list_help) = self.get_header(HeaderDef::ListHelp) {
+            list_help == "<https://schleuder.org/>"
+        } else {
+            false
+        }
+    }
+
     pub fn replace_msg_by_error(&mut self, error_msg: &str) {
         self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
@@ -1592,8 +1613,12 @@ impl MimeMessage {
     /// eg. when the user-edited-content is html.
     /// As these footers would appear as repeated, separate text-bubbles,
     /// we remove them.
+    ///
+    /// We make an exception for Schleuder mailing lists
+    /// because they typically create messages with two text parts,
+    /// one for headers and one for the actual contents.
     fn maybe_remove_inline_mailinglist_footer(&mut self) {
-        if self.is_mailinglist_message() {
+        if self.is_mailinglist_message() && !self.is_schleuder_message() {
             let text_part_cnt = self
                 .parts
                 .iter()
@@ -1644,13 +1669,7 @@ impl MimeMessage {
     /// Handle reports
     /// (MDNs = Message Disposition Notification, the message was read
     /// and NDNs = Non delivery notification, the message could not be delivered)
-    pub async fn handle_reports(
-        &self,
-        context: &Context,
-        from_id: ContactId,
-        sent_timestamp: i64,
-        parts: &[Part],
-    ) {
+    pub async fn handle_reports(&self, context: &Context, from_id: ContactId, parts: &[Part]) {
         for report in &self.mdn_reports {
             for original_message_id in report
                 .original_message_id
@@ -1658,7 +1677,7 @@ impl MimeMessage {
                 .chain(&report.additional_message_ids)
             {
                 if let Err(err) =
-                    handle_mdn(context, from_id, original_message_id, sent_timestamp).await
+                    handle_mdn(context, from_id, original_message_id, self.timestamp_sent).await
                 {
                     warn!(context, "Could not handle MDN: {err:#}.");
                 }
@@ -1709,9 +1728,9 @@ async fn update_gossip_peerstates(
     from: &str,
     recipients: &[SingleInfo],
     gossip_headers: Vec<String>,
-) -> Result<HashSet<String>> {
+) -> Result<HashMap<String, SignedPublicKey>> {
     // XXX split the parsing from the modification part
-    let mut gossiped_addr: HashSet<String> = Default::default();
+    let mut gossiped_keys: HashMap<String, SignedPublicKey> = Default::default();
 
     for value in &gossip_headers {
         let header = match value.parse::<Aheader>() {
@@ -1755,10 +1774,10 @@ async fn update_gossip_peerstates(
             .handle_fingerprint_change(context, message_time)
             .await?;
 
-        gossiped_addr.insert(header.addr.to_lowercase());
+        gossiped_keys.insert(header.addr.to_lowercase(), header.public_key);
     }
 
-    Ok(gossiped_addr)
+    Ok(gossiped_keys)
 }
 
 /// Message Disposition Notification (RFC 8098)
@@ -2217,6 +2236,7 @@ mod tests {
         message::{Message, MessageState, MessengerMessage},
         receive_imf::receive_imf,
         test_utils::TestContext,
+        tools::time,
     };
 
     impl AvatarAction {
@@ -2725,6 +2745,7 @@ Chat-Version: 1.0\n\
 Message-ID: <bar@example.org>\n\
 To: Alice <alice@example.org>\n\
 From: Bob <bob@example.org>\n\
+Auto-Submitted: auto-replied\n\
 Content-Type: multipart/report; report-type=disposition-notification;\n\t\
 boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
 \n\
@@ -2760,6 +2781,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.mdn_reports.len(), 1);
+        assert_eq!(message.is_bot, None);
     }
 
     /// Test parsing multiple MDNs combined in a single message.
@@ -3791,4 +3813,76 @@ Content-Disposition: reaction\n\
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_schleuder() -> Result<()> {
+        let context = TestContext::new_alice().await;
+        let raw = include_bytes!("../test-data/message/schleuder.eml");
+
+        let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
+            .await
+            .unwrap();
+        assert_eq!(msg.parts.len(), 2);
+
+        // Header part.
+        assert_eq!(msg.parts[0].typ, Viewtype::Text);
+
+        // Actual contents part.
+        assert_eq!(msg.parts[1].typ, Viewtype::Text);
+        assert_eq!(msg.parts[1].msg, "hello,\nbye");
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_tlsrpt() -> Result<()> {
+        let context = TestContext::new_alice().await;
+        let raw = include_bytes!("../test-data/message/tlsrpt.eml");
+
+        let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
+            .await
+            .unwrap();
+        assert_eq!(msg.parts.len(), 1);
+
+        assert_eq!(msg.parts[0].typ, Viewtype::File);
+        assert_eq!(msg.parts[0].msg, "Report Domain: nine.testrun.org Submitter: google.com Report-ID: <2024.01.20T00.00.00Z+nine.testrun.org@google.com> – This is an aggregate TLS report from google.com");
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_time_in_future() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let beginning_time = time();
+
+        // Receive a message with a date far in the future (year 3004)
+        // I'm just going to assume that no one uses this code after the year 3000
+        let mime_message = MimeMessage::from_bytes(
+            &alice,
+            b"To: alice@example.org\n\
+              From: bob@example.net\n\
+              Date: Today, 29 February 3004 00:00:10 -800\n\
+              Message-ID: 56789@example.net\n\
+              Subject: Meeting\n\
+              Mime-Version: 1.0 (1.0)\n\
+              Content-Type: text/plain; charset=utf-8\n\
+              \n\
+              Hi",
+            None,
+        )
+        .await?;
+
+        // We do allow the time to be in the future a bit (because of unsynchronized clocks),
+        // but only 60 seconds:
+        assert!(mime_message.decryption_info.message_time <= time() + 60);
+        assert!(mime_message.decryption_info.message_time >= beginning_time + 60);
+        assert_eq!(
+            mime_message.decryption_info.message_time,
+            mime_message.timestamp_sent
+        );
+        assert!(mime_message.timestamp_rcvd <= time());
+
+        Ok(())
+    }
 }

src/param.rs

@@ -65,7 +65,7 @@ pub enum Param {
     /// For Messages: the message is a reaction.
     Reaction = b'x',
 
-    /// For Messages: a message with Auto-Submitted header ("bot").
+    /// For Messages: a message with "Auto-Submitted: auto-generated" header ("bot").
     Bot = b'b',
 
     /// For Messages: unset or 0=not forwarded,
@@ -84,7 +84,7 @@ pub enum Param {
     /// For Messages
     Arg2 = b'F',
 
-    /// For Messages
+    /// `Secure-Join-Fingerprint` header for `{vc,vg}-request-with-auth` messages.
     Arg3 = b'G',
 
     /// For Messages

src/peerstate.rs

@@ -6,6 +6,7 @@ use num_traits::FromPrimitive;
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::chat::{self, Chat};
 use crate::chatlist::Chatlist;
+use crate::config::Config;
 use crate::constants::Chattype;
 use crate::contact::{addr_cmp, Contact, ContactAddress, Origin};
 use crate::context::Context;
@@ -83,6 +84,10 @@ pub struct Peerstate {
     /// The address that introduced secondary verified key.
     pub secondary_verifier: Option<String>,
 
+    /// Row ID of the key in the `keypairs` table
+    /// that we think the peer knows as verified.
+    pub backward_verified_key_id: Option<i64>,
+
     /// True if it was detected
     /// that the fingerprint of the key used in chats with
     /// opportunistic encryption was changed after Peerstate creation.
@@ -92,13 +97,28 @@ pub struct Peerstate {
 impl Peerstate {
     /// Creates a peerstate from the `Autocrypt` header.
     pub fn from_header(header: &Aheader, message_time: i64) -> Self {
+        Self::from_public_key(
+            &header.addr,
+            message_time,
+            header.prefer_encrypt,
+            &header.public_key,
+        )
+    }
+
+    /// Creates a peerstate from the given public key.
+    pub fn from_public_key(
+        addr: &str,
+        last_seen: i64,
+        prefer_encrypt: EncryptPreference,
+        public_key: &SignedPublicKey,
+    ) -> Self {
         Peerstate {
-            addr: header.addr.clone(),
-            last_seen: message_time,
-            last_seen_autocrypt: message_time,
-            prefer_encrypt: header.prefer_encrypt,
-            public_key: Some(header.public_key.clone()),
-            public_key_fingerprint: Some(header.public_key.fingerprint()),
+            addr: addr.to_string(),
+            last_seen,
+            last_seen_autocrypt: last_seen,
+            prefer_encrypt,
+            public_key: Some(public_key.clone()),
+            public_key_fingerprint: Some(public_key.fingerprint()),
             gossip_key: None,
             gossip_key_fingerprint: None,
             gossip_timestamp: 0,
@@ -108,6 +128,7 @@ impl Peerstate {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         }
     }
@@ -137,6 +158,7 @@ impl Peerstate {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         }
     }
@@ -148,7 +170,8 @@ impl Peerstate {
                      verified_key, verified_key_fingerprint, \
                      verifier, \
                      secondary_verified_key, secondary_verified_key_fingerprint, \
-                     secondary_verifier \
+                     secondary_verifier, \
+                     backward_verified_key_id \
                      FROM acpeerstates \
                      WHERE addr=? COLLATE NOCASE LIMIT 1;";
         Self::from_stmt(context, query, (addr,)).await
@@ -164,7 +187,8 @@ impl Peerstate {
                      verified_key, verified_key_fingerprint, \
                      verifier, \
                      secondary_verified_key, secondary_verified_key_fingerprint, \
-                     secondary_verifier \
+                     secondary_verifier, \
+                     backward_verified_key_id \
                      FROM acpeerstates  \
                      WHERE public_key_fingerprint=? \
                      OR gossip_key_fingerprint=? \
@@ -187,7 +211,8 @@ impl Peerstate {
                      verified_key, verified_key_fingerprint, \
                      verifier, \
                      secondary_verified_key, secondary_verified_key_fingerprint, \
-                     secondary_verifier \
+                     secondary_verifier, \
+                     backward_verified_key_id \
                      FROM acpeerstates  \
                      WHERE verified_key_fingerprint=? \
                      OR addr=? COLLATE NOCASE \
@@ -255,6 +280,7 @@ impl Peerstate {
                         let secondary_verifier: Option<String> = row.get("secondary_verifier")?;
                         secondary_verifier.filter(|s| !s.is_empty())
                     },
+                    backward_verified_key_id: row.get("backward_verified_key_id")?,
                     fingerprint_changed: false,
                 };
 
@@ -435,59 +461,55 @@ impl Peerstate {
         verified.is_some() && verified == self.peek_key_fingerprint(false)
     }
 
-    /// Set this peerstate to verified
-    /// Make sure to call `self.save_to_db` to save these changes
+    pub(crate) async fn is_backward_verified(&self, context: &Context) -> Result<bool> {
+        let Some(backward_verified_key_id) = self.backward_verified_key_id else {
+            return Ok(false);
+        };
+
+        let self_key_id = context.get_config_i64(Config::KeyId).await?;
+
+        let backward_verified = backward_verified_key_id == self_key_id;
+        Ok(backward_verified)
+    }
+
+    /// Set this peerstate to verified;
+    /// make sure to call `self.save_to_db` to save these changes.
+    ///
     /// Params:
-    /// verifier:
+    ///
+    /// * key: The new verified key.
+    /// * fingerprint: Only set to verified if the key's fingerprint matches this.
+    /// * verifier:
     ///   The address which introduces the given contact.
     ///   If we are verifying the contact, use that contacts address.
     pub fn set_verified(
         &mut self,
-        which_key: PeerstateKeyType,
+        key: SignedPublicKey,
         fingerprint: Fingerprint,
         verifier: String,
     ) -> Result<()> {
-        match which_key {
-            PeerstateKeyType::PublicKey => {
-                if self.public_key_fingerprint.is_some()
-                    && self.public_key_fingerprint.as_ref().unwrap() == &fingerprint
-                {
-                    self.verified_key = self.public_key.clone();
-                    self.verified_key_fingerprint = Some(fingerprint);
-                    self.verifier = Some(verifier);
-                    Ok(())
-                } else {
-                    Err(Error::msg(format!(
-                        "{fingerprint} is not peer's public key fingerprint",
-                    )))
-                }
-            }
-            PeerstateKeyType::GossipKey => {
-                if self.gossip_key_fingerprint.is_some()
-                    && self.gossip_key_fingerprint.as_ref().unwrap() == &fingerprint
-                {
-                    self.verified_key = self.gossip_key.clone();
-                    self.verified_key_fingerprint = Some(fingerprint);
-                    self.verifier = Some(verifier);
-                    Ok(())
-                } else {
-                    Err(Error::msg(format!(
-                        "{fingerprint} is not peer's gossip key fingerprint",
-                    )))
-                }
-            }
+        if key.fingerprint() == fingerprint {
+            self.verified_key = Some(key);
+            self.verified_key_fingerprint = Some(fingerprint);
+            self.verifier = Some(verifier);
+            Ok(())
+        } else {
+            Err(Error::msg(format!(
+                "{fingerprint} is not peer's key fingerprint",
+            )))
         }
     }
 
-    /// Sets current gossiped key as the secondary verified key.
+    /// Sets the gossiped key as the secondary verified key.
     ///
     /// If gossiped key is the same as the current verified key,
     /// do nothing to avoid overwriting secondary verified key
     /// which may be different.
-    pub fn set_secondary_verified_key_from_gossip(&mut self, verifier: String) {
-        if self.gossip_key_fingerprint != self.verified_key_fingerprint {
-            self.secondary_verified_key = self.gossip_key.clone();
-            self.secondary_verified_key_fingerprint = self.gossip_key_fingerprint.clone();
+    pub fn set_secondary_verified_key(&mut self, gossip_key: SignedPublicKey, verifier: String) {
+        let fingerprint = gossip_key.fingerprint();
+        if self.verified_key_fingerprint.as_ref() != Some(&fingerprint) {
+            self.secondary_verified_key = Some(gossip_key);
+            self.secondary_verified_key_fingerprint = Some(fingerprint);
             self.secondary_verifier = Some(verifier);
         }
     }
@@ -510,8 +532,9 @@ impl Peerstate {
                 secondary_verified_key,
                 secondary_verified_key_fingerprint,
                 secondary_verifier,
+                backward_verified_key_id,
                 addr)
-                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                 ON CONFLICT (addr)
                 DO UPDATE SET
                   last_seen = excluded.last_seen,
@@ -527,7 +550,8 @@ impl Peerstate {
                   verifier = excluded.verifier,
                   secondary_verified_key = excluded.secondary_verified_key,
                   secondary_verified_key_fingerprint = excluded.secondary_verified_key_fingerprint,
-                  secondary_verifier = excluded.secondary_verifier",
+                  secondary_verifier = excluded.secondary_verifier,
+                  backward_verified_key_id = excluded.backward_verified_key_id",
             (
                 self.last_seen,
                 self.last_seen_autocrypt,
@@ -545,6 +569,7 @@ impl Peerstate {
                     .as_ref()
                     .map(|fp| fp.hex()),
                 self.secondary_verifier.as_deref().unwrap_or(""),
+                self.backward_verified_key_id,
                 &self.addr,
             ),
         )
@@ -696,44 +721,46 @@ pub(crate) async fn maybe_do_aeap_transition(
     mime_parser: &mut crate::mimeparser::MimeMessage,
 ) -> Result<()> {
     let info = &mime_parser.decryption_info;
-    if let Some(peerstate) = &info.peerstate {
-        // If the from addr is different from the peerstate address we know,
-        // we may want to do an AEAP transition.
-        if !addr_cmp(&peerstate.addr, &mime_parser.from.addr)
-                // Check if it's a chat message; we do this to avoid
-                // some accidental transitions if someone writes from multiple
-                // addresses with an MUA.
-                && mime_parser.has_chat_version()
-                // Check if the message is signed correctly.
-                // Although checking `from_is_signed` below is sufficient, let's play it safe.
-                && !mime_parser.signatures.is_empty()
-                // Check if the From: address was also in the signed part of the email.
-                // Without this check, an attacker could replay a message from Alice 
-                // to Bob. Then Bob's device would do an AEAP transition from Alice's
-                // to the attacker's address, allowing for easier phishing.
-                && mime_parser.from_is_signed
-                && info.message_time > peerstate.last_seen
-        {
-            let info = &mut mime_parser.decryption_info;
-            let peerstate = info.peerstate.as_mut().context("no peerstate??")?;
-            // Add info messages to chats with this (verified) contact
-            //
-            peerstate
-                .handle_setup_change(
-                    context,
-                    info.message_time,
-                    PeerstateChange::Aeap(info.from.clone()),
-                )
-                .await?;
+    let Some(peerstate) = &info.peerstate else {
+        return Ok(());
+    };
 
-            peerstate.addr = info.from.clone();
-            let header = info.autocrypt_header.as_ref().context(
-                "Internal error: Tried to do an AEAP transition without an autocrypt header??",
-            )?;
-            peerstate.apply_header(header, info.message_time);
+    // If the from addr is different from the peerstate address we know,
+    // we may want to do an AEAP transition.
+    if !addr_cmp(&peerstate.addr, &mime_parser.from.addr)
+            // Check if it's a chat message; we do this to avoid
+            // some accidental transitions if someone writes from multiple
+            // addresses with an MUA.
+            && mime_parser.has_chat_version()
+            // Check if the message is signed correctly.
+            // Although checking `from_is_signed` below is sufficient, let's play it safe.
+            && !mime_parser.signatures.is_empty()
+            // Check if the From: address was also in the signed part of the email.
+            // Without this check, an attacker could replay a message from Alice
+            // to Bob. Then Bob's device would do an AEAP transition from Alice's
+            // to the attacker's address, allowing for easier phishing.
+            && mime_parser.from_is_signed
+            && info.message_time > peerstate.last_seen
+    {
+        let info = &mut mime_parser.decryption_info;
+        let peerstate = info.peerstate.as_mut().context("no peerstate??")?;
+        // Add info messages to chats with this (verified) contact
+        //
+        peerstate
+            .handle_setup_change(
+                context,
+                info.message_time,
+                PeerstateChange::Aeap(info.from.clone()),
+            )
+            .await?;
 
-            peerstate.save_to_db(&context.sql).await?;
-        }
+        peerstate.addr = info.from.clone();
+        let header = info.autocrypt_header.as_ref().context(
+            "Internal error: Tried to do an AEAP transition without an autocrypt header??",
+        )?;
+        peerstate.apply_header(header, info.message_time);
+
+        peerstate.save_to_db(&context.sql).await?;
     }
 
     Ok(())
@@ -804,6 +831,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 
@@ -847,6 +875,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 
@@ -883,6 +912,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 
@@ -949,6 +979,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 

src/provider.rs

@@ -215,8 +215,18 @@ pub async fn get_provider_info(
 
 /// Finds a provider in offline database based on domain.
 pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
-    if let Some(provider) = PROVIDER_DATA.get(domain.to_lowercase().as_str()) {
-        return Some(*provider);
+    let domain = domain.to_lowercase();
+    for (pattern, provider) in PROVIDER_DATA {
+        if let Some(suffix) = pattern.strip_prefix('*') {
+            // Wildcard domain pattern.
+            //
+            // For example, `suffix` is ".hermes.radio" for "*.hermes.radio" pattern.
+            if domain.ends_with(suffix) {
+                return Some(provider);
+            }
+        } else if pattern == domain {
+            return Some(provider);
+        }
     }
 
     None
@@ -226,33 +236,42 @@ pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
 ///
 /// For security reasons, only Gmail can be configured this way.
 pub async fn get_provider_by_mx(context: &Context, domain: &str) -> Option<&'static Provider> {
-    if let Ok(resolver) = get_resolver() {
-        let mut fqdn: String = domain.to_string();
-        if !fqdn.ends_with('.') {
-            fqdn.push('.');
+    let Ok(resolver) = get_resolver() else {
+        warn!(context, "Cannot get a resolver to check MX records.");
+        return None;
+    };
+
+    let mut fqdn: String = domain.to_string();
+    if !fqdn.ends_with('.') {
+        fqdn.push('.');
+    }
+
+    let Ok(mx_domains) = resolver.mx_lookup(fqdn).await else {
+        warn!(context, "Cannot resolve MX records for {domain:?}.");
+        return None;
+    };
+
+    for (provider_domain_pattern, provider) in PROVIDER_DATA {
+        if provider.id != "gmail" {
+            // MX lookup is limited to Gmail for security reasons
+            continue;
         }
 
-        if let Ok(mx_domains) = resolver.mx_lookup(fqdn).await {
-            for (provider_domain, provider) in &*PROVIDER_DATA {
-                if provider.id != "gmail" {
-                    // MX lookup is limited to Gmail for security reasons
-                    continue;
-                }
+        if provider_domain_pattern.starts_with('*') {
+            // Skip wildcard patterns.
+            continue;
+        }
 
-                let provider_fqdn = provider_domain.to_string() + ".";
-                let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
+        let provider_fqdn = provider_domain_pattern.to_string() + ".";
+        let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
 
-                for mx_domain in mx_domains.iter() {
-                    let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
+        for mx_domain in mx_domains.iter() {
+            let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
 
-                    if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
-                        return Some(provider);
-                    }
-                }
+            if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
+                return Some(provider);
             }
         }
-    } else {
-        warn!(context, "cannot get a resolver to check MX records.");
     }
 
     None

src/provider/data.rs

@@ -593,7 +593,7 @@ static P_GMX_NET: Provider = Provider {
     oauth2_authorizer: None,
 };
 
-// hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, ec.hermes.radio, ec1.hermes.radio, ec2.hermes.radio, ec3.hermes.radio, ec4.hermes.radio, ec5.hermes.radio, ec6.hermes.radio, ec7.hermes.radio, ec8.hermes.radio, ec9.hermes.radio, ec10.hermes.radio, ec11.hermes.radio, ec12.hermes.radio, ec13.hermes.radio, ec14.hermes.radio, ec15.hermes.radio, hermes.radio
+// hermes.radio.md: *.hermes.radio, *.aco-connexion.org
 static P_HERMES_RADIO: Provider = Provider {
     id: "hermes.radio",
     status: Status::Ok,
@@ -1608,375 +1608,326 @@ static P_ZOHO: Provider = Provider {
     oauth2_authorizer: None,
 };
 
-pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
-    HashMap::from([
-        ("163.com", &P_163),
-        ("aktivix.org", &P_AKTIVIX_ORG),
-        ("aol.com", &P_AOL),
-        ("arcor.de", &P_ARCOR_DE),
-        ("autistici.org", &P_AUTISTICI_ORG),
-        ("delta.blinzeln.de", &P_BLINDZELN_ORG),
-        ("delta.blindzeln.org", &P_BLINDZELN_ORG),
-        ("bluewin.ch", &P_BLUEWIN_CH),
-        ("buzon.uy", &P_BUZON_UY),
-        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
-        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
-        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
-        ("chello.at", &P_CHELLO_AT),
-        ("xfinity.com", &P_COMCAST),
-        ("comcast.net", &P_COMCAST),
-        ("dismail.de", &P_DISMAIL_DE),
-        ("disroot.org", &P_DISROOT),
-        ("e.email", &P_E_EMAIL),
-        ("espiv.net", &P_ESPIV_NET),
-        ("example.com", &P_EXAMPLE_COM),
-        ("example.org", &P_EXAMPLE_COM),
-        ("example.net", &P_EXAMPLE_COM),
-        ("123mail.org", &P_FASTMAIL),
-        ("150mail.com", &P_FASTMAIL),
-        ("150ml.com", &P_FASTMAIL),
-        ("16mail.com", &P_FASTMAIL),
-        ("2-mail.com", &P_FASTMAIL),
-        ("4email.net", &P_FASTMAIL),
-        ("50mail.com", &P_FASTMAIL),
-        ("airpost.net", &P_FASTMAIL),
-        ("allmail.net", &P_FASTMAIL),
-        ("bestmail.us", &P_FASTMAIL),
-        ("cluemail.com", &P_FASTMAIL),
-        ("elitemail.org", &P_FASTMAIL),
-        ("emailcorner.net", &P_FASTMAIL),
-        ("emailengine.net", &P_FASTMAIL),
-        ("emailengine.org", &P_FASTMAIL),
-        ("emailgroups.net", &P_FASTMAIL),
-        ("emailplus.org", &P_FASTMAIL),
-        ("emailuser.net", &P_FASTMAIL),
-        ("eml.cc", &P_FASTMAIL),
-        ("f-m.fm", &P_FASTMAIL),
-        ("fast-email.com", &P_FASTMAIL),
-        ("fast-mail.org", &P_FASTMAIL),
-        ("fastem.com", &P_FASTMAIL),
-        ("fastemail.us", &P_FASTMAIL),
-        ("fastemailer.com", &P_FASTMAIL),
-        ("fastest.cc", &P_FASTMAIL),
-        ("fastimap.com", &P_FASTMAIL),
-        ("fastmail.cn", &P_FASTMAIL),
-        ("fastmail.co.uk", &P_FASTMAIL),
-        ("fastmail.com", &P_FASTMAIL),
-        ("fastmail.com.au", &P_FASTMAIL),
-        ("fastmail.de", &P_FASTMAIL),
-        ("fastmail.es", &P_FASTMAIL),
-        ("fastmail.fm", &P_FASTMAIL),
-        ("fastmail.fr", &P_FASTMAIL),
-        ("fastmail.im", &P_FASTMAIL),
-        ("fastmail.in", &P_FASTMAIL),
-        ("fastmail.jp", &P_FASTMAIL),
-        ("fastmail.mx", &P_FASTMAIL),
-        ("fastmail.net", &P_FASTMAIL),
-        ("fastmail.nl", &P_FASTMAIL),
-        ("fastmail.org", &P_FASTMAIL),
-        ("fastmail.se", &P_FASTMAIL),
-        ("fastmail.to", &P_FASTMAIL),
-        ("fastmail.tw", &P_FASTMAIL),
-        ("fastmail.uk", &P_FASTMAIL),
-        ("fastmail.us", &P_FASTMAIL),
-        ("fastmailbox.net", &P_FASTMAIL),
-        ("fastmessaging.com", &P_FASTMAIL),
-        ("fea.st", &P_FASTMAIL),
-        ("fmail.co.uk", &P_FASTMAIL),
-        ("fmailbox.com", &P_FASTMAIL),
-        ("fmgirl.com", &P_FASTMAIL),
-        ("fmguy.com", &P_FASTMAIL),
-        ("ftml.net", &P_FASTMAIL),
-        ("h-mail.us", &P_FASTMAIL),
-        ("hailmail.net", &P_FASTMAIL),
-        ("imap-mail.com", &P_FASTMAIL),
-        ("imap.cc", &P_FASTMAIL),
-        ("imapmail.org", &P_FASTMAIL),
-        ("inoutbox.com", &P_FASTMAIL),
-        ("internet-e-mail.com", &P_FASTMAIL),
-        ("internet-mail.org", &P_FASTMAIL),
-        ("internetemails.net", &P_FASTMAIL),
-        ("internetmailing.net", &P_FASTMAIL),
-        ("jetemail.net", &P_FASTMAIL),
-        ("justemail.net", &P_FASTMAIL),
-        ("letterboxes.org", &P_FASTMAIL),
-        ("mail-central.com", &P_FASTMAIL),
-        ("mail-page.com", &P_FASTMAIL),
-        ("mailandftp.com", &P_FASTMAIL),
-        ("mailas.com", &P_FASTMAIL),
-        ("mailbolt.com", &P_FASTMAIL),
-        ("mailc.net", &P_FASTMAIL),
-        ("mailcan.com", &P_FASTMAIL),
-        ("mailforce.net", &P_FASTMAIL),
-        ("mailftp.com", &P_FASTMAIL),
-        ("mailhaven.com", &P_FASTMAIL),
-        ("mailingaddress.org", &P_FASTMAIL),
-        ("mailite.com", &P_FASTMAIL),
-        ("mailmight.com", &P_FASTMAIL),
-        ("mailnew.com", &P_FASTMAIL),
-        ("mailsent.net", &P_FASTMAIL),
-        ("mailservice.ms", &P_FASTMAIL),
-        ("mailup.net", &P_FASTMAIL),
-        ("mailworks.org", &P_FASTMAIL),
-        ("ml1.net", &P_FASTMAIL),
-        ("mm.st", &P_FASTMAIL),
-        ("myfastmail.com", &P_FASTMAIL),
-        ("mymacmail.com", &P_FASTMAIL),
-        ("nospammail.net", &P_FASTMAIL),
-        ("ownmail.net", &P_FASTMAIL),
-        ("petml.com", &P_FASTMAIL),
-        ("postinbox.com", &P_FASTMAIL),
-        ("postpro.net", &P_FASTMAIL),
-        ("proinbox.com", &P_FASTMAIL),
-        ("promessage.com", &P_FASTMAIL),
-        ("realemail.net", &P_FASTMAIL),
-        ("reallyfast.biz", &P_FASTMAIL),
-        ("reallyfast.info", &P_FASTMAIL),
-        ("rushpost.com", &P_FASTMAIL),
-        ("sent.as", &P_FASTMAIL),
-        ("sent.at", &P_FASTMAIL),
-        ("sent.com", &P_FASTMAIL),
-        ("speedpost.net", &P_FASTMAIL),
-        ("speedymail.org", &P_FASTMAIL),
-        ("ssl-mail.com", &P_FASTMAIL),
-        ("swift-mail.com", &P_FASTMAIL),
-        ("the-fastest.net", &P_FASTMAIL),
-        ("the-quickest.com", &P_FASTMAIL),
-        ("theinternetemail.com", &P_FASTMAIL),
-        ("veryfast.biz", &P_FASTMAIL),
-        ("veryspeedy.net", &P_FASTMAIL),
-        ("warpmail.net", &P_FASTMAIL),
-        ("xsmail.com", &P_FASTMAIL),
-        ("yepmail.net", &P_FASTMAIL),
-        ("your-mail.com", &P_FASTMAIL),
-        ("firemail.at", &P_FIREMAIL_DE),
-        ("firemail.de", &P_FIREMAIL_DE),
-        ("five.chat", &P_FIVE_CHAT),
-        ("freenet.de", &P_FREENET_DE),
-        ("gmail.com", &P_GMAIL),
-        ("googlemail.com", &P_GMAIL),
-        ("google.com", &P_GMAIL),
-        ("gmx.net", &P_GMX_NET),
-        ("gmx.de", &P_GMX_NET),
-        ("gmx.at", &P_GMX_NET),
-        ("gmx.ch", &P_GMX_NET),
-        ("gmx.org", &P_GMX_NET),
-        ("gmx.eu", &P_GMX_NET),
-        ("gmx.info", &P_GMX_NET),
-        ("gmx.biz", &P_GMX_NET),
-        ("gmx.com", &P_GMX_NET),
-        ("ac.hermes.radio", &P_HERMES_RADIO),
-        ("ac1.hermes.radio", &P_HERMES_RADIO),
-        ("ac2.hermes.radio", &P_HERMES_RADIO),
-        ("ac3.hermes.radio", &P_HERMES_RADIO),
-        ("ac4.hermes.radio", &P_HERMES_RADIO),
-        ("ac5.hermes.radio", &P_HERMES_RADIO),
-        ("ac6.hermes.radio", &P_HERMES_RADIO),
-        ("ac7.hermes.radio", &P_HERMES_RADIO),
-        ("ac8.hermes.radio", &P_HERMES_RADIO),
-        ("ac9.hermes.radio", &P_HERMES_RADIO),
-        ("ac10.hermes.radio", &P_HERMES_RADIO),
-        ("ac11.hermes.radio", &P_HERMES_RADIO),
-        ("ac12.hermes.radio", &P_HERMES_RADIO),
-        ("ac13.hermes.radio", &P_HERMES_RADIO),
-        ("ac14.hermes.radio", &P_HERMES_RADIO),
-        ("ac15.hermes.radio", &P_HERMES_RADIO),
-        ("ka.hermes.radio", &P_HERMES_RADIO),
-        ("ka1.hermes.radio", &P_HERMES_RADIO),
-        ("ka2.hermes.radio", &P_HERMES_RADIO),
-        ("ka3.hermes.radio", &P_HERMES_RADIO),
-        ("ka4.hermes.radio", &P_HERMES_RADIO),
-        ("ka5.hermes.radio", &P_HERMES_RADIO),
-        ("ka6.hermes.radio", &P_HERMES_RADIO),
-        ("ka7.hermes.radio", &P_HERMES_RADIO),
-        ("ka8.hermes.radio", &P_HERMES_RADIO),
-        ("ka9.hermes.radio", &P_HERMES_RADIO),
-        ("ka10.hermes.radio", &P_HERMES_RADIO),
-        ("ka11.hermes.radio", &P_HERMES_RADIO),
-        ("ka12.hermes.radio", &P_HERMES_RADIO),
-        ("ka13.hermes.radio", &P_HERMES_RADIO),
-        ("ka14.hermes.radio", &P_HERMES_RADIO),
-        ("ka15.hermes.radio", &P_HERMES_RADIO),
-        ("ec.hermes.radio", &P_HERMES_RADIO),
-        ("ec1.hermes.radio", &P_HERMES_RADIO),
-        ("ec2.hermes.radio", &P_HERMES_RADIO),
-        ("ec3.hermes.radio", &P_HERMES_RADIO),
-        ("ec4.hermes.radio", &P_HERMES_RADIO),
-        ("ec5.hermes.radio", &P_HERMES_RADIO),
-        ("ec6.hermes.radio", &P_HERMES_RADIO),
-        ("ec7.hermes.radio", &P_HERMES_RADIO),
-        ("ec8.hermes.radio", &P_HERMES_RADIO),
-        ("ec9.hermes.radio", &P_HERMES_RADIO),
-        ("ec10.hermes.radio", &P_HERMES_RADIO),
-        ("ec11.hermes.radio", &P_HERMES_RADIO),
-        ("ec12.hermes.radio", &P_HERMES_RADIO),
-        ("ec13.hermes.radio", &P_HERMES_RADIO),
-        ("ec14.hermes.radio", &P_HERMES_RADIO),
-        ("ec15.hermes.radio", &P_HERMES_RADIO),
-        ("hermes.radio", &P_HERMES_RADIO),
-        ("hey.com", &P_HEY_COM),
-        ("i.ua", &P_I_UA),
-        ("i3.net", &P_I3_NET),
-        ("icloud.com", &P_ICLOUD),
-        ("me.com", &P_ICLOUD),
-        ("mac.com", &P_ICLOUD),
-        ("ik.me", &P_INFOMANIAK_COM),
-        ("kolst.com", &P_KOLST_COM),
-        ("kontent.com", &P_KONTENT_COM),
-        ("mail.de", &P_MAIL_DE),
-        ("mail.ru", &P_MAIL_RU),
-        ("inbox.ru", &P_MAIL_RU),
-        ("internet.ru", &P_MAIL_RU),
-        ("bk.ru", &P_MAIL_RU),
-        ("list.ru", &P_MAIL_RU),
-        ("mail2tor.com", &P_MAIL2TOR),
-        ("mailbox.org", &P_MAILBOX_ORG),
-        ("secure.mailbox.org", &P_MAILBOX_ORG),
-        ("mailo.com", &P_MAILO_COM),
-        ("nauta.cu", &P_NAUTA_CU),
-        ("naver.com", &P_NAVER),
-        ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
-        ("nubo.coop", &P_NUBO_COOP),
-        ("hotmail.com", &P_OUTLOOK_COM),
-        ("outlook.com", &P_OUTLOOK_COM),
-        ("office365.com", &P_OUTLOOK_COM),
-        ("outlook.com.tr", &P_OUTLOOK_COM),
-        ("live.com", &P_OUTLOOK_COM),
-        ("outlook.de", &P_OUTLOOK_COM),
-        ("ouvaton.org", &P_OUVATON_COOP),
-        ("posteo.de", &P_POSTEO),
-        ("posteo.af", &P_POSTEO),
-        ("posteo.at", &P_POSTEO),
-        ("posteo.be", &P_POSTEO),
-        ("posteo.ca", &P_POSTEO),
-        ("posteo.ch", &P_POSTEO),
-        ("posteo.cl", &P_POSTEO),
-        ("posteo.co", &P_POSTEO),
-        ("posteo.co.uk", &P_POSTEO),
-        ("posteo.com.br", &P_POSTEO),
-        ("posteo.cr", &P_POSTEO),
-        ("posteo.cz", &P_POSTEO),
-        ("posteo.dk", &P_POSTEO),
-        ("posteo.ee", &P_POSTEO),
-        ("posteo.es", &P_POSTEO),
-        ("posteo.eu", &P_POSTEO),
-        ("posteo.fi", &P_POSTEO),
-        ("posteo.gl", &P_POSTEO),
-        ("posteo.gr", &P_POSTEO),
-        ("posteo.hn", &P_POSTEO),
-        ("posteo.hr", &P_POSTEO),
-        ("posteo.hu", &P_POSTEO),
-        ("posteo.ie", &P_POSTEO),
-        ("posteo.in", &P_POSTEO),
-        ("posteo.is", &P_POSTEO),
-        ("posteo.it", &P_POSTEO),
-        ("posteo.jp", &P_POSTEO),
-        ("posteo.la", &P_POSTEO),
-        ("posteo.li", &P_POSTEO),
-        ("posteo.lt", &P_POSTEO),
-        ("posteo.lu", &P_POSTEO),
-        ("posteo.me", &P_POSTEO),
-        ("posteo.mx", &P_POSTEO),
-        ("posteo.my", &P_POSTEO),
-        ("posteo.net", &P_POSTEO),
-        ("posteo.nl", &P_POSTEO),
-        ("posteo.no", &P_POSTEO),
-        ("posteo.nz", &P_POSTEO),
-        ("posteo.org", &P_POSTEO),
-        ("posteo.pe", &P_POSTEO),
-        ("posteo.pl", &P_POSTEO),
-        ("posteo.pm", &P_POSTEO),
-        ("posteo.pt", &P_POSTEO),
-        ("posteo.ro", &P_POSTEO),
-        ("posteo.ru", &P_POSTEO),
-        ("posteo.se", &P_POSTEO),
-        ("posteo.sg", &P_POSTEO),
-        ("posteo.si", &P_POSTEO),
-        ("posteo.tn", &P_POSTEO),
-        ("posteo.uk", &P_POSTEO),
-        ("posteo.us", &P_POSTEO),
-        ("protonmail.com", &P_PROTONMAIL),
-        ("protonmail.ch", &P_PROTONMAIL),
-        ("pm.me", &P_PROTONMAIL),
-        ("qq.com", &P_QQ),
-        ("foxmail.com", &P_QQ),
-        ("riseup.net", &P_RISEUP_NET),
-        ("rogers.com", &P_ROGERS_COM),
-        ("sonic.net", &P_SONIC),
-        ("systemausfall.org", &P_SYSTEMAUSFALL_ORG),
-        ("solidaris.me", &P_SYSTEMAUSFALL_ORG),
-        ("systemli.org", &P_SYSTEMLI_ORG),
-        ("t-online.de", &P_T_ONLINE),
-        ("magenta.de", &P_T_ONLINE),
-        ("testrun.org", &P_TESTRUN),
-        ("tiscali.it", &P_TISCALI_IT),
-        ("tutanota.com", &P_TUTANOTA),
-        ("tutanota.de", &P_TUTANOTA),
-        ("tutamail.com", &P_TUTANOTA),
-        ("tuta.io", &P_TUTANOTA),
-        ("keemail.me", &P_TUTANOTA),
-        ("ukr.net", &P_UKR_NET),
-        ("undernet.uy", &P_UNDERNET_UY),
-        ("vfemail.net", &P_VFEMAIL),
-        ("vivaldi.net", &P_VIVALDI),
-        ("vodafone.de", &P_VODAFONE_DE),
-        ("vodafonemail.de", &P_VODAFONE_DE),
-        ("web.de", &P_WEB_DE),
-        ("email.de", &P_WEB_DE),
-        ("flirt.ms", &P_WEB_DE),
-        ("hallo.ms", &P_WEB_DE),
-        ("kuss.ms", &P_WEB_DE),
-        ("love.ms", &P_WEB_DE),
-        ("magic.ms", &P_WEB_DE),
-        ("singles.ms", &P_WEB_DE),
-        ("cool.ms", &P_WEB_DE),
-        ("kanzler.ms", &P_WEB_DE),
-        ("okay.ms", &P_WEB_DE),
-        ("party.ms", &P_WEB_DE),
-        ("pop.ms", &P_WEB_DE),
-        ("stars.ms", &P_WEB_DE),
-        ("techno.ms", &P_WEB_DE),
-        ("clever.ms", &P_WEB_DE),
-        ("deutschland.ms", &P_WEB_DE),
-        ("genial.ms", &P_WEB_DE),
-        ("ich.ms", &P_WEB_DE),
-        ("online.ms", &P_WEB_DE),
-        ("smart.ms", &P_WEB_DE),
-        ("wichtig.ms", &P_WEB_DE),
-        ("action.ms", &P_WEB_DE),
-        ("fussball.ms", &P_WEB_DE),
-        ("joker.ms", &P_WEB_DE),
-        ("planet.ms", &P_WEB_DE),
-        ("power.ms", &P_WEB_DE),
-        ("yahoo.com", &P_YAHOO),
-        ("yahoo.de", &P_YAHOO),
-        ("yahoo.it", &P_YAHOO),
-        ("yahoo.fr", &P_YAHOO),
-        ("yahoo.es", &P_YAHOO),
-        ("yahoo.se", &P_YAHOO),
-        ("yahoo.co.uk", &P_YAHOO),
-        ("yahoo.co.nz", &P_YAHOO),
-        ("yahoo.com.au", &P_YAHOO),
-        ("yahoo.com.ar", &P_YAHOO),
-        ("yahoo.com.br", &P_YAHOO),
-        ("yahoo.com.mx", &P_YAHOO),
-        ("ymail.com", &P_YAHOO),
-        ("rocketmail.com", &P_YAHOO),
-        ("yahoodns.net", &P_YAHOO),
-        ("yandex.com", &P_YANDEX_RU),
-        ("yandex.by", &P_YANDEX_RU),
-        ("yandex.kz", &P_YANDEX_RU),
-        ("yandex.ru", &P_YANDEX_RU),
-        ("yandex.ua", &P_YANDEX_RU),
-        ("ya.ru", &P_YANDEX_RU),
-        ("narod.ru", &P_YANDEX_RU),
-        ("yggmail", &P_YGGMAIL),
-        ("ziggo.nl", &P_ZIGGO_NL),
-        ("zohomail.eu", &P_ZOHO),
-        ("zohomail.com", &P_ZOHO),
-        ("zoho.com", &P_ZOHO),
-    ])
-});
+pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
+    ("163.com", &P_163),
+    ("aktivix.org", &P_AKTIVIX_ORG),
+    ("aol.com", &P_AOL),
+    ("arcor.de", &P_ARCOR_DE),
+    ("autistici.org", &P_AUTISTICI_ORG),
+    ("delta.blinzeln.de", &P_BLINDZELN_ORG),
+    ("delta.blindzeln.org", &P_BLINDZELN_ORG),
+    ("bluewin.ch", &P_BLUEWIN_CH),
+    ("buzon.uy", &P_BUZON_UY),
+    ("c1.testrun.org", &P_C1_TESTRUN_ORG),
+    ("c2.testrun.org", &P_C2_TESTRUN_ORG),
+    ("c3.testrun.org", &P_C3_TESTRUN_ORG),
+    ("chello.at", &P_CHELLO_AT),
+    ("xfinity.com", &P_COMCAST),
+    ("comcast.net", &P_COMCAST),
+    ("dismail.de", &P_DISMAIL_DE),
+    ("disroot.org", &P_DISROOT),
+    ("e.email", &P_E_EMAIL),
+    ("espiv.net", &P_ESPIV_NET),
+    ("example.com", &P_EXAMPLE_COM),
+    ("example.org", &P_EXAMPLE_COM),
+    ("example.net", &P_EXAMPLE_COM),
+    ("123mail.org", &P_FASTMAIL),
+    ("150mail.com", &P_FASTMAIL),
+    ("150ml.com", &P_FASTMAIL),
+    ("16mail.com", &P_FASTMAIL),
+    ("2-mail.com", &P_FASTMAIL),
+    ("4email.net", &P_FASTMAIL),
+    ("50mail.com", &P_FASTMAIL),
+    ("airpost.net", &P_FASTMAIL),
+    ("allmail.net", &P_FASTMAIL),
+    ("bestmail.us", &P_FASTMAIL),
+    ("cluemail.com", &P_FASTMAIL),
+    ("elitemail.org", &P_FASTMAIL),
+    ("emailcorner.net", &P_FASTMAIL),
+    ("emailengine.net", &P_FASTMAIL),
+    ("emailengine.org", &P_FASTMAIL),
+    ("emailgroups.net", &P_FASTMAIL),
+    ("emailplus.org", &P_FASTMAIL),
+    ("emailuser.net", &P_FASTMAIL),
+    ("eml.cc", &P_FASTMAIL),
+    ("f-m.fm", &P_FASTMAIL),
+    ("fast-email.com", &P_FASTMAIL),
+    ("fast-mail.org", &P_FASTMAIL),
+    ("fastem.com", &P_FASTMAIL),
+    ("fastemail.us", &P_FASTMAIL),
+    ("fastemailer.com", &P_FASTMAIL),
+    ("fastest.cc", &P_FASTMAIL),
+    ("fastimap.com", &P_FASTMAIL),
+    ("fastmail.cn", &P_FASTMAIL),
+    ("fastmail.co.uk", &P_FASTMAIL),
+    ("fastmail.com", &P_FASTMAIL),
+    ("fastmail.com.au", &P_FASTMAIL),
+    ("fastmail.de", &P_FASTMAIL),
+    ("fastmail.es", &P_FASTMAIL),
+    ("fastmail.fm", &P_FASTMAIL),
+    ("fastmail.fr", &P_FASTMAIL),
+    ("fastmail.im", &P_FASTMAIL),
+    ("fastmail.in", &P_FASTMAIL),
+    ("fastmail.jp", &P_FASTMAIL),
+    ("fastmail.mx", &P_FASTMAIL),
+    ("fastmail.net", &P_FASTMAIL),
+    ("fastmail.nl", &P_FASTMAIL),
+    ("fastmail.org", &P_FASTMAIL),
+    ("fastmail.se", &P_FASTMAIL),
+    ("fastmail.to", &P_FASTMAIL),
+    ("fastmail.tw", &P_FASTMAIL),
+    ("fastmail.uk", &P_FASTMAIL),
+    ("fastmail.us", &P_FASTMAIL),
+    ("fastmailbox.net", &P_FASTMAIL),
+    ("fastmessaging.com", &P_FASTMAIL),
+    ("fea.st", &P_FASTMAIL),
+    ("fmail.co.uk", &P_FASTMAIL),
+    ("fmailbox.com", &P_FASTMAIL),
+    ("fmgirl.com", &P_FASTMAIL),
+    ("fmguy.com", &P_FASTMAIL),
+    ("ftml.net", &P_FASTMAIL),
+    ("h-mail.us", &P_FASTMAIL),
+    ("hailmail.net", &P_FASTMAIL),
+    ("imap-mail.com", &P_FASTMAIL),
+    ("imap.cc", &P_FASTMAIL),
+    ("imapmail.org", &P_FASTMAIL),
+    ("inoutbox.com", &P_FASTMAIL),
+    ("internet-e-mail.com", &P_FASTMAIL),
+    ("internet-mail.org", &P_FASTMAIL),
+    ("internetemails.net", &P_FASTMAIL),
+    ("internetmailing.net", &P_FASTMAIL),
+    ("jetemail.net", &P_FASTMAIL),
+    ("justemail.net", &P_FASTMAIL),
+    ("letterboxes.org", &P_FASTMAIL),
+    ("mail-central.com", &P_FASTMAIL),
+    ("mail-page.com", &P_FASTMAIL),
+    ("mailandftp.com", &P_FASTMAIL),
+    ("mailas.com", &P_FASTMAIL),
+    ("mailbolt.com", &P_FASTMAIL),
+    ("mailc.net", &P_FASTMAIL),
+    ("mailcan.com", &P_FASTMAIL),
+    ("mailforce.net", &P_FASTMAIL),
+    ("mailftp.com", &P_FASTMAIL),
+    ("mailhaven.com", &P_FASTMAIL),
+    ("mailingaddress.org", &P_FASTMAIL),
+    ("mailite.com", &P_FASTMAIL),
+    ("mailmight.com", &P_FASTMAIL),
+    ("mailnew.com", &P_FASTMAIL),
+    ("mailsent.net", &P_FASTMAIL),
+    ("mailservice.ms", &P_FASTMAIL),
+    ("mailup.net", &P_FASTMAIL),
+    ("mailworks.org", &P_FASTMAIL),
+    ("ml1.net", &P_FASTMAIL),
+    ("mm.st", &P_FASTMAIL),
+    ("myfastmail.com", &P_FASTMAIL),
+    ("mymacmail.com", &P_FASTMAIL),
+    ("nospammail.net", &P_FASTMAIL),
+    ("ownmail.net", &P_FASTMAIL),
+    ("petml.com", &P_FASTMAIL),
+    ("postinbox.com", &P_FASTMAIL),
+    ("postpro.net", &P_FASTMAIL),
+    ("proinbox.com", &P_FASTMAIL),
+    ("promessage.com", &P_FASTMAIL),
+    ("realemail.net", &P_FASTMAIL),
+    ("reallyfast.biz", &P_FASTMAIL),
+    ("reallyfast.info", &P_FASTMAIL),
+    ("rushpost.com", &P_FASTMAIL),
+    ("sent.as", &P_FASTMAIL),
+    ("sent.at", &P_FASTMAIL),
+    ("sent.com", &P_FASTMAIL),
+    ("speedpost.net", &P_FASTMAIL),
+    ("speedymail.org", &P_FASTMAIL),
+    ("ssl-mail.com", &P_FASTMAIL),
+    ("swift-mail.com", &P_FASTMAIL),
+    ("the-fastest.net", &P_FASTMAIL),
+    ("the-quickest.com", &P_FASTMAIL),
+    ("theinternetemail.com", &P_FASTMAIL),
+    ("veryfast.biz", &P_FASTMAIL),
+    ("veryspeedy.net", &P_FASTMAIL),
+    ("warpmail.net", &P_FASTMAIL),
+    ("xsmail.com", &P_FASTMAIL),
+    ("yepmail.net", &P_FASTMAIL),
+    ("your-mail.com", &P_FASTMAIL),
+    ("firemail.at", &P_FIREMAIL_DE),
+    ("firemail.de", &P_FIREMAIL_DE),
+    ("five.chat", &P_FIVE_CHAT),
+    ("freenet.de", &P_FREENET_DE),
+    ("gmail.com", &P_GMAIL),
+    ("googlemail.com", &P_GMAIL),
+    ("google.com", &P_GMAIL),
+    ("gmx.net", &P_GMX_NET),
+    ("gmx.de", &P_GMX_NET),
+    ("gmx.at", &P_GMX_NET),
+    ("gmx.ch", &P_GMX_NET),
+    ("gmx.org", &P_GMX_NET),
+    ("gmx.eu", &P_GMX_NET),
+    ("gmx.info", &P_GMX_NET),
+    ("gmx.biz", &P_GMX_NET),
+    ("gmx.com", &P_GMX_NET),
+    ("*.hermes.radio", &P_HERMES_RADIO),
+    ("*.aco-connexion.org", &P_HERMES_RADIO),
+    ("hey.com", &P_HEY_COM),
+    ("i.ua", &P_I_UA),
+    ("i3.net", &P_I3_NET),
+    ("icloud.com", &P_ICLOUD),
+    ("me.com", &P_ICLOUD),
+    ("mac.com", &P_ICLOUD),
+    ("ik.me", &P_INFOMANIAK_COM),
+    ("kolst.com", &P_KOLST_COM),
+    ("kontent.com", &P_KONTENT_COM),
+    ("mail.de", &P_MAIL_DE),
+    ("mail.ru", &P_MAIL_RU),
+    ("inbox.ru", &P_MAIL_RU),
+    ("internet.ru", &P_MAIL_RU),
+    ("bk.ru", &P_MAIL_RU),
+    ("list.ru", &P_MAIL_RU),
+    ("mail2tor.com", &P_MAIL2TOR),
+    ("mailbox.org", &P_MAILBOX_ORG),
+    ("secure.mailbox.org", &P_MAILBOX_ORG),
+    ("mailo.com", &P_MAILO_COM),
+    ("nauta.cu", &P_NAUTA_CU),
+    ("naver.com", &P_NAVER),
+    ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
+    ("nubo.coop", &P_NUBO_COOP),
+    ("hotmail.com", &P_OUTLOOK_COM),
+    ("outlook.com", &P_OUTLOOK_COM),
+    ("office365.com", &P_OUTLOOK_COM),
+    ("outlook.com.tr", &P_OUTLOOK_COM),
+    ("live.com", &P_OUTLOOK_COM),
+    ("outlook.de", &P_OUTLOOK_COM),
+    ("ouvaton.org", &P_OUVATON_COOP),
+    ("posteo.de", &P_POSTEO),
+    ("posteo.af", &P_POSTEO),
+    ("posteo.at", &P_POSTEO),
+    ("posteo.be", &P_POSTEO),
+    ("posteo.ca", &P_POSTEO),
+    ("posteo.ch", &P_POSTEO),
+    ("posteo.cl", &P_POSTEO),
+    ("posteo.co", &P_POSTEO),
+    ("posteo.co.uk", &P_POSTEO),
+    ("posteo.com.br", &P_POSTEO),
+    ("posteo.cr", &P_POSTEO),
+    ("posteo.cz", &P_POSTEO),
+    ("posteo.dk", &P_POSTEO),
+    ("posteo.ee", &P_POSTEO),
+    ("posteo.es", &P_POSTEO),
+    ("posteo.eu", &P_POSTEO),
+    ("posteo.fi", &P_POSTEO),
+    ("posteo.gl", &P_POSTEO),
+    ("posteo.gr", &P_POSTEO),
+    ("posteo.hn", &P_POSTEO),
+    ("posteo.hr", &P_POSTEO),
+    ("posteo.hu", &P_POSTEO),
+    ("posteo.ie", &P_POSTEO),
+    ("posteo.in", &P_POSTEO),
+    ("posteo.is", &P_POSTEO),
+    ("posteo.it", &P_POSTEO),
+    ("posteo.jp", &P_POSTEO),
+    ("posteo.la", &P_POSTEO),
+    ("posteo.li", &P_POSTEO),
+    ("posteo.lt", &P_POSTEO),
+    ("posteo.lu", &P_POSTEO),
+    ("posteo.me", &P_POSTEO),
+    ("posteo.mx", &P_POSTEO),
+    ("posteo.my", &P_POSTEO),
+    ("posteo.net", &P_POSTEO),
+    ("posteo.nl", &P_POSTEO),
+    ("posteo.no", &P_POSTEO),
+    ("posteo.nz", &P_POSTEO),
+    ("posteo.org", &P_POSTEO),
+    ("posteo.pe", &P_POSTEO),
+    ("posteo.pl", &P_POSTEO),
+    ("posteo.pm", &P_POSTEO),
+    ("posteo.pt", &P_POSTEO),
+    ("posteo.ro", &P_POSTEO),
+    ("posteo.ru", &P_POSTEO),
+    ("posteo.se", &P_POSTEO),
+    ("posteo.sg", &P_POSTEO),
+    ("posteo.si", &P_POSTEO),
+    ("posteo.tn", &P_POSTEO),
+    ("posteo.uk", &P_POSTEO),
+    ("posteo.us", &P_POSTEO),
+    ("protonmail.com", &P_PROTONMAIL),
+    ("protonmail.ch", &P_PROTONMAIL),
+    ("pm.me", &P_PROTONMAIL),
+    ("qq.com", &P_QQ),
+    ("foxmail.com", &P_QQ),
+    ("riseup.net", &P_RISEUP_NET),
+    ("rogers.com", &P_ROGERS_COM),
+    ("sonic.net", &P_SONIC),
+    ("systemausfall.org", &P_SYSTEMAUSFALL_ORG),
+    ("solidaris.me", &P_SYSTEMAUSFALL_ORG),
+    ("systemli.org", &P_SYSTEMLI_ORG),
+    ("t-online.de", &P_T_ONLINE),
+    ("magenta.de", &P_T_ONLINE),
+    ("testrun.org", &P_TESTRUN),
+    ("tiscali.it", &P_TISCALI_IT),
+    ("tutanota.com", &P_TUTANOTA),
+    ("tutanota.de", &P_TUTANOTA),
+    ("tutamail.com", &P_TUTANOTA),
+    ("tuta.io", &P_TUTANOTA),
+    ("keemail.me", &P_TUTANOTA),
+    ("ukr.net", &P_UKR_NET),
+    ("undernet.uy", &P_UNDERNET_UY),
+    ("vfemail.net", &P_VFEMAIL),
+    ("vivaldi.net", &P_VIVALDI),
+    ("vodafone.de", &P_VODAFONE_DE),
+    ("vodafonemail.de", &P_VODAFONE_DE),
+    ("web.de", &P_WEB_DE),
+    ("email.de", &P_WEB_DE),
+    ("flirt.ms", &P_WEB_DE),
+    ("hallo.ms", &P_WEB_DE),
+    ("kuss.ms", &P_WEB_DE),
+    ("love.ms", &P_WEB_DE),
+    ("magic.ms", &P_WEB_DE),
+    ("singles.ms", &P_WEB_DE),
+    ("cool.ms", &P_WEB_DE),
+    ("kanzler.ms", &P_WEB_DE),
+    ("okay.ms", &P_WEB_DE),
+    ("party.ms", &P_WEB_DE),
+    ("pop.ms", &P_WEB_DE),
+    ("stars.ms", &P_WEB_DE),
+    ("techno.ms", &P_WEB_DE),
+    ("clever.ms", &P_WEB_DE),
+    ("deutschland.ms", &P_WEB_DE),
+    ("genial.ms", &P_WEB_DE),
+    ("ich.ms", &P_WEB_DE),
+    ("online.ms", &P_WEB_DE),
+    ("smart.ms", &P_WEB_DE),
+    ("wichtig.ms", &P_WEB_DE),
+    ("action.ms", &P_WEB_DE),
+    ("fussball.ms", &P_WEB_DE),
+    ("joker.ms", &P_WEB_DE),
+    ("planet.ms", &P_WEB_DE),
+    ("power.ms", &P_WEB_DE),
+    ("yahoo.com", &P_YAHOO),
+    ("yahoo.de", &P_YAHOO),
+    ("yahoo.it", &P_YAHOO),
+    ("yahoo.fr", &P_YAHOO),
+    ("yahoo.es", &P_YAHOO),
+    ("yahoo.se", &P_YAHOO),
+    ("yahoo.co.uk", &P_YAHOO),
+    ("yahoo.co.nz", &P_YAHOO),
+    ("yahoo.com.au", &P_YAHOO),
+    ("yahoo.com.ar", &P_YAHOO),
+    ("yahoo.com.br", &P_YAHOO),
+    ("yahoo.com.mx", &P_YAHOO),
+    ("ymail.com", &P_YAHOO),
+    ("rocketmail.com", &P_YAHOO),
+    ("yahoodns.net", &P_YAHOO),
+    ("yandex.com", &P_YANDEX_RU),
+    ("yandex.by", &P_YANDEX_RU),
+    ("yandex.kz", &P_YANDEX_RU),
+    ("yandex.ru", &P_YANDEX_RU),
+    ("yandex.ua", &P_YANDEX_RU),
+    ("ya.ru", &P_YANDEX_RU),
+    ("narod.ru", &P_YANDEX_RU),
+    ("yggmail", &P_YGGMAIL),
+    ("ziggo.nl", &P_ZIGGO_NL),
+    ("zohomail.eu", &P_ZOHO),
+    ("zohomail.com", &P_ZOHO),
+    ("zoho.com", &P_ZOHO),
+];
 
 pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
     HashMap::from([
@@ -2050,4 +2001,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static _PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 11, 5).unwrap());
+    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2024, 2, 5).unwrap());

src/qr.rs

@@ -25,6 +25,8 @@ use crate::socks::Socks5Config;
 use crate::token;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
+const IDELTACHAT_SCHEME: &str = "https://i.delta.chat/#";
+const IDELTACHAT_NOSLASH_SCHEME: &str = "https://i.delta.chat#";
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
 pub(super) const DCLOGIN_SCHEME: &str = "DCLOGIN:";
 const DCWEBRTC_SCHEME: &str = "DCWEBRTC:";
@@ -253,6 +255,10 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result<Qr> {
         decode_openpgp(context, qr)
             .await
             .context("failed to decode OPENPGP4FPR QR code")?
+    } else if qr.starts_with(IDELTACHAT_SCHEME) {
+        decode_ideltachat(context, IDELTACHAT_SCHEME, qr).await?
+    } else if qr.starts_with(IDELTACHAT_NOSLASH_SCHEME) {
+        decode_ideltachat(context, IDELTACHAT_NOSLASH_SCHEME, qr).await?
     } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(qr)?
     } else if starts_with_ignore_case(qr, DCLOGIN_SCHEME) {
@@ -301,11 +307,12 @@ pub fn format_backup(qr: &Qr) -> Result<String> {
 async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
-    let (fingerprint, fragment) = match payload.find('#').map(|offset| {
-        let (fp, rest) = payload.split_at(offset);
-        // need to remove the # from the fragment
-        (fp, &rest[1..])
-    }) {
+    // macOS and iOS sometimes replace the # with %23 (uri encode it), we should be able to parse this wrong format too.
+    // see issue https://github.com/deltachat/deltachat-core-rust/issues/1969 for more info
+    let (fingerprint, fragment) = match payload
+        .split_once('#')
+        .or_else(|| payload.split_once("%23"))
+    {
         Some(pair) => pair,
         None => (payload, ""),
     };
@@ -376,7 +383,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
                 .await
                 .with_context(|| format!("can't check if address {addr:?} is our address"))?
             {
-                if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await {
+                if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await? {
                     Ok(Qr::WithdrawVerifyGroup {
                         grpname,
                         grpid,
@@ -406,7 +413,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
                 })
             }
         } else if context.is_self_addr(&addr).await? {
-            if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await {
+            if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await? {
                 Ok(Qr::WithdrawVerifyContact {
                     contact_id,
                     fingerprint,
@@ -453,6 +460,15 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
     }
 }
 
+/// scheme: `https://i.delta.chat[/]#FINGERPRINT&a=ADDR[&OPTIONAL_PARAMS]`
+async fn decode_ideltachat(context: &Context, prefix: &str, qr: &str) -> Result<Qr> {
+    let qr = qr.replacen(prefix, OPENPGP4FPR_SCHEME, 1);
+    let qr = qr.replacen('&', "#", 1);
+    decode_openpgp(context, &qr)
+        .await
+        .context("failed to decode {prefix} QR code")
+}
+
 /// scheme: `DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3`
 fn decode_account(qr: &str) -> Result<Qr> {
     let payload = qr
@@ -544,8 +560,12 @@ async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> {
                     "Cannot create account, response from {url_str:?} is malformed:\n{response_text:?}"
                 )
             })?;
-        context.set_config(Config::Addr, Some(&email)).await?;
-        context.set_config(Config::MailPw, Some(&password)).await?;
+        context
+            .set_config_internal(Config::Addr, Some(&email))
+            .await?;
+        context
+            .set_config_internal(Config::MailPw, Some(&password))
+            .await?;
 
         Ok(())
     } else {
@@ -573,7 +593,7 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
             instance_pattern,
         } => {
             context
-                .set_config(Config::WebrtcInstance, Some(&instance_pattern))
+                .set_config_internal(Config::WebrtcInstance, Some(&instance_pattern))
                 .await?;
         }
         Qr::WithdrawVerifyContact {
@@ -943,6 +963,40 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decode_ideltachat_link() -> Result<()> {
+        let ctx = TestContext::new().await;
+
+        let qr = check_qr(
+            &ctx.ctx,
+            "https://i.delta.chat/#79252762C34C5096AF57958F4FC3D21A81B0F0A7&a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await?;
+        assert!(matches!(qr, Qr::AskVerifyGroup { .. }));
+
+        let qr = check_qr(
+            &ctx.ctx,
+            "https://i.delta.chat#79252762C34C5096AF57958F4FC3D21A81B0F0A7&a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await?;
+        assert!(matches!(qr, Qr::AskVerifyGroup { .. }));
+
+        Ok(())
+    }
+
+    // macOS and iOS sometimes replace the # with %23 (uri encode it), we should be able to parse this wrong format too.
+    // see issue https://github.com/deltachat/deltachat-core-rust/issues/1969 for more info
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decode_openpgp_tolerance_for_issue_1969() -> Result<()> {
+        let ctx = TestContext::new().await;
+
+        let qr = check_qr(
+            &ctx.ctx,
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7%23a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await?;
+
+        assert!(matches!(qr, Qr::AskVerifyGroup { .. }));
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_decode_openpgp_group() -> Result<()> {
         let ctx = TestContext::new().await;
@@ -1057,6 +1111,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
         assert!(

src/qr/dclogin_scheme.rs

@@ -163,7 +163,9 @@ pub(crate) async fn configure_from_login_qr(
     address: &str,
     options: LoginOptions,
 ) -> Result<()> {
-    context.set_config(Config::Addr, Some(address)).await?;
+    context
+        .set_config_internal(Config::Addr, Some(address))
+        .await?;
 
     match options {
         LoginOptions::V1 {
@@ -181,27 +183,35 @@ pub(crate) async fn configure_from_login_qr(
             smtp_security,
             smtp_certificate_checks,
         } => {
-            context.set_config(Config::MailPw, Some(&mail_pw)).await?;
+            context
+                .set_config_internal(Config::MailPw, Some(&mail_pw))
+                .await?;
             if let Some(value) = imap_host {
-                context.set_config(Config::MailServer, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::MailServer, Some(&value))
+                    .await?;
             }
             if let Some(value) = imap_port {
                 context
-                    .set_config(Config::MailPort, Some(&value.to_string()))
+                    .set_config_internal(Config::MailPort, Some(&value.to_string()))
                     .await?;
             }
             if let Some(value) = imap_username {
-                context.set_config(Config::MailUser, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::MailUser, Some(&value))
+                    .await?;
             }
             if let Some(value) = imap_password {
-                context.set_config(Config::MailPw, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::MailPw, Some(&value))
+                    .await?;
             }
             if let Some(value) = imap_security {
                 let code = value
                     .to_u8()
                     .context("could not convert imap security value to number")?;
                 context
-                    .set_config(Config::MailSecurity, Some(&code.to_string()))
+                    .set_config_internal(Config::MailSecurity, Some(&code.to_string()))
                     .await?;
             }
             if let Some(value) = imap_certificate_checks {
@@ -209,29 +219,35 @@ pub(crate) async fn configure_from_login_qr(
                     .to_u32()
                     .context("could not convert imap certificate checks value to number")?;
                 context
-                    .set_config(Config::ImapCertificateChecks, Some(&code.to_string()))
+                    .set_config_internal(Config::ImapCertificateChecks, Some(&code.to_string()))
                     .await?;
             }
             if let Some(value) = smtp_host {
-                context.set_config(Config::SendServer, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::SendServer, Some(&value))
+                    .await?;
             }
             if let Some(value) = smtp_port {
                 context
-                    .set_config(Config::SendPort, Some(&value.to_string()))
+                    .set_config_internal(Config::SendPort, Some(&value.to_string()))
                     .await?;
             }
             if let Some(value) = smtp_username {
-                context.set_config(Config::SendUser, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::SendUser, Some(&value))
+                    .await?;
             }
             if let Some(value) = smtp_password {
-                context.set_config(Config::SendPw, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::SendPw, Some(&value))
+                    .await?;
             }
             if let Some(value) = smtp_security {
                 let code = value
                     .to_u8()
                     .context("could not convert smtp security value to number")?;
                 context
-                    .set_config(Config::SendSecurity, Some(&code.to_string()))
+                    .set_config_internal(Config::SendSecurity, Some(&code.to_string()))
                     .await?;
             }
             if let Some(value) = smtp_certificate_checks {
@@ -239,7 +255,7 @@ pub(crate) async fn configure_from_login_qr(
                     .to_u32()
                     .context("could not convert smtp certificate checks value to number")?;
                 context
-                    .set_config(Config::SmtpCertificateChecks, Some(&code.to_string()))
+                    .set_config_internal(Config::SmtpCertificateChecks, Some(&code.to_string()))
                     .await?;
             }
             Ok(())

src/quota.rs

@@ -12,7 +12,7 @@ use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::session::Session as ImapSession;
 use crate::imap::Imap;
 use crate::message::{Message, Viewtype};
-use crate::tools::time;
+use crate::tools;
 use crate::{stock_str, EventType};
 
 /// warn about a nearly full mailbox after this usage percentage is reached.
@@ -40,8 +40,8 @@ pub struct QuotaInfo {
     /// set to `Ok()` for valid quota information.
     pub(crate) recent: Result<BTreeMap<String, Vec<QuotaResource>>>,
 
-    /// Timestamp when structure was modified.
-    pub(crate) modified: i64,
+    /// When the structure was modified.
+    pub(crate) modified: tools::Time,
 }
 
 async fn get_unique_quota_roots_and_usage(
@@ -132,13 +132,17 @@ impl Context {
                         highest,
                         self.get_config_int(Config::QuotaExceeding).await? as u64,
                     ) {
-                        self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
-                            .await?;
+                        self.set_config_internal(
+                            Config::QuotaExceeding,
+                            Some(&highest.to_string()),
+                        )
+                        .await?;
                         let mut msg = Message::new(Viewtype::Text);
                         msg.text = stock_str::quota_exceeding(self, highest).await;
                         add_device_msg_with_importance(self, None, Some(&mut msg), true).await?;
                     } else if highest <= QUOTA_ALLCLEAR_PERCENTAGE {
-                        self.set_config(Config::QuotaExceeding, None).await?;
+                        self.set_config_internal(Config::QuotaExceeding, None)
+                            .await?;
                     }
                 }
                 Err(err) => warn!(self, "cannot get highest quota usage: {:#}", err),
@@ -147,7 +151,7 @@ impl Context {
 
         *self.quota.write().await = Some(QuotaInfo {
             recent: quota,
-            modified: time(),
+            modified: tools::Time::now(),
         });
 
         self.emit_event(EventType::ConnectivityChanged);

src/reaction.rs

@@ -252,7 +252,7 @@ pub(crate) async fn set_msg_reaction(
     contact_id: ContactId,
     reaction: Reaction,
 ) -> Result<()> {
-    if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? {
+    if let Some((msg_id, _)) = rfc724_mid_exists(context, in_reply_to).await? {
         set_msg_id_reaction(context, msg_id, chat_id, contact_id, reaction).await
     } else {
         info!(
@@ -316,7 +316,7 @@ mod tests {
     use crate::contact::{Contact, ContactAddress, Origin};
     use crate::download::DownloadState;
     use crate::message::MessageState;
-    use crate::receive_imf::{receive_imf, receive_imf_inner};
+    use crate::receive_imf::{receive_imf, receive_imf_from_inbox};
     use crate::test_utils::TestContext;
     use crate::test_utils::TestContextManager;
 
@@ -425,7 +425,7 @@ Content-Disposition: reaction\n\
         let contacts = reactions.contacts();
         assert_eq!(contacts.len(), 1);
 
-        assert_eq!(contacts.get(0), Some(&bob_id));
+        assert_eq!(contacts.first(), Some(&bob_id));
         let bob_reaction = reactions.get(bob_id);
         assert_eq!(bob_reaction.is_empty(), false);
         assert_eq!(bob_reaction.emojis(), vec!["👍"]);
@@ -526,7 +526,7 @@ Here's my footer -- bob@example.net"
         assert_eq!(reactions.to_string(), "👍1");
         let contacts = reactions.contacts();
         assert_eq!(contacts.len(), 1);
-        let bob_id = contacts.get(0).unwrap();
+        let bob_id = contacts.first().unwrap();
         let bob_reaction = reactions.get(*bob_id);
         assert_eq!(bob_reaction.is_empty(), false);
         assert_eq!(bob_reaction.emojis(), vec!["👍"]);
@@ -568,7 +568,7 @@ Here's my footer -- bob@example.net"
         let msg_full = format!("{msg_header}\n\n100k text...");
 
         // Alice downloads message from Bob partially.
-        let alice_received_message = receive_imf_inner(
+        let alice_received_message = receive_imf_from_inbox(
             &alice,
             "first@example.org",
             msg_header.as_bytes(),
@@ -578,13 +578,13 @@ Here's my footer -- bob@example.net"
         )
         .await?
         .unwrap();
-        let alice_msg_id = *alice_received_message.msg_ids.get(0).unwrap();
+        let alice_msg_id = *alice_received_message.msg_ids.first().unwrap();
 
         // Bob downloads own message on the other device.
         let bob_received_message = receive_imf(&bob, msg_full.as_bytes(), false)
             .await?
             .unwrap();
-        let bob_msg_id = *bob_received_message.msg_ids.get(0).unwrap();
+        let bob_msg_id = *bob_received_message.msg_ids.first().unwrap();
 
         // Bob reacts to own message.
         send_reaction(&bob, bob_msg_id, "👍").await.unwrap();
@@ -599,7 +599,7 @@ Here's my footer -- bob@example.net"
         assert_eq!(msg.download_state(), DownloadState::Available);
 
         // Alice downloads full message.
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &alice,
             "first@example.org",
             msg_full.as_bytes(),

src/receive_imf.rs

@@ -1,6 +1,5 @@
 //! Internet Message Format reception pipeline.
 
-use std::cmp::min;
 use std::collections::HashSet;
 use std::convert::TryFrom;
 
@@ -10,6 +9,7 @@ use num_traits::FromPrimitive;
 use once_cell::sync::Lazy;
 use regex::Regex;
 
+use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, ShowEmails, DC_CHAT_ID_TRASH};
@@ -31,16 +31,14 @@ use crate::message::{
 };
 use crate::mimeparser::{parse_message_ids, AvatarAction, MimeMessage, SystemMessage};
 use crate::param::{Param, Params};
-use crate::peerstate::{Peerstate, PeerstateKeyType};
+use crate::peerstate::Peerstate;
 use crate::reaction::{set_msg_reaction, Reaction};
 use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
 use crate::simplify;
 use crate::sql;
 use crate::stock_str;
 use crate::sync::Sync::*;
-use crate::tools::{
-    buf_compress, extract_grpid_from_rfc724_mid, smeared_time, strip_rtlo_characters,
-};
+use crate::tools::{self, buf_compress, extract_grpid_from_rfc724_mid, strip_rtlo_characters};
 use crate::{contact, imap};
 
 /// This is the struct that is returned after receiving one email (aka MIME message).
@@ -89,7 +87,7 @@ pub async fn receive_imf(
                 .split("\r\n\r\n")
                 .next()
                 .context("No empty line in the message")?;
-            return receive_imf_inner(
+            return receive_imf_from_inbox(
                 context,
                 &rfc724_mid,
                 head.as_bytes(),
@@ -100,7 +98,32 @@ pub async fn receive_imf(
             .await;
         }
     }
-    receive_imf_inner(context, &rfc724_mid, imf_raw, seen, None, false).await
+    receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None, false).await
+}
+
+/// Emulates reception of a message from "INBOX".
+///
+/// Only used for tests and REPL tool, not actual message reception pipeline.
+pub(crate) async fn receive_imf_from_inbox(
+    context: &Context,
+    rfc724_mid: &str,
+    imf_raw: &[u8],
+    seen: bool,
+    is_partial_download: Option<u32>,
+    fetching_existing_messages: bool,
+) -> Result<Option<ReceivedMsg>> {
+    receive_imf_inner(
+        context,
+        "INBOX",
+        0,
+        0,
+        rfc724_mid,
+        imf_raw,
+        seen,
+        is_partial_download,
+        fetching_existing_messages,
+    )
+    .await
 }
 
 /// Inserts a tombstone into `msgs` table
@@ -130,17 +153,20 @@ async fn insert_tombstone(context: &Context, rfc724_mid: &str) -> Result<MsgId>
 /// returns `Ok(None)`.
 ///
 /// If `is_partial_download` is set, it contains the full message size in bytes.
-/// Do not confuse that with `replace_partial_download` that will be set when the full message is loaded later.
+/// Do not confuse that with `replace_msg_id` that will be set when the full message is loaded
+/// later.
+#[allow(clippy::too_many_arguments)]
 pub(crate) async fn receive_imf_inner(
     context: &Context,
+    folder: &str,
+    uidvalidity: u32,
+    uid: u32,
     rfc724_mid: &str,
     imf_raw: &[u8],
     seen: bool,
     is_partial_download: Option<u32>,
     fetching_existing_messages: bool,
 ) -> Result<Option<ReceivedMsg>> {
-    info!(context, "Receiving message, seen={seen}...");
-
     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
         info!(
             context,
@@ -173,28 +199,110 @@ pub(crate) async fn receive_imf_inner(
         Ok(mime_parser) => mime_parser,
     };
 
-    info!(context, "Received message has Message-Id: {rfc724_mid}");
+    crate::peerstate::maybe_do_aeap_transition(context, &mut mime_parser).await?;
+    if let Some(peerstate) = &mime_parser.decryption_info.peerstate {
+        peerstate
+            .handle_fingerprint_change(context, mime_parser.timestamp_sent)
+            .await?;
+        // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
+        // of an error. Otherwise we don't save peerstate until get here to reduce the number of
+        // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
+        // successfully.
+        if peerstate.prefer_encrypt != EncryptPreference::Mutual {
+            peerstate.save_to_db(&context.sql).await?;
+        }
+    }
+
+    let rfc724_mid_orig = &mime_parser
+        .get_rfc724_mid()
+        .unwrap_or(rfc724_mid.to_string());
+    info!(
+        context,
+        "Receiving message {rfc724_mid_orig:?}, seen={seen}...",
+    );
 
     // check, if the mail is already in our database.
     // make sure, this check is done eg. before securejoin-processing.
-    let (replace_partial_download, replace_chat_id) =
-        if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
-            let msg = Message::load_from_db(context, old_msg_id).await?;
-            if msg.download_state() != DownloadState::Done && is_partial_download.is_none() {
-                // the message was partially downloaded before and is fully downloaded now.
-                info!(
-                    context,
-                    "Message already partly in DB, replacing by full message."
-                );
-                (Some(old_msg_id), Some(msg.chat_id))
-            } else {
-                // the message was probably moved around.
-                info!(context, "Message already in DB, doing nothing.");
-                return Ok(None);
-            }
+    let (replace_msg_id, replace_chat_id);
+    if let Some((old_msg_id, _)) = message::rfc724_mid_exists(context, rfc724_mid).await? {
+        if is_partial_download.is_some() {
+            // Should never happen, see imap::prefetch_should_download(), but still.
+            info!(
+                context,
+                "Got a partial download and message is already in DB."
+            );
+            return Ok(None);
+        }
+        let msg = Message::load_from_db(context, old_msg_id).await?;
+        replace_msg_id = Some(old_msg_id);
+        replace_chat_id = if msg.download_state() != DownloadState::Done {
+            // the message was partially downloaded before and is fully downloaded now.
+            info!(
+                context,
+                "Message already partly in DB, replacing by full message."
+            );
+            Some(msg.chat_id)
         } else {
-            (None, None)
+            None
         };
+    } else {
+        replace_msg_id = if rfc724_mid_orig == rfc724_mid {
+            None
+        } else if let Some((old_msg_id, old_ts_sent)) =
+            message::rfc724_mid_exists(context, rfc724_mid_orig).await?
+        {
+            if imap::is_dup_msg(
+                mime_parser.has_chat_version(),
+                mime_parser.timestamp_sent,
+                old_ts_sent,
+            ) {
+                info!(context, "Deleting duplicate message {rfc724_mid_orig}.");
+                let target = context.get_delete_msgs_target().await?;
+                context
+                    .sql
+                    .execute(
+                        "UPDATE imap SET target=? WHERE folder=? AND uidvalidity=? AND uid=?",
+                        (target, folder, uidvalidity, uid),
+                    )
+                    .await?;
+            }
+            Some(old_msg_id)
+        } else {
+            None
+        };
+        replace_chat_id = None;
+    }
+
+    if replace_chat_id.is_some() {
+        // Need to update chat id in the db.
+    } else if let Some(msg_id) = replace_msg_id {
+        info!(context, "Message is already downloaded.");
+        if mime_parser.incoming {
+            return Ok(None);
+        }
+        // For the case if we missed a successful SMTP response. Be optimistic that the message is
+        // delivered also.
+        let self_addr = context.get_primary_self_addr().await?;
+        context
+            .sql
+            .execute(
+                "DELETE FROM smtp \
+                WHERE rfc724_mid=?1 AND (recipients LIKE ?2 OR recipients LIKE ('% ' || ?2))",
+                (rfc724_mid_orig, &self_addr),
+            )
+            .await?;
+        if !context
+            .sql
+            .exists(
+                "SELECT COUNT(*) FROM smtp WHERE rfc724_mid=?",
+                (rfc724_mid_orig,),
+            )
+            .await?
+        {
+            msg_id.set_delivered(context).await?;
+        }
+        return Ok(None);
+    };
 
     let prevent_rename =
         mime_parser.is_mailinglist_message() || mime_parser.get_header(HeaderDef::Sender).is_some();
@@ -219,12 +327,10 @@ pub(crate) async fn receive_imf_inner(
             }
         };
 
-    let incoming = from_id != ContactId::SELF;
-
     let to_ids = add_or_lookup_contacts_by_address_list(
         context,
         &mime_parser.recipients,
-        if !incoming {
+        if !mime_parser.incoming {
             Origin::OutgoingTo
         } else if incoming_origin.is_known() {
             Origin::IncomingTo
@@ -234,23 +340,12 @@ pub(crate) async fn receive_imf_inner(
     )
     .await?;
 
-    let rcvd_timestamp = smeared_time(context);
-
-    // Sender timestamp is allowed to be a bit in the future due to
-    // unsynchronized clocks, but not too much.
-    let sent_timestamp = mime_parser
-        .get_header(HeaderDef::Date)
-        .and_then(|value| mailparse::dateparse(value).ok())
-        .map_or(rcvd_timestamp, |value| min(value, rcvd_timestamp + 60));
-
     update_verified_keys(context, &mut mime_parser, from_id).await?;
 
     let received_msg;
-    if let Some(securejoin_step) = mime_parser.get_header(HeaderDef::SecureJoin) {
-        info!(context, "Received securejoin step {securejoin_step}.");
-
+    if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
         let res;
-        if incoming {
+        if mime_parser.incoming {
             res = handle_securejoin_handshake(context, &mime_parser, from_id)
                 .await
                 .context("error in Secure-Join message handling")?;
@@ -260,7 +355,7 @@ pub(crate) async fn receive_imf_inner(
             mime_parser.decryption_info.peerstate =
                 Peerstate::from_addr(context, contact.get_addr()).await?;
         } else {
-            let to_id = to_ids.get(0).copied().unwrap_or_default();
+            let to_id = to_ids.first().copied().unwrap_or_default();
             // handshake may mark contacts as verified and must be processed before chats are created
             res = observe_securejoin_on_other_device(context, &mime_parser, to_id)
                 .await
@@ -273,7 +368,7 @@ pub(crate) async fn receive_imf_inner(
                 received_msg = Some(ReceivedMsg {
                     chat_id: DC_CHAT_ID_TRASH,
                     state: MessageState::InSeen,
-                    sort_timestamp: sent_timestamp,
+                    sort_timestamp: mime_parser.timestamp_sent,
                     msg_ids: vec![msg_id],
                     needs_delete_job: res == securejoin::HandshakeMessage::Done,
                     #[cfg(test)]
@@ -291,6 +386,24 @@ pub(crate) async fn receive_imf_inner(
     let verified_encryption =
         has_verified_encryption(context, &mime_parser, from_id, &to_ids).await?;
 
+    if verified_encryption == VerifiedEncryption::Verified
+        && mime_parser.get_header(HeaderDef::ChatVerified).is_some()
+    {
+        if let Some(peerstate) = &mut mime_parser.decryption_info.peerstate {
+            // NOTE: it might be better to remember ID of the key
+            // that we used to decrypt the message, but
+            // it is unlikely that default key ever changes
+            // as it only happens when user imports a new default key.
+            //
+            // Backward verification is not security-critical,
+            // it is only needed to avoid adding user who does not
+            // have our key as verified to protected chats.
+            peerstate.backward_verified_key_id =
+                Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+            peerstate.save_to_db(&context.sql).await?;
+        }
+    }
+
     let received_msg = if let Some(received_msg) = received_msg {
         received_msg
     } else {
@@ -299,15 +412,12 @@ pub(crate) async fn receive_imf_inner(
             context,
             &mut mime_parser,
             imf_raw,
-            incoming,
             &to_ids,
-            rfc724_mid,
-            sent_timestamp,
-            rcvd_timestamp,
+            rfc724_mid_orig,
             from_id,
-            seen || replace_partial_download.is_some(),
+            seen,
             is_partial_download,
-            replace_partial_download,
+            replace_msg_id,
             fetching_existing_messages,
             prevent_rename,
             verified_encryption,
@@ -317,7 +427,7 @@ pub(crate) async fn receive_imf_inner(
     };
 
     if !from_id.is_special() {
-        contact::update_last_seen(context, from_id, sent_timestamp).await?;
+        contact::update_last_seen(context, from_id, mime_parser.timestamp_sent).await?;
     }
 
     // Update gossiped timestamp for the chat if someone else or our other device sent
@@ -328,15 +438,15 @@ pub(crate) async fn receive_imf_inner(
         && mime_parser
             .recipients
             .iter()
-            .all(|recipient| mime_parser.gossiped_addr.contains(&recipient.addr))
+            .all(|recipient| mime_parser.gossiped_keys.contains_key(&recipient.addr))
     {
         info!(
             context,
-            "Received message contains Autocrypt-Gossip for all members, updating timestamp."
+            "Received message contains Autocrypt-Gossip for all members of {chat_id}, updating timestamp."
         );
-        if chat_id.get_gossiped_timestamp(context).await? < sent_timestamp {
+        if chat_id.get_gossiped_timestamp(context).await? < mime_parser.timestamp_sent {
             chat_id
-                .set_gossiped_timestamp(context, sent_timestamp)
+                .set_gossiped_timestamp(context, mime_parser.timestamp_sent)
                 .await?;
         }
     }
@@ -373,7 +483,11 @@ pub(crate) async fn receive_imf_inner(
     if let Some(avatar_action) = &mime_parser.user_avatar {
         if from_id != ContactId::UNDEFINED
             && context
-                .update_contacts_timestamp(from_id, Param::AvatarTimestamp, sent_timestamp)
+                .update_contacts_timestamp(
+                    from_id,
+                    Param::AvatarTimestamp,
+                    mime_parser.timestamp_sent,
+                )
                 .await?
         {
             if let Err(err) = contact::set_profile_image(
@@ -394,7 +508,11 @@ pub(crate) async fn receive_imf_inner(
         if !mime_parser.is_mailinglist_message()
             && from_id != ContactId::UNDEFINED
             && context
-                .update_contacts_timestamp(from_id, Param::StatusTimestamp, sent_timestamp)
+                .update_contacts_timestamp(
+                    from_id,
+                    Param::StatusTimestamp,
+                    mime_parser.timestamp_sent,
+                )
                 .await?
         {
             if let Err(err) = contact::set_status(
@@ -415,20 +533,34 @@ pub(crate) async fn receive_imf_inner(
     let delete_server_after = context.get_config_delete_server_after().await?;
 
     if !received_msg.msg_ids.is_empty() {
-        if received_msg.needs_delete_job
+        let target = if received_msg.needs_delete_job
             || (delete_server_after == Some(0) && is_partial_download.is_none())
         {
-            let target = context.get_delete_msgs_target().await?;
+            Some(context.get_delete_msgs_target().await?)
+        } else {
+            None
+        };
+        if target.is_some() || rfc724_mid_orig != rfc724_mid {
+            let target_subst = match &target {
+                Some(_) => "target=?1,",
+                None => "",
+            };
             context
                 .sql
                 .execute(
-                    "UPDATE imap SET target=? WHERE rfc724_mid=?",
-                    (target, rfc724_mid),
+                    &format!("UPDATE imap SET {target_subst} rfc724_mid=?2 WHERE rfc724_mid=?3"),
+                    (
+                        target.as_deref().unwrap_or_default(),
+                        rfc724_mid_orig,
+                        rfc724_mid,
+                    ),
                 )
                 .await?;
-        } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+        }
+        if target.is_none() && !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version()
+        {
             // This is a Delta Chat MDN. Mark as read.
-            markseen_on_imap_table(context, rfc724_mid).await?;
+            markseen_on_imap_table(context, rfc724_mid_orig).await?;
         }
     }
 
@@ -437,16 +569,18 @@ pub(crate) async fn receive_imf_inner(
     } else if !chat_id.is_trash() {
         let fresh = received_msg.state == MessageState::InFresh;
         for msg_id in &received_msg.msg_ids {
-            chat_id.emit_msg_event(context, *msg_id, incoming && fresh);
+            chat_id.emit_msg_event(context, *msg_id, mime_parser.incoming && fresh);
         }
     }
     context.new_msgs_notify.notify_one();
 
     mime_parser
-        .handle_reports(context, from_id, sent_timestamp, &mime_parser.parts)
+        .handle_reports(context, from_id, &mime_parser.parts)
         .await;
 
-    from_id.mark_bot(context, mime_parser.is_bot).await?;
+    if let Some(is_bot) = mime_parser.is_bot {
+        from_id.mark_bot(context, is_bot).await?;
+    }
 
     Ok(Some(received_msg))
 }
@@ -511,11 +645,8 @@ async fn add_parts(
     context: &Context,
     mime_parser: &mut MimeMessage,
     imf_raw: &[u8],
-    incoming: bool,
     to_ids: &[ContactId],
     rfc724_mid: &str,
-    sent_timestamp: i64,
-    rcvd_timestamp: i64,
     from_id: ContactId,
     seen: bool,
     is_partial_download: Option<u32>,
@@ -524,6 +655,10 @@ async fn add_parts(
     prevent_rename: bool,
     verified_encryption: VerifiedEncryption,
 ) -> Result<ReceivedMsg> {
+    let rfc724_mid_orig = &mime_parser
+        .get_rfc724_mid()
+        .unwrap_or(rfc724_mid.to_string());
+
     let mut chat_id = None;
     let mut chat_id_blocked = Blocked::Not;
 
@@ -577,8 +712,9 @@ async fn add_parts(
     // (of course, the user can add other chats manually later)
     let to_id: ContactId;
     let state: MessageState;
+    let mut hidden = false;
     let mut needs_delete_job = false;
-    if incoming {
+    if mime_parser.incoming {
         to_id = ContactId::SELF;
 
         let test_normal_chat = if from_id == ContactId::UNDEFINED {
@@ -638,7 +774,7 @@ async fn add_parts(
             create_blocked_default
         };
 
-        if chat_id.is_none() {
+        if chat_id.is_none() && !is_mdn {
             // try to create a group
 
             if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group(
@@ -654,7 +790,6 @@ async fn add_parts(
                 from_id,
                 to_ids,
                 &verified_encryption,
-                sent_timestamp,
             )
             .await?
             {
@@ -700,7 +835,6 @@ async fn add_parts(
             group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
-                sent_timestamp,
                 group_chat_id,
                 from_id,
                 to_ids,
@@ -718,7 +852,6 @@ async fn add_parts(
                     allow_creation,
                     mailinglist_header,
                     mime_parser,
-                    sent_timestamp,
                 )
                 .await?
                 {
@@ -729,7 +862,7 @@ async fn add_parts(
         }
 
         if let Some(chat_id) = chat_id {
-            apply_mailinglist_changes(context, mime_parser, sent_timestamp, chat_id).await?;
+            apply_mailinglist_changes(context, mime_parser, chat_id).await?;
         }
 
         // if contact renaming is prevented (for mailinglists and bots),
@@ -815,11 +948,13 @@ async fn add_parts(
                         // The message itself will be sorted under the device message since the device
                         // message is `MessageState::InNoticed`, which means that all following
                         // messages are sorted under it.
-                        let sort_timestamp =
-                            calc_sort_timestamp(context, sent_timestamp, chat_id, true, incoming)
-                                .await?;
                         chat_id
-                            .set_protection(context, new_protection, sort_timestamp, Some(from_id))
+                            .set_protection(
+                                context,
+                                new_protection,
+                                mime_parser.timestamp_sent,
+                                Some(from_id),
+                            )
                             .await?;
                     }
                 }
@@ -843,7 +978,7 @@ async fn add_parts(
         // the mail is on the IMAP server, probably it is also delivered.
         // We cannot recreate other states (read, error).
         state = MessageState::OutDelivered;
-        to_id = to_ids.get(0).copied().unwrap_or_default();
+        to_id = to_ids.first().copied().unwrap_or_default();
 
         let self_sent =
             from_id == ContactId::SELF && to_ids.len() == 1 && to_ids.contains(&ContactId::SELF);
@@ -876,6 +1011,37 @@ async fn add_parts(
             }
         }
 
+        if mime_parser.decrypting_failed && !fetching_existing_messages {
+            if chat_id.is_none() {
+                chat_id = Some(DC_CHAT_ID_TRASH);
+            } else {
+                hidden = true;
+            }
+            let last_time = context
+                .get_config_i64(Config::LastCantDecryptOutgoingMsgs)
+                .await?;
+            let now = tools::time();
+            let update_config = if last_time.saturating_add(24 * 60 * 60) <= now {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = stock_str::cant_decrypt_outgoing_msgs(context).await;
+                chat::add_device_msg(context, None, Some(&mut msg))
+                    .await
+                    .log_err(context)
+                    .ok();
+                true
+            } else {
+                last_time > now
+            };
+            if update_config {
+                context
+                    .set_config_internal(
+                        Config::LastCantDecryptOutgoingMsgs,
+                        Some(&now.to_string()),
+                    )
+                    .await?;
+            }
+        }
+
         if !to_ids.is_empty() {
             if chat_id.is_none() {
                 if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group(
@@ -887,7 +1053,6 @@ async fn add_parts(
                     from_id,
                     to_ids,
                     &verified_encryption,
-                    sent_timestamp,
                 )
                 .await?
                 {
@@ -925,7 +1090,6 @@ async fn add_parts(
             group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
-                sent_timestamp,
                 chat_id,
                 from_id,
                 to_ids,
@@ -1014,8 +1178,15 @@ async fn add_parts(
     };
 
     let in_fresh = state == MessageState::InFresh;
-    let sort_timestamp =
-        calc_sort_timestamp(context, sent_timestamp, chat_id, false, incoming).await?;
+    let sort_to_bottom = false;
+    let sort_timestamp = chat_id
+        .calc_sort_timestamp(
+            context,
+            mime_parser.timestamp_sent,
+            sort_to_bottom,
+            mime_parser.incoming,
+        )
+        .await?;
 
     // Apply ephemeral timer changes to the chat.
     //
@@ -1045,7 +1216,11 @@ async fn add_parts(
                 "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} to avoid rollback.",
             );
         } else if chat_id
-            .update_timestamp(context, Param::EphemeralSettingsTimestamp, sent_timestamp)
+            .update_timestamp(
+                context,
+                Param::EphemeralSettingsTimestamp,
+                mime_parser.timestamp_sent,
+            )
             .await?
         {
             if let Err(err) = chat_id
@@ -1103,7 +1278,7 @@ async fn add_parts(
         //    -> Showing info messages everytime would be a lot of noise
         // 3. The info messages that are shown to the user ("Your chat partner
         //    likely reinstalled DC" or similar) would be wrong.
-        if chat.is_protected() && (incoming || chat.typ != Chattype::Single) {
+        if chat.is_protected() && (mime_parser.incoming || chat.typ != Chattype::Single) {
             if let VerifiedEncryption::NotVerified(err) = verified_encryption {
                 warn!(context, "Verification problem: {err:#}.");
                 let s = format!("{err}. See 'Info' for more details");
@@ -1248,7 +1423,7 @@ async fn add_parts(
             match ephemeral_timer {
                 EphemeralTimer::Disabled => 0,
                 EphemeralTimer::Enabled { duration } => {
-                    rcvd_timestamp.saturating_add(duration.into())
+                    mime_parser.timestamp_rcvd.saturating_add(duration.into())
                 }
             }
         };
@@ -1269,7 +1444,7 @@ INSERT INTO msgs
     rfc724_mid, chat_id,
     from_id, to_id, timestamp, timestamp_sent, 
     timestamp_rcvd, type, state, msgrmsg, 
-    txt, subject, txt_raw, param, 
+    txt, subject, txt_raw, param, hidden,
     bytes, mime_headers, mime_compressed, mime_in_reply_to,
     mime_references, mime_modified, error, ephemeral_timer,
     ephemeral_timestamp, download_state, hop_info
@@ -1278,7 +1453,7 @@ INSERT INTO msgs
     ?,
     ?, ?, ?, ?,
     ?, ?, ?, ?,
-    ?, ?, ?, ?,
+    ?, ?, ?, ?, ?,
     ?, ?, ?, ?, 1,
     ?, ?, ?, ?,
     ?, ?, ?, ?
@@ -1288,7 +1463,7 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
     from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent,
     type=excluded.type, msgrmsg=excluded.msgrmsg,
     txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param,
-    bytes=excluded.bytes, mime_headers=excluded.mime_headers,
+    hidden=excluded.hidden,bytes=excluded.bytes, mime_headers=excluded.mime_headers,
     mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to,
     mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer,
     ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info
@@ -1296,13 +1471,13 @@ RETURNING id
 "#)?;
                 let row_id: MsgId = stmt.query_row(params![
                     replace_msg_id,
-                    rfc724_mid,
+                    rfc724_mid_orig,
                     if trash { DC_CHAT_ID_TRASH } else { chat_id },
                     if trash { ContactId::UNDEFINED } else { from_id },
                     if trash { ContactId::UNDEFINED } else { to_id },
                     sort_timestamp,
-                    sent_timestamp,
-                    rcvd_timestamp,
+                    mime_parser.timestamp_sent,
+                    mime_parser.timestamp_rcvd,
                     typ,
                     state,
                     is_dc_message,
@@ -1315,6 +1490,7 @@ RETURNING id
                     } else {
                         param.to_string()
                     },
+                    hidden,
                     part.bytes as isize,
                     if (save_mime_headers || mime_modified) && !trash {
                         mime_headers.clone()
@@ -1380,7 +1556,7 @@ RETURNING id
     );
 
     // new outgoing message from another device marks the chat as noticed.
-    if !incoming && !chat_id.is_special() {
+    if !mime_parser.incoming && !chat_id.is_special() {
         chat::marknoticed_chat_if_older_than(context, chat_id, sort_timestamp).await?;
     }
 
@@ -1403,7 +1579,7 @@ RETURNING id
         }
     }
 
-    if !incoming && is_mdn && is_dc_message == MessengerMessage::Yes {
+    if !mime_parser.incoming && is_mdn && is_dc_message == MessengerMessage::Yes {
         // Normally outgoing MDNs sent by us never appear in mailboxes, but Gmail saves all
         // outgoing messages, including MDNs, to the Sent folder. If we detect such saved MDN,
         // delete it.
@@ -1474,53 +1650,6 @@ async fn save_locations(
     Ok(())
 }
 
-async fn calc_sort_timestamp(
-    context: &Context,
-    message_timestamp: i64,
-    chat_id: ChatId,
-    always_sort_to_bottom: bool,
-    incoming: bool,
-) -> Result<i64> {
-    let mut sort_timestamp = min(message_timestamp, smeared_time(context));
-
-    let last_msg_time: Option<i64> = if always_sort_to_bottom {
-        // get newest message for this chat
-        context
-            .sql
-            .query_get_value(
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=?",
-                (chat_id,),
-            )
-            .await?
-    } else if incoming {
-        // get newest non fresh message for this chat.
-
-        // If a user hasn't been online for some time, the Inbox is fetched first and then the
-        // Sentbox. In order for Inbox and Sent messages to be allowed to mingle, outgoing messages
-        // are purely sorted by their sent timestamp. NB: The Inbox must be fetched first otherwise
-        // Inbox messages would be always below old Sentbox messages. We could take in the query
-        // below only incoming messages, but then new incoming messages would mingle with just sent
-        // outgoing ones and apear somewhere in the middle of the chat.
-        context
-            .sql
-            .query_get_value(
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
-                (chat_id, MessageState::InFresh),
-            )
-            .await?
-    } else {
-        None
-    };
-
-    if let Some(last_msg_time) = last_msg_time {
-        if last_msg_time > sort_timestamp {
-            sort_timestamp = last_msg_time;
-        }
-    }
-
-    Ok(sort_timestamp)
-}
-
 async fn lookup_chat_by_reply(
     context: &Context,
     mime_parser: &MimeMessage,
@@ -1533,27 +1662,10 @@ async fn lookup_chat_by_reply(
     let Some(parent) = parent else {
         return Ok(None);
     };
-
-    let parent_chat = Chat::load_from_db(context, parent.chat_id).await?;
-
-    if parent.download_state != DownloadState::Done
-        // TODO (2023-09-12): Added for backward compatibility with versions that did not have
-        // `DownloadState::Undecipherable`. Remove eventually with the comment in
-        // `MimeMessage::from_bytes()`.
-        || parent
-            .error
-            .as_ref()
-            .filter(|e| e.starts_with("Decrypting failed:"))
-            .is_some()
-    {
-        // If the parent msg is not fully downloaded or undecipherable, it may have been
-        // assigned to the wrong chat (they often get assigned to the 1:1 chat with the sender).
+    let Some(parent_chat_id) = ChatId::lookup_by_message(parent) else {
         return Ok(None);
-    }
-
-    if parent_chat.id == DC_CHAT_ID_TRASH {
-        return Ok(None);
-    }
+    };
+    let parent_chat = Chat::load_from_db(context, parent_chat_id).await?;
 
     // If this was a private message just to self, it was probably a private reply.
     // It should not go into the group then, but into the private chat.
@@ -1626,7 +1738,6 @@ async fn create_or_lookup_group(
     from_id: ContactId,
     to_ids: &[ContactId],
     verified_encryption: &VerifiedEncryption,
-    timestamp: i64,
 ) -> Result<Option<(ChatId, Blocked)>> {
     let grpid = if let Some(grpid) = try_getting_grpid(mime_parser) {
         grpid
@@ -1639,7 +1750,7 @@ async fn create_or_lookup_group(
             member_ids.push(ContactId::SELF);
         }
 
-        let res = create_adhoc_group(context, mime_parser, create_blocked, &member_ids, timestamp)
+        let res = create_adhoc_group(context, mime_parser, create_blocked, &member_ids)
             .await
             .context("could not create ad hoc group")?
             .map(|chat_id| (chat_id, create_blocked));
@@ -1721,7 +1832,7 @@ async fn create_or_lookup_group(
             create_blocked,
             create_protected,
             None,
-            timestamp,
+            mime_parser.timestamp_sent,
         )
         .await
         .with_context(|| format!("Failed to create group '{grpname}' for grpid={grpid}"))?;
@@ -1768,7 +1879,6 @@ async fn create_or_lookup_group(
 async fn apply_group_changes(
     context: &Context,
     mime_parser: &mut MimeMessage,
-    sent_timestamp: i64,
     chat_id: ChatId,
     from_id: ContactId,
     to_ids: &[ContactId],
@@ -1806,7 +1916,11 @@ async fn apply_group_changes(
     let allow_member_list_changes = !is_partial_download
         && is_from_in_chat
         && chat_id
-            .update_timestamp(context, Param::MemberListTimestamp, sent_timestamp)
+            .update_timestamp(
+                context,
+                Param::MemberListTimestamp,
+                mime_parser.timestamp_sent,
+            )
             .await?;
 
     // Whether to rebuild the member list from scratch.
@@ -1845,7 +1959,7 @@ async fn apply_group_changes(
                 .set_protection(
                     context,
                     ProtectionStatus::Protected,
-                    smeared_time(context),
+                    mime_parser.timestamp_sent,
                     Some(from_id),
                 )
                 .await?;
@@ -1899,7 +2013,11 @@ async fn apply_group_changes(
             .filter(|grpname| grpname.len() < 200)
         {
             if chat_id
-                .update_timestamp(context, Param::GroupNameTimestamp, sent_timestamp)
+                .update_timestamp(
+                    context,
+                    Param::GroupNameTimestamp,
+                    mime_parser.timestamp_sent,
+                )
                 .await?
             {
                 info!(context, "Updating grpname for chat {chat_id}.");
@@ -2006,7 +2124,7 @@ async fn apply_group_changes(
             info!(context, "Group-avatar change for {chat_id}.");
             if chat
                 .param
-                .update_timestamp(Param::AvatarTimestamp, sent_timestamp)?
+                .update_timestamp(Param::AvatarTimestamp, mime_parser.timestamp_sent)?
             {
                 match avatar_action {
                     AvatarAction::Change(profile_image) => {
@@ -2055,7 +2173,6 @@ async fn create_or_lookup_mailinglist(
     allow_creation: bool,
     list_id_header: &str,
     mime_parser: &MimeMessage,
-    timestamp: i64,
 ) -> Result<Option<(ChatId, Blocked)>> {
     let listid = mailinglist_header_listid(list_id_header)?;
 
@@ -2087,7 +2204,7 @@ async fn create_or_lookup_mailinglist(
             blocked,
             ProtectionStatus::Unprotected,
             param,
-            timestamp,
+            mime_parser.timestamp_sent,
         )
         .await
         .with_context(|| {
@@ -2175,7 +2292,6 @@ fn compute_mailinglist_name(
 async fn apply_mailinglist_changes(
     context: &Context,
     mime_parser: &MimeMessage,
-    sent_timestamp: i64,
     chat_id: ChatId,
 ) -> Result<()> {
     let Some(mailinglist_header) = mime_parser.get_mailinglist_header() else {
@@ -2191,7 +2307,11 @@ async fn apply_mailinglist_changes(
     let new_name = compute_mailinglist_name(mailinglist_header, listid, mime_parser);
     if chat.name != new_name
         && chat_id
-            .update_timestamp(context, Param::GroupNameTimestamp, sent_timestamp)
+            .update_timestamp(
+                context,
+                Param::GroupNameTimestamp,
+                mime_parser.timestamp_sent,
+            )
             .await?
     {
         info!(context, "Updating listname for chat {chat_id}.");
@@ -2272,14 +2392,8 @@ async fn create_adhoc_group(
     mime_parser: &MimeMessage,
     create_blocked: Blocked,
     member_ids: &[ContactId],
-    timestamp: i64,
 ) -> Result<Option<ChatId>> {
     if mime_parser.is_mailinglist_message() {
-        info!(
-            context,
-            "Not creating ad-hoc group for mailing list message."
-        );
-
         return Ok(None);
     }
 
@@ -2300,7 +2414,6 @@ async fn create_adhoc_group(
     }
 
     if member_ids.len() < 3 {
-        info!(context, "Not creating ad-hoc group: too few contacts.");
         return Ok(None);
     }
 
@@ -2317,9 +2430,14 @@ async fn create_adhoc_group(
         create_blocked,
         ProtectionStatus::Unprotected,
         None,
-        timestamp,
+        mime_parser.timestamp_sent,
     )
     .await?;
+
+    info!(
+        context,
+        "Created ad-hoc group id={new_chat_id}, name={grpname:?}."
+    );
     chat::add_to_chat_contacts_table(context, new_chat_id, member_ids).await?;
 
     context.emit_event(EventType::ChatModified(new_chat_id));
@@ -2327,6 +2445,7 @@ async fn create_adhoc_group(
     Ok(Some(new_chat_id))
 }
 
+#[derive(Debug, PartialEq, Eq)]
 enum VerifiedEncryption {
     Verified,
     NotVerified(String), // The string contains the reason why it's not verified
@@ -2483,7 +2602,7 @@ async fn mark_recipients_as_verified(
 
     for (to_addr, is_verified) in rows {
         // mark gossiped keys (if any) as verified
-        if mimeparser.gossiped_addr.contains(&to_addr.to_lowercase()) {
+        if let Some(gossiped_key) = mimeparser.gossiped_keys.get(&to_addr.to_lowercase()) {
             if let Some(mut peerstate) = Peerstate::from_addr(context, &to_addr).await? {
                 // If we're here, we know the gossip key is verified.
                 //
@@ -2498,24 +2617,29 @@ async fn mark_recipients_as_verified(
                 if !is_verified {
                     info!(context, "{verifier_addr} has verified {to_addr}.");
                     if let Some(fp) = peerstate.gossip_key_fingerprint.clone() {
-                        peerstate.set_verified(PeerstateKeyType::GossipKey, fp, verifier_addr)?;
+                        peerstate.set_verified(gossiped_key.clone(), fp, verifier_addr)?;
+                        peerstate.backward_verified_key_id =
+                            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
                         peerstate.save_to_db(&context.sql).await?;
 
-                        if !is_verified {
-                            let (to_contact_id, _) = Contact::add_or_lookup(
-                                context,
-                                "",
-                                &ContactAddress::new(&to_addr)?,
-                                Origin::Hidden,
-                            )
-                            .await?;
-                            ChatId::set_protection_for_contact(context, to_contact_id).await?;
-                        }
+                        let (to_contact_id, _) = Contact::add_or_lookup(
+                            context,
+                            "",
+                            &ContactAddress::new(&to_addr)?,
+                            Origin::Hidden,
+                        )
+                        .await?;
+                        ChatId::set_protection_for_contact(
+                            context,
+                            to_contact_id,
+                            mimeparser.timestamp_sent,
+                        )
+                        .await?;
                     }
                 } else {
                     // The contact already has a verified key.
                     // Store gossiped key as the secondary verified key.
-                    peerstate.set_secondary_verified_key_from_gossip(verifier_addr);
+                    peerstate.set_secondary_verified_key(gossiped_key.clone(), verifier_addr);
                     peerstate.save_to_db(&context.sql).await?;
                 }
             }
@@ -2536,7 +2660,7 @@ async fn get_previous_message(
 ) -> Result<Option<Message>> {
     if let Some(field) = mime_parser.get_header(HeaderDef::References) {
         if let Some(rfc724mid) = parse_message_ids(field).last() {
-            if let Some(msg_id) = rfc724_mid_exists(context, rfc724mid).await? {
+            if let Some((msg_id, _)) = rfc724_mid_exists(context, rfc724mid).await? {
                 return Ok(Some(Message::load_from_db(context, msg_id).await?));
             }
         }
@@ -2548,20 +2672,7 @@ async fn get_previous_message(
 ///
 /// Only messages that are not in the trash chat are considered.
 async fn get_rfc724_mid_in_list(context: &Context, mid_list: &str) -> Result<Option<Message>> {
-    if mid_list.is_empty() {
-        return Ok(None);
-    }
-
-    for id in parse_message_ids(mid_list).iter().rev() {
-        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
-            let msg = Message::load_from_db(context, msg_id).await?;
-            if msg.chat_id != DC_CHAT_ID_TRASH {
-                return Ok(Some(msg));
-            }
-        }
-    }
-
-    Ok(None)
+    message::get_latest_by_rfc724_mids(context, &parse_message_ids(mid_list)).await
 }
 
 /// Returns the last message referenced from References: header found in the database.

src/receive_imf/tests.rs

@@ -2,11 +2,11 @@ use tokio::fs;
 
 use super::*;
 use crate::aheader::EncryptPreference;
+use crate::chat::{self, get_chat_msgs, ChatItem, ChatVisibility};
 use crate::chat::{
     add_contact_to_chat, add_to_chat_contacts_table, create_group_chat, get_chat_contacts,
     is_contact_in_chat, remove_contact_from_chat, send_text_msg,
 };
-use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::{DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS};
@@ -28,11 +28,24 @@ async fn test_grpid_simple() {
     let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
         .await
         .unwrap();
+    assert_eq!(mimeparser.incoming, true);
     assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), None);
     let grpid = Some("HcxyMARjyJy");
     assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid);
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_outgoing() -> Result<()> {
+    let context = TestContext::new_alice().await;
+    let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: alice@example.org\n\
+                \n\
+                hello";
+    let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await?;
+    assert_eq!(mimeparser.incoming, false);
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_bad_from() {
     let context = TestContext::new_alice().await;
@@ -310,6 +323,56 @@ async fn test_read_receipt_and_unarchive() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_mdn_and_alias() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    let bob = tcm.bob().await;
+    let alice_chat = alice.create_chat(&bob).await;
+    let sent = alice.send_text(alice_chat.id, "alice -> bob").await;
+    let msg_id = sent.sender_msg_id;
+    receive_imf(
+        &alice,
+        format!(
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+             From: bob@example.net\n\
+             To: alicechat@example.org\n\
+             Subject: message opened\n\
+             Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+             Chat-Version: 1.0\n\
+             Message-ID: <aranudiaerudiaduiaertd@example.com>\n\
+             Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+             \n\
+             \n\
+             --SNIPP\n\
+             Content-Type: text/plain; charset=utf-8\n\
+             \n\
+             Read receipts do not guarantee sth. was read.\n\
+             \n\
+             \n\
+             --SNIPP\n\
+             Content-Type: message/disposition-notification\n\
+             \n\
+             Reporting-UA: Delta Chat 1.28.0\n\
+             Original-Recipient: rfc822;bob@example.com\n\
+             Final-Recipient: rfc822;bob@example.com\n\
+             Original-Message-ID: <{msg_id}>\n\
+             Disposition: manual-action/MDN-sent-automatically; displayed\n\
+             \n\
+             \n\
+             --SNIPP--",
+        )
+        .as_bytes(),
+        false,
+    )
+    .await?;
+
+    let chats = Chatlist::try_load(&alice, 0, None, None).await?;
+    assert_eq!(chats.len(), 1);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_no_from() {
     // if there is no from given, from_id stays 0 which is just fine. These messages
@@ -628,7 +691,7 @@ async fn test_parse_ndn(
     rfc724_mid_outgoing: &str,
     raw_ndn: &[u8],
     error_msg: Option<&str>,
-) {
+) -> (TestContext, MsgId) {
     let t = TestContext::new().await;
     t.configure_addr(self_addr).await;
 
@@ -675,6 +738,40 @@ async fn test_parse_ndn(
     );
 
     assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
+    (t, msg_id)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_resend_after_ndn() -> Result<()> {
+    let (t, msg_id) = test_parse_ndn(
+            "alice@testrun.org",
+            "hcksocnsofoejx@five.chat",
+            "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
+            include_bytes!("../../test-data/message/testrun_ndn.eml"),
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"),
+        )
+        .await;
+    chat::resend_msgs(&t, &[msg_id]).await?;
+    let msg = Message::load_from_db(&t, msg_id).await?;
+    assert_eq!(msg.state, MessageState::OutPending);
+    assert_eq!(msg.error(), None);
+    // Alice receives a BCC-self copy of their message.
+    receive_imf(
+        &t,
+        "To: hcksocnsofoejx@five.chat\n\
+        From: alice@testrun.org\n\
+        Date: Today, 2 January 2024 00:00:00 -300\n\
+        Message-ID: Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org\n\
+        \n\
+        hi"
+        .as_bytes(),
+        false,
+    )
+    .await?;
+    let msg = t.get_last_msg().await;
+    assert_eq!(msg.state, MessageState::OutDelivered);
+    assert_eq!(msg.error(), None);
+    Ok(())
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1798,7 +1895,7 @@ async fn create_test_alias(chat_request: bool, group_request: bool) -> (TestCont
         .await
         .unwrap();
 
-    let msg_id = rfc724_mid_exists(&claire, "non-dc-1@example.org")
+    let (msg_id, _) = rfc724_mid_exists(&claire, "non-dc-1@example.org")
         .await
         .unwrap()
         .unwrap();
@@ -2600,6 +2697,36 @@ async fn test_read_receipts_dont_create_chats() -> Result<()> {
     Ok(())
 }
 
+/// Test that read receipts don't unmark contacts as bots.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_read_receipts_dont_unmark_bots() -> Result<()> {
+    let alice = &TestContext::new_alice().await;
+    let bob = &TestContext::new_bob().await;
+    let ab_contact = alice.add_or_lookup_contact(bob).await;
+    ab_contact.id.mark_bot(alice, true).await?;
+    let alice_chat = alice.create_chat(bob).await;
+
+    // Alice sends and Bob receives a message.
+    bob.recv_msg(&alice.send_text(alice_chat.id, "Message").await)
+        .await;
+    let received_msg = bob.get_last_msg().await;
+
+    // Bob sends a read receipt.
+    let mdn_mimefactory =
+        crate::mimefactory::MimeFactory::from_mdn(bob, &received_msg, vec![]).await?;
+    let rendered_mdn = mdn_mimefactory.render(bob).await?;
+    let mdn_body = rendered_mdn.message;
+
+    // Alice receives the read receipt.
+    receive_imf(alice, mdn_body.as_bytes(), false).await?;
+    let msg = alice.get_last_msg_in(alice_chat.id).await;
+    assert_eq!(msg.state, MessageState::OutMdnRcvd);
+    let ab_contact = alice.add_or_lookup_contact(bob).await;
+    assert!(ab_contact.is_bot());
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_gmx_forwarded_msg() -> Result<()> {
     let t = TestContext::new_alice().await;
@@ -3105,6 +3232,42 @@ async fn test_blocked_contact_creates_group() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_outgoing_undecryptable() -> Result<()> {
+    let alice = &TestContext::new().await;
+    alice.configure_addr("alice@example.org").await;
+
+    let raw = include_bytes!("../../test-data/message/thunderbird_with_autocrypt.eml");
+    receive_imf(alice, raw, false).await?;
+
+    let bob_contact_id = Contact::lookup_id_by_addr(alice, "bob@example.net", Origin::OutgoingTo)
+        .await?
+        .unwrap();
+    assert!(ChatId::lookup_by_contact(alice, bob_contact_id)
+        .await?
+        .is_none());
+
+    let dev_chat_id = ChatId::lookup_by_contact(alice, ContactId::DEVICE)
+        .await?
+        .unwrap();
+    let dev_msg = alice.get_last_msg_in(dev_chat_id).await;
+    assert!(dev_msg.error().is_none());
+    assert!(dev_msg
+        .text
+        .contains(&stock_str::cant_decrypt_outgoing_msgs(alice).await));
+
+    let raw = include_bytes!("../../test-data/message/thunderbird_encrypted_signed.eml");
+    receive_imf(alice, raw, false).await?;
+
+    assert!(ChatId::lookup_by_contact(alice, bob_contact_id)
+        .await?
+        .is_none());
+    // The device message mustn't be added too frequently.
+    assert_eq!(alice.get_last_msg_in(dev_chat_id).await.id, dev_msg.id);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_thunderbird_autocrypt() -> Result<()> {
     let t = TestContext::new_bob().await;
@@ -3223,6 +3386,22 @@ async fn test_thunderbird_unsigned_with_unencrypted_subject() -> Result<()> {
     Ok(())
 }
 
+/// Tests that DC takes the correct Message-ID from the encrypted message part, not the unencrypted
+/// one messed up by the server.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_messed_up_message_id() -> Result<()> {
+    let t = TestContext::new_bob().await;
+
+    let raw = include_bytes!("../../test-data/message/messed_up_message_id.eml");
+    receive_imf(&t, raw, false).await?;
+    assert_eq!(
+        t.get_last_msg().await.rfc724_mid,
+        "0bb9ffe1-2596-d997-95b4-1fef8cc4808e@example.org"
+    );
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_mua_user_adds_member() -> Result<()> {
     let t = TestContext::new_alice().await;
@@ -3945,7 +4124,7 @@ async fn test_partial_group_consistency() -> Result<()> {
         .unwrap();
 
     // Bob receives partial message.
-    let msg_id = receive_imf_inner(
+    let msg_id = receive_imf_from_inbox(
         &bob,
         "first@example.org",
         b"From: Alice <alice@example.org>\n\
@@ -3998,7 +4177,7 @@ Chat-Group-Member-Added: charlie@example.com",
     assert_eq!(contacts.len(), 3);
 
     // Bob fully reives the partial message.
-    let msg_id = receive_imf_inner(
+    let msg_id = receive_imf_from_inbox(
         &bob,
         "first@example.org",
         b"From: Alice <alice@example.org>\n\

src/scheduler.rs

@@ -2,6 +2,7 @@ use std::cmp;
 use std::iter::{self, once};
 use std::num::NonZeroUsize;
 use std::sync::atomic::Ordering;
+use std::time::Duration;
 
 use anyhow::{bail, Context as _, Error, Result};
 use async_channel::{self as channel, Receiver, Sender};
@@ -12,10 +13,10 @@ use tokio::sync::{oneshot, RwLock, RwLockWriteGuard};
 use tokio::task;
 
 use self::connectivity::ConnectivityStore;
-use crate::config::Config;
+use crate::config::{self, Config};
 use crate::contact::{ContactId, RecentlySeenLoop};
 use crate::context::Context;
-use crate::download::download_msg;
+use crate::download::{download_msg, DownloadState};
 use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::events::EventType;
 use crate::imap::{FolderMeaning, Imap};
@@ -24,7 +25,7 @@ use crate::log::LogExt;
 use crate::message::MsgId;
 use crate::smtp::{send_smtp_messages, Smtp};
 use crate::sql;
-use crate::tools::{duration_to_str, maybe_add_time_based_warnings, time};
+use crate::tools::{self, duration_to_str, maybe_add_time_based_warnings, time, time_elapsed};
 
 pub(crate) mod connectivity;
 
@@ -70,8 +71,11 @@ impl SchedulerState {
         context.new_msgs_notify.notify_one();
 
         let ctx = context.clone();
-        match Scheduler::start(context).await {
-            Ok(scheduler) => *inner = InnerSchedulerState::Started(scheduler),
+        match Scheduler::start(&context).await {
+            Ok(scheduler) => {
+                *inner = InnerSchedulerState::Started(scheduler);
+                context.emit_event(EventType::ConnectivityChanged);
+            }
             Err(err) => error!(&ctx, "Failed to start IO: {:#}", err),
         }
     }
@@ -116,6 +120,7 @@ impl SchedulerState {
             debug_logging.loop_handle.abort();
         }
         let prev_state = std::mem::replace(&mut *inner, new_state);
+        context.emit_event(EventType::ConnectivityChanged);
         match prev_state {
             InnerSchedulerState::Started(scheduler) => scheduler.stop(context).await,
             InnerSchedulerState::Stopped | InnerSchedulerState::Paused { .. } => (),
@@ -286,7 +291,7 @@ enum InnerSchedulerState {
 ///
 /// Returned by [`SchedulerState::pause`].  To resume the IO scheduler simply drop this
 /// guard.
-#[derive(Debug)]
+#[derive(Default, Debug)]
 pub(crate) struct IoPausedGuard {
     sender: Option<oneshot::Sender<()>>,
 }
@@ -346,6 +351,16 @@ async fn download_msgs(context: &Context, imap: &mut Imap) -> Result<()> {
     for msg_id in msg_ids {
         if let Err(err) = download_msg(context, msg_id, imap).await {
             warn!(context, "Failed to download message {msg_id}: {:#}.", err);
+
+            // Update download state to failure
+            // so it can be retried.
+            //
+            // On success update_download_state() is not needed
+            // as receive_imf() already
+            // set the state and emitted the event.
+            msg_id
+                .update_download_state(context, DownloadState::Failure)
+                .await?;
         }
         context
             .sql
@@ -384,7 +399,7 @@ async fn inbox_loop(
                     let quota = ctx.quota.read().await;
                     quota
                         .as_ref()
-                        .filter(|quota| quota.modified + 60 > time())
+                        .filter(|quota| time_elapsed(&quota.modified) > Duration::from_secs(60))
                         .is_none()
                 };
 
@@ -425,8 +440,12 @@ async fn inbox_loop(
                         //
                         // This operation is not critical enough to retry,
                         // especially if the error is persistent.
-                        if let Err(err) =
-                            ctx.set_config_bool(Config::FetchedExistingMsgs, true).await
+                        if let Err(err) = ctx
+                            .set_config_internal(
+                                Config::FetchedExistingMsgs,
+                                config::from_bool(true),
+                            )
+                            .await
                         {
                             warn!(ctx, "Can't set Config::FetchedExistingMsgs: {:#}", err);
                         }
@@ -446,6 +465,10 @@ async fn inbox_loop(
                 warn!(ctx, "Failed to download messages: {:#}", err);
             }
 
+            if let Err(err) = connection.fetch_metadata(&ctx).await {
+                warn!(ctx, "Failed to fetch metadata: {err:#}.");
+            }
+
             fetch_idle(&ctx, &mut connection, FolderMeaning::Inbox).await;
         }
     };
@@ -459,6 +482,39 @@ async fn inbox_loop(
         .await;
 }
 
+/// Convert folder meaning
+/// used internally by [fetch_idle] and [Context::background_fetch]
+pub async fn convert_folder_meaning(
+    ctx: &Context,
+    folder_meaning: FolderMeaning,
+) -> Result<(Config, String)> {
+    let folder_config = match folder_meaning.to_config() {
+        Some(c) => c,
+        None => {
+            bail!("Bad folder meaning: {}", folder_meaning);
+        }
+    };
+
+    let folder = match ctx.get_config(folder_config).await {
+        Ok(folder) => folder,
+        Err(err) => {
+            bail!(
+                "Can not watch {} folder, failed to retrieve config: {:#}",
+                folder_config,
+                err
+            );
+        }
+    };
+
+    let watch_folder = if let Some(watch_folder) = folder {
+        watch_folder
+    } else {
+        bail!("Can not watch {} folder, not set", folder_config);
+    };
+
+    Ok((folder_config, watch_folder))
+}
+
 /// Implement a single iteration of IMAP loop.
 ///
 /// This function performs all IMAP operations on a single folder, selecting it if necessary and
@@ -466,39 +522,33 @@ async fn inbox_loop(
 /// critical operation fails such as fetching new messages fails, connection is reset via
 /// `trigger_reconnect`, so a fresh one can be opened.
 async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: FolderMeaning) {
-    let folder_config = match folder_meaning.to_config() {
-        Some(c) => c,
-        None => {
-            error!(ctx, "Bad folder meaning: {}", folder_meaning);
-            connection
-                .fake_idle(ctx, None, FolderMeaning::Unknown)
-                .await;
-            return;
-        }
-    };
-    let folder = match ctx.get_config(folder_config).await {
-        Ok(folder) => folder,
-        Err(err) => {
-            warn!(
-                ctx,
-                "Can not watch {} folder, failed to retrieve config: {:#}", folder_config, err
-            );
-            connection
-                .fake_idle(ctx, None, FolderMeaning::Unknown)
-                .await;
-            return;
-        }
-    };
-
-    let watch_folder = if let Some(watch_folder) = folder {
-        watch_folder
-    } else {
-        connection.connectivity.set_not_configured(ctx).await;
-        info!(ctx, "Can not watch {} folder, not set", folder_config);
+    let create_mvbox = true;
+    if let Err(err) = connection
+        .ensure_configured_folders(ctx, create_mvbox)
+        .await
+    {
+        warn!(
+            ctx,
+            "Cannot watch {folder_meaning}, ensure_configured_folders() failed: {:#}", err,
+        );
         connection
             .fake_idle(ctx, None, FolderMeaning::Unknown)
             .await;
         return;
+    }
+    let (folder_config, watch_folder) = match convert_folder_meaning(ctx, folder_meaning).await {
+        Ok(meaning) => meaning,
+        Err(error) => {
+            // Warning instead of error because the folder may not be configured.
+            // For example, this happens if the server does not have Sent folder
+            // but watching Sent folder is enabled.
+            warn!(ctx, "Error converting IMAP Folder name: {:?}", error);
+            connection.connectivity.set_not_configured(ctx).await;
+            connection
+                .fake_idle(ctx, None, FolderMeaning::Unknown)
+                .await;
+            return;
+        }
     };
 
     // connect and fake idle if unable to connect
@@ -590,7 +640,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
         .log_err(ctx)
         .ok();
 
-    connection.connectivity.set_connected(ctx).await;
+    connection.connectivity.set_idle(ctx).await;
 
     ctx.emit_event(EventType::ImapInboxIdle);
     let Some(session) = connection.session.take() else {
@@ -727,7 +777,7 @@ async fn smtp_loop(
             // Fake Idle
             info!(ctx, "smtp fake idle - started");
             match &connection.last_send_error {
-                None => connection.connectivity.set_connected(&ctx).await,
+                None => connection.connectivity.set_idle(&ctx).await,
                 Some(err) => connection.connectivity.set_err(&ctx, err).await,
             }
 
@@ -736,7 +786,7 @@ async fn smtp_loop(
             // again, we increase the timeout exponentially, in order not to do lots of
             // unnecessary retries.
             if let Some(t) = timeout {
-                let now = tokio::time::Instant::now();
+                let now = tools::Time::now();
                 info!(
                     ctx,
                     "smtp has messages to retry, planning to retry {} seconds later", t,
@@ -747,7 +797,7 @@ async fn smtp_loop(
                 })
                 .await
                 .unwrap_or_default();
-                let slept = (tokio::time::Instant::now() - now).as_secs();
+                let slept = time_elapsed(&now).as_secs();
                 timeout = Some(cmp::max(
                     t,
                     slept.saturating_add(rand::thread_rng().gen_range((slept / 2)..=slept)),
@@ -772,7 +822,7 @@ async fn smtp_loop(
 
 impl Scheduler {
     /// Start the scheduler.
-    pub async fn start(ctx: Context) -> Result<Self> {
+    pub async fn start(ctx: &Context) -> Result<Self> {
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
 
         let (smtp_start_send, smtp_start_recv) = oneshot::channel();
@@ -782,7 +832,7 @@ impl Scheduler {
         let mut oboxes = Vec::new();
         let mut start_recvs = Vec::new();
 
-        let (conn_state, inbox_handlers) = ImapConnectionState::new(&ctx).await?;
+        let (conn_state, inbox_handlers) = ImapConnectionState::new(ctx).await?;
         let (inbox_start_send, inbox_start_recv) = oneshot::channel();
         let handle = {
             let ctx = ctx.clone();
@@ -803,7 +853,7 @@ impl Scheduler {
             ),
         ] {
             if should_watch? {
-                let (conn_state, handlers) = ImapConnectionState::new(&ctx).await?;
+                let (conn_state, handlers) = ImapConnectionState::new(ctx).await?;
                 let (start_send, start_recv) = oneshot::channel();
                 let ctx = ctx.clone();
                 let handle = task::spawn(simple_imap_loop(ctx, start_send, handlers, meaning));

src/scheduler/connectivity.rs

@@ -33,10 +33,19 @@ enum DetailedConnectivity {
     #[default]
     Uninitialized,
     Connecting,
-    Working,
-    InterruptingIdle,
+
+    /// Connection is just established, but there may be work to do.
     Connected,
 
+    /// There is actual work to do, e.g. there are messages in SMTP queue
+    /// or we detected a message that should be downloaded.
+    Working,
+
+    InterruptingIdle,
+
+    /// Connection is established and is idle.
+    Idle,
+
     /// The folder was configured not to be watched or configured_*_folder is not set
     NotConfigured,
 }
@@ -54,6 +63,8 @@ impl DetailedConnectivity {
             // Just don't return a connectivity, probably the folder is configured not to be
             // watched or there is e.g. no "Sent" folder, so we are not interested in it
             DetailedConnectivity::NotConfigured => None,
+
+            DetailedConnectivity::Idle => Some(Connectivity::Connected),
         }
     }
 
@@ -65,7 +76,8 @@ impl DetailedConnectivity {
             DetailedConnectivity::Connecting => "<span class=\"yellow dot\"></span>".to_string(),
             DetailedConnectivity::Working
             | DetailedConnectivity::InterruptingIdle
-            | DetailedConnectivity::Connected => "<span class=\"green dot\"></span>".to_string(),
+            | DetailedConnectivity::Connected
+            | DetailedConnectivity::Idle => "<span class=\"green dot\"></span>".to_string(),
         }
     }
 
@@ -75,9 +87,9 @@ impl DetailedConnectivity {
             DetailedConnectivity::Uninitialized => "Not started".to_string(),
             DetailedConnectivity::Connecting => stock_str::connecting(context).await,
             DetailedConnectivity::Working => stock_str::updating(context).await,
-            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                stock_str::connected(context).await
-            }
+            DetailedConnectivity::InterruptingIdle
+            | DetailedConnectivity::Connected
+            | DetailedConnectivity::Idle => stock_str::connected(context).await,
             DetailedConnectivity::NotConfigured => "Not configured".to_string(),
         }
     }
@@ -94,9 +106,9 @@ impl DetailedConnectivity {
             // We don't know any more than that the last message was sent successfully;
             // since sending the last message, connectivity could have changed, which we don't notice
             // until another message is sent
-            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                stock_str::last_msg_sent_successfully(context).await
-            }
+            DetailedConnectivity::InterruptingIdle
+            | DetailedConnectivity::Connected
+            | DetailedConnectivity::Idle => stock_str::last_msg_sent_successfully(context).await,
             DetailedConnectivity::NotConfigured => "Not configured".to_string(),
         }
     }
@@ -108,8 +120,9 @@ impl DetailedConnectivity {
             DetailedConnectivity::Connecting => false,
             DetailedConnectivity::Working => false,
             DetailedConnectivity::InterruptingIdle => false,
-            DetailedConnectivity::Connected => true,
+            DetailedConnectivity::Connected => false, // Just connected, there may still be work to do.
             DetailedConnectivity::NotConfigured => true,
+            DetailedConnectivity::Idle => true,
         }
     }
 }
@@ -141,6 +154,9 @@ impl ConnectivityStore {
     pub(crate) async fn set_not_configured(&self, context: &Context) {
         self.set(context, DetailedConnectivity::NotConfigured).await;
     }
+    pub(crate) async fn set_idle(&self, context: &Context) {
+        self.set(context, DetailedConnectivity::Idle).await;
+    }
 
     async fn get_detailed(&self) -> DetailedConnectivity {
         self.0.lock().await.deref().clone()
@@ -164,6 +180,7 @@ pub(crate) async fn idle_interrupted(inbox: ConnectivityStore, oboxes: Vec<Conne
     // return Connected until DC is completely done with fetching folders; this also
     // includes scan_folders() which happens on the inbox thread.
     if *connectivity_lock == DetailedConnectivity::Connected
+        || *connectivity_lock == DetailedConnectivity::Idle
         || *connectivity_lock == DetailedConnectivity::NotConfigured
     {
         *connectivity_lock = DetailedConnectivity::InterruptingIdle;
@@ -172,7 +189,9 @@ pub(crate) async fn idle_interrupted(inbox: ConnectivityStore, oboxes: Vec<Conne
 
     for state in oboxes {
         let mut connectivity_lock = state.0.lock().await;
-        if *connectivity_lock == DetailedConnectivity::Connected {
+        if *connectivity_lock == DetailedConnectivity::Connected
+            || *connectivity_lock == DetailedConnectivity::Idle
+        {
             *connectivity_lock = DetailedConnectivity::InterruptingIdle;
         }
     }

src/securejoin.rs

@@ -18,8 +18,9 @@ use crate::key::{load_self_public_key, DcKey, Fingerprint};
 use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
-use crate::peerstate::{Peerstate, PeerstateKeyType};
+use crate::peerstate::Peerstate;
 use crate::qr::check_qr;
+use crate::securejoin::bob::JoinerProgress;
 use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::token;
@@ -173,7 +174,6 @@ async fn send_alice_handshake_msg(
     context: &Context,
     contact_id: ContactId,
     step: &str,
-    fingerprint: Option<Fingerprint>,
 ) -> Result<()> {
     let mut msg = Message {
         viewtype: Viewtype::Text,
@@ -183,9 +183,6 @@ async fn send_alice_handshake_msg(
     };
     msg.param.set_cmd(SystemMessage::SecurejoinMessage);
     msg.param.set(Param::Arg, step);
-    if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp.hex());
-    }
     msg.param.set_int(Param::GuaranteeE2ee, 1);
     chat::send_msg(
         context,
@@ -204,7 +201,9 @@ async fn info_chat_id(context: &Context, contact_id: ContactId) -> Result<ChatId
     Ok(chat_id_blocked.id)
 }
 
-async fn fingerprint_equals_sender(
+/// Checks fingerprint and marks the contact as forward verified
+/// if fingerprint matches.
+async fn verify_sender_by_fingerprint(
     context: &Context,
     fingerprint: &Fingerprint,
     contact_id: ContactId,
@@ -223,14 +222,20 @@ async fn fingerprint_equals_sender(
         }
     };
 
-    if let Some(peerstate) = peerstate {
+    if let Some(mut peerstate) = peerstate {
         if peerstate
             .public_key_fingerprint
             .as_ref()
             .filter(|&fp| fp == fingerprint)
             .is_some()
         {
-            return Ok(true);
+            if let Some(public_key) = &peerstate.public_key {
+                let verifier = contact.get_addr().to_owned();
+                peerstate.set_verified(public_key.clone(), fingerprint.clone(), verifier)?;
+                peerstate.prefer_encrypt = EncryptPreference::Mutual;
+                peerstate.save_to_db(&context.sql).await?;
+                return Ok(true);
+            }
         }
     }
 
@@ -287,10 +292,7 @@ pub(crate) async fn handle_securejoin_handshake(
         .get_header(HeaderDef::SecureJoin)
         .context("Not a Secure-Join message")?;
 
-    info!(
-        context,
-        ">>>>>>>>>>>>>>>>>>>>>>>>> secure-join message \'{}\' received", step,
-    );
+    info!(context, "Received secure-join message {step:?}.");
 
     let join_vg = step.starts_with("vg-");
 
@@ -312,11 +314,10 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !token::exists(context, token::Namespace::InviteNumber, invitenumber).await {
+            if !token::exists(context, token::Namespace::InviteNumber, invitenumber).await? {
                 warn!(context, "Secure-join denied (bad invitenumber).");
                 return Ok(HandshakeMessage::Ignore);
             }
-            info!(context, "Secure-join requested.",);
 
             inviter_progress(context, contact_id, 300);
 
@@ -331,7 +332,6 @@ pub(crate) async fn handle_securejoin_handshake(
                 context,
                 contact_id,
                 &format!("{}-auth-required", &step[..2]),
-                None,
             )
             .await
             .context("failed sending auth-required handshake message")?;
@@ -376,7 +376,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if !fingerprint_equals_sender(context, &fingerprint, contact_id).await? {
+            if !verify_sender_by_fingerprint(context, &fingerprint, contact_id).await? {
                 could_not_establish_secure_connection(
                     context,
                     contact_id,
@@ -388,20 +388,17 @@ pub(crate) async fn handle_securejoin_handshake(
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
-            let auth_0 = match mime_message.get_header(HeaderDef::SecureJoinAuth) {
-                Some(auth) => auth,
-                None => {
-                    could_not_establish_secure_connection(
-                        context,
-                        contact_id,
-                        info_chat_id(context, contact_id).await?,
-                        "Auth not provided.",
-                    )
-                    .await?;
-                    return Ok(HandshakeMessage::Ignore);
-                }
+            let Some(auth) = mime_message.get_header(HeaderDef::SecureJoinAuth) else {
+                could_not_establish_secure_connection(
+                    context,
+                    contact_id,
+                    info_chat_id(context, contact_id).await?,
+                    "Auth not provided.",
+                )
+                .await?;
+                return Ok(HandshakeMessage::Ignore);
             };
-            if !token::exists(context, token::Namespace::Auth, auth_0).await {
+            if !token::exists(context, token::Namespace::Auth, auth).await? {
                 could_not_establish_secure_connection(
                     context,
                     contact_id,
@@ -415,8 +412,14 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?
                 .get_addr()
                 .to_owned();
-            let fingerprint_found =
-                mark_peer_as_verified(context, fingerprint.clone(), contact_addr).await?;
+            let backward_verified = true;
+            let fingerprint_found = mark_peer_as_verified(
+                context,
+                fingerprint.clone(),
+                contact_addr,
+                backward_verified,
+            )
+            .await?;
             if !fingerprint_found {
                 could_not_establish_secure_connection(
                     context,
@@ -445,7 +448,13 @@ pub(crate) async fn handle_securejoin_handshake(
                 };
                 match chat::get_chat_id_by_grpid(context, field_grpid).await? {
                     Some((group_chat_id, _, _)) => {
-                        secure_connection_established(context, contact_id, group_chat_id).await?;
+                        secure_connection_established(
+                            context,
+                            contact_id,
+                            group_chat_id,
+                            mime_message.timestamp_sent,
+                        )
+                        .await?;
                         chat::add_contact_to_chat_ex(
                             context,
                             Nosync,
@@ -465,16 +474,12 @@ pub(crate) async fn handle_securejoin_handshake(
                     context,
                     contact_id,
                     info_chat_id(context, contact_id).await?,
+                    mime_message.timestamp_sent,
                 )
                 .await?;
-                send_alice_handshake_msg(
-                    context,
-                    contact_id,
-                    "vc-contact-confirm",
-                    Some(fingerprint),
-                )
-                .await
-                .context("failed sending vc-contact-confirm message")?;
+                send_alice_handshake_msg(context, contact_id, "vc-contact-confirm")
+                    .await
+                    .context("failed sending vc-contact-confirm message")?;
 
                 inviter_progress(context, contact_id, 1000);
             }
@@ -484,11 +489,18 @@ pub(crate) async fn handle_securejoin_handshake(
         ====             Bob - the joiner's side             ====
         ====   Step 7 in "Setup verified contact" protocol   ====
         =======================================================*/
-        "vc-contact-confirm" => match BobState::from_db(&context.sql).await? {
-            Some(bobstate) => bob::handle_contact_confirm(context, bobstate, mime_message).await,
-            None => Ok(HandshakeMessage::Ignore),
-        },
+        "vc-contact-confirm" => {
+            if let Some(mut bobstate) = BobState::from_db(&context.sql).await? {
+                if !bobstate.is_msg_expected(context, step.as_str()) {
+                    warn!(context, "Unexpected vc-contact-confirm.");
+                    return Ok(HandshakeMessage::Ignore);
+                }
 
+                bobstate.step_contact_confirm(context).await?;
+                bobstate.emit_progress(context, JoinerProgress::Succeeded);
+            }
+            Ok(HandshakeMessage::Ignore)
+        }
         "vg-member-added" => {
             let Some(member_added) = mime_message
                 .get_header(HeaderDef::ChatGroupMemberAdded)
@@ -496,23 +508,27 @@ pub(crate) async fn handle_securejoin_handshake(
             else {
                 warn!(
                     context,
-                    "vg-member-added without Chat-Group-Member-Added header"
+                    "vg-member-added without Chat-Group-Member-Added header."
                 );
                 return Ok(HandshakeMessage::Propagate);
             };
             if !context.is_self_addr(member_added).await? {
                 info!(
                     context,
-                    "Member {member_added} added by unrelated SecureJoin process"
+                    "Member {member_added} added by unrelated SecureJoin process."
                 );
                 return Ok(HandshakeMessage::Propagate);
             }
-            match BobState::from_db(&context.sql).await? {
-                Some(bobstate) => {
-                    bob::handle_contact_confirm(context, bobstate, mime_message).await
+            if let Some(mut bobstate) = BobState::from_db(&context.sql).await? {
+                if !bobstate.is_msg_expected(context, step.as_str()) {
+                    warn!(context, "Unexpected vg-member-added.");
+                    return Ok(HandshakeMessage::Propagate);
                 }
-                None => Ok(HandshakeMessage::Propagate),
+
+                bobstate.step_contact_confirm(context).await?;
+                bobstate.emit_progress(context, JoinerProgress::Succeeded);
             }
+            Ok(HandshakeMessage::Propagate)
         }
 
         "vg-member-added-received" | "vc-contact-confirm-received" => {
@@ -526,23 +542,25 @@ pub(crate) async fn handle_securejoin_handshake(
     }
 }
 
-/// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen.
+/// Observe self-sent Securejoin message.
 ///
-/// in a multi-device-setup, there may be other devices that "see" the handshake messages.
-/// if the seen messages seen are self-sent messages encrypted+signed correctly with our key,
-/// we can make some conclusions of it:
+/// In a multi-device-setup, there may be other devices that "see" the handshake messages.
+/// If we see self-sent messages encrypted+signed correctly with our key,
+/// we can make some conclusions of it.
 ///
-/// - if we see the self-sent-message vg-member-added/vc-contact-confirm,
-///   we know that we're an inviter-observer.
-///   The inviting device has marked a peer as verified on vg-request-with-auth/vc-request-with-auth
-///   before sending vg-member-added/vc-contact-confirm - so, if we observe vg-member-added/vc-contact-confirm,
-///   we can mark the peer as verified as well.
+/// If we see self-sent {vc,vg}-request-with-auth,
+/// we know that we are Bob (joiner-observer)
+/// that just marked peer (Alice) as forward-verified
+/// either after receiving {vc,vg}-auth-required
+/// or immediately after scanning the QR-code
+/// if the key was already known.
 ///
-/// - if we see the self-sent-message vg-request-with-auth/vc-request-with-auth
-///   we know that we're an joiner-observer.
-///   the joining device has marked the peer as verified
-///   before sending vg-request-with-auth/vc-request-with-auth - so, if we observe vg-member-added-received,
-///   we can mark the peer as verified as well.
+/// If we see self-sent vc-contact-confirm or vg-member-added message,
+/// we know that we are Alice (inviter-observer)
+/// that just marked peer (Bob) as forward (and backward)-verified
+/// in response to correct vc-request-with-auth message.
+///
+/// In both cases we can mark the peer as forward-verified.
 pub(crate) async fn observe_securejoin_on_other_device(
     context: &Context,
     mime_message: &MimeMessage,
@@ -554,124 +572,98 @@ pub(crate) async fn observe_securejoin_on_other_device(
     let step = mime_message
         .get_header(HeaderDef::SecureJoin)
         .context("Not a Secure-Join message")?;
-    info!(context, "observing secure-join message \'{}\'", step);
+    info!(context, "Observing secure-join message {step:?}.");
 
-    match step.as_str() {
-        "vg-request-with-auth"
-        | "vc-request-with-auth"
-        | "vg-member-added"
-        | "vc-contact-confirm" => {
-            if !encrypted_and_signed(
-                context,
-                mime_message,
-                get_self_fingerprint(context).await.as_ref(),
-            ) {
-                could_not_establish_secure_connection(
-                    context,
-                    contact_id,
-                    info_chat_id(context, contact_id).await?,
-                    "Message not encrypted correctly.",
-                )
-                .await?;
-                return Ok(HandshakeMessage::Ignore);
-            }
-            let addr = Contact::get_by_id(context, contact_id)
-                .await?
-                .get_addr()
-                .to_lowercase();
-            if mime_message.gossiped_addr.contains(&addr) {
-                let mut peerstate = match Peerstate::from_addr(context, &addr).await? {
-                    Some(p) => p,
-                    None => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_id,
-                            info_chat_id(context, contact_id).await?,
-                            &format!("No peerstate in db for '{}' at step {}", &addr, step),
-                        )
-                        .await?;
-                        return Ok(HandshakeMessage::Ignore);
-                    }
-                };
-                let fingerprint = match peerstate.gossip_key_fingerprint.clone() {
-                    Some(fp) => fp,
-                    None => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_id,
-                            info_chat_id(context, contact_id).await?,
-                            &format!(
-                                "No gossip key fingerprint in db for '{}' at step {}",
-                                &addr, step,
-                            ),
-                        )
-                        .await?;
-                        return Ok(HandshakeMessage::Ignore);
-                    }
-                };
-                peerstate.set_verified(PeerstateKeyType::GossipKey, fingerprint, addr)?;
-                peerstate.prefer_encrypt = EncryptPreference::Mutual;
-                peerstate.save_to_db(&context.sql).await.unwrap_or_default();
+    if !matches!(
+        step.as_str(),
+        "vg-request-with-auth" | "vc-request-with-auth" | "vg-member-added" | "vc-contact-confirm"
+    ) {
+        return Ok(HandshakeMessage::Ignore);
+    };
 
-                ChatId::set_protection_for_contact(context, contact_id).await?;
-            } else if let Some(fingerprint) =
-                mime_message.get_header(HeaderDef::SecureJoinFingerprint)
-            {
-                // FIXME: Old versions of DC send this header instead of gossips. Remove this
-                // eventually.
-                let fingerprint = fingerprint.parse()?;
-                let fingerprint_found = mark_peer_as_verified(
-                    context,
-                    fingerprint,
-                    Contact::get_by_id(context, contact_id)
-                        .await?
-                        .get_addr()
-                        .to_owned(),
-                )
-                .await?;
-                if !fingerprint_found {
-                    could_not_establish_secure_connection(
-                        context,
-                        contact_id,
-                        info_chat_id(context, contact_id).await?,
-                        format!("Fingerprint mismatch on observing {step}.").as_ref(),
-                    )
-                    .await?;
-                    return Ok(HandshakeMessage::Ignore);
-                }
-            } else {
-                could_not_establish_secure_connection(
-                    context,
-                    contact_id,
-                    info_chat_id(context, contact_id).await?,
-                    &format!(
-                        "No gossip header for '{}' at step {}, please update Delta Chat on all \
+    if !encrypted_and_signed(
+        context,
+        mime_message,
+        get_self_fingerprint(context).await.as_ref(),
+    ) {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            "Message not encrypted correctly.",
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    }
+
+    let addr = Contact::get_by_id(context, contact_id)
+        .await?
+        .get_addr()
+        .to_lowercase();
+
+    let Some(key) = mime_message.gossiped_keys.get(&addr) else {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            &format!(
+                "No gossip header for '{}' at step {}, please update Delta Chat on all \
                         your devices.",
-                        &addr, step,
-                    ),
-                )
-                .await?;
-                return Ok(HandshakeMessage::Ignore);
-            }
-            if step.as_str() == "vg-member-added" {
-                inviter_progress(context, contact_id, 800);
-            }
-            if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" {
-                inviter_progress(context, contact_id, 1000);
-            }
-            if step.as_str() == "vg-request-with-auth" || step.as_str() == "vc-request-with-auth" {
-                // This actually reflects what happens on the first device (which does the secure
-                // join) and causes a subsequent "vg-member-added" message to create an unblocked
-                // verified group.
-                ChatId::create_for_contact_with_blocked(context, contact_id, Blocked::Not).await?;
-            }
-            Ok(if step.as_str() == "vg-member-added" {
-                HandshakeMessage::Propagate
-            } else {
-                HandshakeMessage::Ignore
-            })
-        }
-        _ => Ok(HandshakeMessage::Ignore),
+                &addr, step,
+            ),
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    };
+
+    let Some(mut peerstate) = Peerstate::from_addr(context, &addr).await? else {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            &format!("No peerstate in db for '{}' at step {}", &addr, step),
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    };
+
+    let Some(fingerprint) = peerstate.gossip_key_fingerprint.clone() else {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            &format!(
+                "No gossip key fingerprint in db for '{}' at step {}",
+                &addr, step,
+            ),
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    };
+    peerstate.set_verified(key.clone(), fingerprint, addr)?;
+    peerstate.prefer_encrypt = EncryptPreference::Mutual;
+    peerstate.save_to_db(&context.sql).await?;
+
+    ChatId::set_protection_for_contact(context, contact_id, mime_message.timestamp_sent).await?;
+
+    if step.as_str() == "vg-member-added" {
+        inviter_progress(context, contact_id, 800);
+    }
+    if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" {
+        inviter_progress(context, contact_id, 1000);
+    }
+
+    if step.as_str() == "vg-request-with-auth" || step.as_str() == "vc-request-with-auth" {
+        // This actually reflects what happens on the first device (which does the secure
+        // join) and causes a subsequent "vg-member-added" message to create an unblocked
+        // verified group.
+        ChatId::create_for_contact_with_blocked(context, contact_id, Blocked::Not).await?;
+    }
+
+    if step.as_str() == "vg-member-added" {
+        Ok(HandshakeMessage::Propagate)
+    } else {
+        Ok(HandshakeMessage::Ignore)
     }
 }
 
@@ -679,23 +671,19 @@ async fn secure_connection_established(
     context: &Context,
     contact_id: ContactId,
     chat_id: ChatId,
+    timestamp: i64,
 ) -> Result<()> {
-    if context
-        .get_config_bool(Config::VerifiedOneOnOneChats)
+    let private_chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Yes)
         .await?
-    {
-        let private_chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Yes)
-            .await?
-            .id;
-        private_chat_id
-            .set_protection(
-                context,
-                ProtectionStatus::Protected,
-                time(),
-                Some(contact_id),
-            )
-            .await?;
-    }
+        .id;
+    private_chat_id
+        .set_protection(
+            context,
+            ProtectionStatus::Protected,
+            timestamp,
+            Some(contact_id),
+        )
+        .await?;
     context.emit_event(EventType::ChatModified(chat_id));
     Ok(())
 }
@@ -723,13 +711,21 @@ async fn mark_peer_as_verified(
     context: &Context,
     fingerprint: Fingerprint,
     verifier: String,
+    backward_verified: bool,
 ) -> Result<bool> {
     let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, &fingerprint).await? else {
         return Ok(false);
     };
-    peerstate.set_verified(PeerstateKeyType::PublicKey, fingerprint, verifier)?;
+    let Some(ref public_key) = peerstate.public_key else {
+        return Ok(false);
+    };
+    peerstate.set_verified(public_key.clone(), fingerprint, verifier)?;
     peerstate.prefer_encrypt = EncryptPreference::Mutual;
-    peerstate.save_to_db(&context.sql).await.unwrap_or_default();
+    if backward_verified {
+        peerstate.backward_verified_key_id =
+            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+    }
+    peerstate.save_to_db(&context.sql).await?;
     Ok(true)
 }
 
@@ -979,6 +975,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
         peerstate.save_to_db(&bob.ctx.sql).await?;
@@ -1312,4 +1309,70 @@ First thread."#;
 
         Ok(())
     }
+
+    /// Tests that Bob gets Alice as verified
+    /// if `vc-contact-confirm` is lost but Alice then sends
+    /// a message to Bob in a verified 1:1 chat with a `Chat-Verified` header.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_lost_contact_confirm() {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+        alice
+            .set_config(Config::VerifiedOneOnOneChats, Some("1"))
+            .await
+            .unwrap();
+        bob.set_config(Config::VerifiedOneOnOneChats, Some("1"))
+            .await
+            .unwrap();
+
+        let qr = get_securejoin_qr(&alice.ctx, None).await.unwrap();
+        join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        // vc-request
+        let sent = bob.pop_sent_msg().await;
+        alice.recv_msg(&sent).await;
+
+        // vc-auth-required
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        // vc-request-with-auth
+        let sent = bob.pop_sent_msg().await;
+        alice.recv_msg(&sent).await;
+
+        // Alice has Bob verified now.
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown)
+                .await
+                .expect("Error looking up contact")
+                .expect("Contact not found");
+        let contact_bob = Contact::get_by_id(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(contact_bob.is_verified(&alice.ctx).await.unwrap(), true);
+
+        // Alice sends vc-contact-confirm, but it gets lost.
+        let _sent_vc_contact_confirm = alice.pop_sent_msg().await;
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob, "alice@example.org", Origin::Unknown)
+                .await
+                .expect("Error looking up contact")
+                .expect("Contact not found");
+        let contact_alice = Contact::get_by_id(&bob, contact_alice_id).await.unwrap();
+        assert_eq!(contact_alice.is_verified(&bob).await.unwrap(), false);
+
+        // Alice sends a text message to Bob.
+        let received_hello = tcm.send_recv(&alice, &bob, "Hello!").await;
+        let chat_id = received_hello.chat_id;
+        let chat = Chat::load_from_db(&bob, chat_id).await.unwrap();
+        assert_eq!(chat.is_protected(), true);
+
+        // Received text message in a verified 1:1 chat results in backward verification
+        // and Bob now marks alice as verified.
+        let contact_alice = Contact::get_by_id(&bob, contact_alice_id).await.unwrap();
+        assert_eq!(contact_alice.is_verified(&bob).await.unwrap(), true);
+    }
 }

src/securejoin/bob.rs

@@ -9,7 +9,6 @@ use super::bobstate::{BobHandshakeStage, BobState};
 use super::qrinvite::QrInvite;
 use super::HandshakeMessage;
 use crate::chat::{is_contact_in_chat, ChatId, ProtectionStatus};
-use crate::config::Config;
 use crate::constants::{Blocked, Chattype};
 use crate::contact::Contact;
 use crate::context::Context;
@@ -83,65 +82,31 @@ pub(super) async fn handle_auth_required(
     context: &Context,
     message: &MimeMessage,
 ) -> Result<HandshakeMessage> {
-    match BobState::from_db(&context.sql).await? {
-        Some(mut bobstate) => match bobstate.handle_message(context, message).await? {
-            Some(BobHandshakeStage::Terminated(why)) => {
-                bobstate.notify_aborted(context, why).await?;
-                Ok(HandshakeMessage::Done)
-            }
-            Some(_stage) => {
-                if bobstate.is_join_group() {
-                    // The message reads "Alice replied, waiting to be added to the group…",
-                    // so only show it on secure-join and not on setup-contact.
-                    let contact_id = bobstate.invite().contact_id();
-                    let msg = stock_str::secure_join_replies(context, contact_id).await;
-                    let chat_id = bobstate.joining_chat_id(context).await?;
-                    chat::add_info_msg(context, chat_id, &msg, time()).await?;
-                }
-                bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
-                Ok(HandshakeMessage::Done)
-            }
-            None => Ok(HandshakeMessage::Ignore),
-        },
-        None => Ok(HandshakeMessage::Ignore),
-    }
-}
-
-/// Handles `vc-contact-confirm` and `vg-member-added` handshake messages.
-///
-/// # Bob - the joiner's side
-/// ## Step 7 in the "Setup Contact protocol"
-pub(super) async fn handle_contact_confirm(
-    context: &Context,
-    mut bobstate: BobState,
-    message: &MimeMessage,
-) -> Result<HandshakeMessage> {
-    let retval = if bobstate.is_join_group() {
-        HandshakeMessage::Propagate
-    } else {
-        HandshakeMessage::Ignore
+    let Some(mut bobstate) = BobState::from_db(&context.sql).await? else {
+        return Ok(HandshakeMessage::Ignore);
     };
-    match bobstate.handle_message(context, message).await? {
+
+    match bobstate.handle_auth_required(context, message).await? {
         Some(BobHandshakeStage::Terminated(why)) => {
             bobstate.notify_aborted(context, why).await?;
             Ok(HandshakeMessage::Done)
         }
-        Some(BobHandshakeStage::Completed) => {
-            // Note this goes to the 1:1 chat, as when joining a group we implicitly also
-            // verify both contacts (this could be a bug/security issue, see
-            // e.g. https://github.com/deltachat/deltachat-core-rust/issues/1177).
-            bobstate.notify_peer_verified(context).await?;
-            bobstate.emit_progress(context, JoinerProgress::Succeeded);
-            Ok(retval)
+        Some(_stage) => {
+            if bobstate.is_join_group() {
+                // The message reads "Alice replied, waiting to be added to the group…",
+                // so only show it on secure-join and not on setup-contact.
+                let contact_id = bobstate.invite().contact_id();
+                let msg = stock_str::secure_join_replies(context, contact_id).await;
+                let chat_id = bobstate.joining_chat_id(context).await?;
+                chat::add_info_msg(context, chat_id, &msg, time()).await?;
+            }
+            bobstate
+                .set_peer_verified(context, message.timestamp_sent)
+                .await?;
+            bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
+            Ok(HandshakeMessage::Done)
         }
-        Some(_) => {
-            warn!(
-                context,
-                "Impossible state returned from handling handshake message"
-            );
-            Ok(retval)
-        }
-        None => Ok(retval),
+        None => Ok(HandshakeMessage::Ignore),
     }
 }
 
@@ -154,7 +119,7 @@ impl BobState {
         }
     }
 
-    fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
+    pub(crate) fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
         let contact_id = self.invite().contact_id();
         context.emit_event(EventType::SecurejoinJoinerProgress {
             contact_id,
@@ -217,28 +182,17 @@ impl BobState {
         Ok(())
     }
 
-    /// Notifies the user that the SecureJoin peer is verified.
-    ///
-    /// This creates an info message in the chat being joined.
-    async fn notify_peer_verified(&self, context: &Context) -> Result<()> {
+    /// Turns 1:1 chat with SecureJoin peer into protected chat.
+    pub(crate) async fn set_peer_verified(&self, context: &Context, timestamp: i64) -> Result<()> {
         let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        let chat_id = self.joining_chat_id(context).await?;
-
-        if context
-            .get_config_bool(Config::VerifiedOneOnOneChats)
-            .await?
-        {
-            self.alice_chat()
-                .set_protection(
-                    context,
-                    ProtectionStatus::Protected,
-                    time(),
-                    Some(contact.id),
-                )
-                .await?;
-        }
-
-        context.emit_event(EventType::ChatModified(chat_id));
+        self.alice_chat()
+            .set_protection(
+                context,
+                ProtectionStatus::Protected,
+                timestamp,
+                Some(contact.id),
+            )
+            .await?;
         Ok(())
     }
 }
@@ -247,7 +201,7 @@ impl BobState {
 ///
 /// This has an `From<JoinerProgress> for usize` impl yielding numbers between 0 and a 1000
 /// which can be shown as a progress bar.
-enum JoinerProgress {
+pub(crate) enum JoinerProgress {
     /// An error occurred.
     Error,
     /// vg-vc-request-with-auth sent.

src/securejoin/bobstate.rs

@@ -11,8 +11,9 @@ use anyhow::Result;
 use rusqlite::Connection;
 
 use super::qrinvite::QrInvite;
-use super::{encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified};
+use super::{encrypted_and_signed, verify_sender_by_fingerprint};
 use crate::chat::{self, ChatId};
+use crate::config::Config;
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::events::EventType;
@@ -21,7 +22,9 @@ use crate::key::{load_self_public_key, DcKey};
 use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
+use crate::securejoin::Peerstate;
 use crate::sql::Sql;
+use crate::tools::time;
 
 /// The stage of the [`BobState`] securejoin handshake protocol state machine.
 ///
@@ -30,14 +33,9 @@ use crate::sql::Sql;
 #[derive(Clone, Copy, Debug, Display)]
 pub enum BobHandshakeStage {
     /// Step 2 completed: (vc|vg)-request message sent.
-    ///
-    /// Note that this is only ever returned by [`BobState::start_protocol`] and never by
-    /// [`BobState::handle_message`].
     RequestSent,
     /// Step 4 completed: (vc|vg)-request-with-auth message sent.
     RequestWithAuthSent,
-    /// The protocol completed successfully.
-    Completed,
     /// The protocol prematurely terminated with given reason.
     Terminated(&'static str),
 }
@@ -92,21 +90,26 @@ impl BobState {
         invite: QrInvite,
         chat_id: ChatId,
     ) -> Result<(Self, BobHandshakeStage, Vec<Self>)> {
-        let (stage, next) =
-            if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await?
-            {
-                // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
-                info!(context, "Taking securejoin protocol shortcut");
-                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
-                    .await?;
-                (
-                    BobHandshakeStage::RequestWithAuthSent,
-                    SecureJoinStep::ContactConfirm,
-                )
-            } else {
-                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
-                (BobHandshakeStage::RequestSent, SecureJoinStep::AuthRequired)
-            };
+        let peer_verified =
+            verify_sender_by_fingerprint(context, invite.fingerprint(), invite.contact_id())
+                .await?;
+
+        let (stage, next);
+        if peer_verified {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+
+            stage = BobHandshakeStage::RequestWithAuthSent;
+            next = SecureJoinStep::ContactConfirm;
+        } else {
+            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
+
+            stage = BobHandshakeStage::RequestSent;
+            next = SecureJoinStep::AuthRequired;
+        };
+
         let (id, aborted_states) =
             Self::insert_new_db_entry(context, next, invite.clone(), chat_id).await?;
         let state = Self {
@@ -115,6 +118,12 @@ impl BobState {
             next,
             chat_id,
         };
+
+        if peer_verified {
+            // Mark 1:1 chat as verified already.
+            state.set_peer_verified(context, time()).await?;
+        }
+
         Ok((state, stage, aborted_states))
     }
 
@@ -230,13 +239,13 @@ impl BobState {
         Ok(())
     }
 
-    /// Handles the given message for the securejoin handshake for Bob.
+    /// Handles {vc,vg}-auth-required message of the securejoin handshake for Bob.
     ///
     /// If the message was not used for this handshake `None` is returned, otherwise the new
-    /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
-    /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
+    /// stage is returned.  Once [`BobHandshakeStage::Terminated`] is reached this
+    /// [`BobState`] should be destroyed,
     /// further calling it will just result in the messages being unused by this handshake.
-    pub(crate) async fn handle_message(
+    pub(crate) async fn handle_auth_required(
         &mut self,
         context: &Context,
         mime_message: &MimeMessage,
@@ -256,42 +265,10 @@ impl BobState {
             info!(context, "{} message out of sync for BobState", step);
             return Ok(None);
         }
-        match step.as_str() {
-            "vg-auth-required" | "vc-auth-required" => {
-                self.step_auth_required(context, mime_message).await
-            }
-            "vg-member-added" | "vc-contact-confirm" => {
-                self.step_contact_confirm(context, mime_message).await
-            }
-            _ => {
-                warn!(context, "Invalid step for BobState: {}", step);
-                Ok(None)
-            }
-        }
-    }
 
-    /// Returns `true` if the message is expected according to the protocol.
-    fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
-        let variant_matches = match self.invite {
-            QrInvite::Contact { .. } => step.starts_with("vc-"),
-            QrInvite::Group { .. } => step.starts_with("vg-"),
-        };
-        let step_matches = self.next.matches(context, step);
-        variant_matches && step_matches
-    }
-
-    /// Handles a *vc-auth-required* or *vg-auth-required* message.
-    ///
-    /// # Bob - the joiner's side
-    /// ## Step 4 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    async fn step_auth_required(
-        &mut self,
-        context: &Context,
-        mime_message: &MimeMessage,
-    ) -> Result<Option<BobHandshakeStage>> {
         info!(
             context,
-            "Bob Step 4 - handling vc-auth-require/vg-auth-required message"
+            "Bob Step 4 - handling {{vc,vg}}-auth-required message."
         );
         if !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint())) {
             let reason = if mime_message.was_encrypted() {
@@ -303,14 +280,19 @@ impl BobState {
                 .await?;
             return Ok(Some(BobHandshakeStage::Terminated(reason)));
         }
-        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.invite.contact_id())
-            .await?
+        if !verify_sender_by_fingerprint(
+            context,
+            self.invite.fingerprint(),
+            self.invite.contact_id(),
+        )
+        .await?
         {
             self.update_next(&context.sql, SecureJoinStep::Terminated)
                 .await?;
             return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
         }
         info!(context, "Fingerprint verified.",);
+
         self.update_next(&context.sql, SecureJoinStep::ContactConfirm)
             .await?;
         self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
@@ -318,36 +300,39 @@ impl BobState {
         Ok(Some(BobHandshakeStage::RequestWithAuthSent))
     }
 
+    /// Returns `true` if the message is expected according to the protocol.
+    pub(crate) fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
+        let variant_matches = match self.invite {
+            QrInvite::Contact { .. } => step.starts_with("vc-"),
+            QrInvite::Group { .. } => step.starts_with("vg-"),
+        };
+        let step_matches = self.next.matches(context, step);
+        variant_matches && step_matches
+    }
+
     /// Handles a *vc-contact-confirm* or *vg-member-added* message.
     ///
     /// # Bob - the joiner's side
     /// ## Step 7 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    ///
-    /// This deviates from the protocol by also sending a confirmation message in response
-    /// to the *vc-contact-confirm* message.  This has no specific value to the protocol and
-    /// is only done out of symmetry with *vg-member-added* handling.
-    async fn step_contact_confirm(
-        &mut self,
-        context: &Context,
-        mime_message: &MimeMessage,
-    ) -> Result<Option<BobHandshakeStage>> {
-        info!(
-            context,
-            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message"
-        );
-        mark_peer_as_verified(
-            context,
-            self.invite.fingerprint().clone(),
-            mime_message.from.addr.to_string(),
-        )
-        .await?;
+    pub(crate) async fn step_contact_confirm(&mut self, context: &Context) -> Result<()> {
+        let fingerprint = self.invite.fingerprint();
+        let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, fingerprint).await?
+        else {
+            return Ok(());
+        };
+
+        // Mark peer as backward verified.
+        peerstate.backward_verified_key_id =
+            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+        peerstate.save_to_db(&context.sql).await?;
+
         Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
             .await?;
         context.emit_event(EventType::ContactsChanged(None));
 
         self.update_next(&context.sql, SecureJoinStep::Completed)
             .await?;
-        Ok(Some(BobHandshakeStage::Completed))
+        Ok(())
     }
 
     /// Sends the requested handshake message to Alice.
@@ -389,13 +374,13 @@ async fn send_handshake_message(
             // Sends the Secure-Join-Auth header in mimefactory.rs.
             msg.param.set(Param::Arg2, invite.authcode());
             msg.param.set_int(Param::GuaranteeE2ee, 1);
+
+            // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+            let bob_fp = load_self_public_key(context).await?.fingerprint();
+            msg.param.set(Param::Arg3, bob_fp.hex());
         }
     };
 
-    // Sends our own fingerprint in the Secure-Join-Fingerprint header.
-    let bob_fp = load_self_public_key(context).await?.fingerprint();
-    msg.param.set(Param::Arg3, bob_fp.hex());
-
     // Sends the grpid in the Secure-Join-Group header.
     if let QrInvite::Group { ref grpid, .. } = invite {
         msg.param.set(Param::Arg4, grpid);

src/smtp.rs

@@ -2,7 +2,7 @@
 
 pub mod send;
 
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use anyhow::{bail, format_err, Context as _, Error, Result};
 use async_smtp::response::{Category, Code, Detail};
@@ -10,6 +10,7 @@ use async_smtp::{self as smtp, EmailAddress, SmtpTransport};
 use tokio::io::BufStream;
 use tokio::task;
 
+use crate::chat::{add_info_msg_with_cmd, ChatId};
 use crate::config::Config;
 use crate::contact::{Contact, ContactId};
 use crate::context::Context;
@@ -26,6 +27,8 @@ use crate::provider::Socket;
 use crate::scheduler::connectivity::ConnectivityStore;
 use crate::socks::Socks5Config;
 use crate::sql;
+use crate::stock_str::unencrypted_email;
+use crate::tools::{self, time_elapsed};
 
 /// SMTP connection, write and read timeout.
 const SMTP_TIMEOUT: Duration = Duration::from_secs(60);
@@ -41,7 +44,7 @@ pub(crate) struct Smtp {
     /// Timestamp of last successful send/receive network interaction
     /// (eg connect or send succeeded). On initialization and disconnect
     /// it is set to None.
-    last_success: Option<SystemTime>,
+    last_success: Option<tools::Time>,
 
     pub(crate) connectivity: ConnectivityStore,
 
@@ -70,11 +73,7 @@ impl Smtp {
     /// have been successfully used the last 60 seconds
     pub fn has_maybe_stale_connection(&self) -> bool {
         if let Some(last_success) = self.last_success {
-            SystemTime::now()
-                .duration_since(last_success)
-                .unwrap_or_default()
-                .as_secs()
-                > 60
+            time_elapsed(&last_success).as_secs() > 60
         } else {
             false
         }
@@ -334,7 +333,7 @@ impl Smtp {
         }
 
         self.transport = Some(transport);
-        self.last_success = Some(SystemTime::now());
+        self.last_success = Some(tools::Time::now());
 
         context.emit_event(EventType::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
@@ -584,7 +583,46 @@ pub(crate) async fn send_msg_to_smtp(
 
     match status {
         SendResult::Retry => {}
-        SendResult::Success | SendResult::Failure(_) => {
+        SendResult::Success => {
+            context
+                .sql
+                .execute("DELETE FROM smtp WHERE id=?", (rowid,))
+                .await?;
+        }
+        SendResult::Failure(ref err) => {
+            if err.to_string().contains("Invalid unencrypted mail") {
+                let res = context
+                    .sql
+                    .query_row_optional(
+                        "SELECT chat_id, timestamp FROM msgs WHERE id=?;",
+                        (msg_id,),
+                        |row| Ok((row.get::<_, ChatId>(0)?, row.get::<_, i64>(1)?)),
+                    )
+                    .await?;
+
+                if let Some((chat_id, timestamp_sort)) = res {
+                    let addr = context.get_config(Config::ConfiguredAddr).await?;
+                    let text = unencrypted_email(
+                        context,
+                        addr.unwrap_or_default()
+                            .split('@')
+                            .nth(1)
+                            .unwrap_or_default(),
+                    )
+                    .await;
+                    add_info_msg_with_cmd(
+                        context,
+                        chat_id,
+                        &text,
+                        crate::mimeparser::SystemMessage::InvalidUnencryptedMail,
+                        timestamp_sort,
+                        None,
+                        None,
+                        None,
+                    )
+                    .await?;
+                };
+            }
             context
                 .sql
                 .execute("DELETE FROM smtp WHERE id=?", (rowid,))
@@ -595,7 +633,13 @@ pub(crate) async fn send_msg_to_smtp(
     match status {
         SendResult::Retry => Err(format_err!("Retry")),
         SendResult::Success => {
-            msg_id.set_delivered(context).await?;
+            if !context
+                .sql
+                .exists("SELECT COUNT(*) FROM smtp WHERE msg_id=?", (msg_id,))
+                .await?
+            {
+                msg_id.set_delivered(context).await?;
+            }
             Ok(())
         }
         SendResult::Failure(err) => Err(format_err!("{}", err)),
@@ -612,7 +656,7 @@ async fn send_mdns(context: &Context, connection: &mut Smtp) -> Result<()> {
 
         let more_mdns = send_mdn(context, connection).await?;
         if !more_mdns {
-            // No more MDNs to send.
+            // No more MDNs to send or one of them failed.
             return Ok(());
         }
     }
@@ -746,7 +790,7 @@ async fn send_mdn_msg_id(
     }
 }
 
-/// Tries to send a single MDN. Returns false if there are no MDNs to send.
+/// Tries to send a single MDN. Returns true if more MDNs should be sent.
 async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
     let mdns_enabled = context.get_config_bool(Config::MdnsEnabled).await?;
     if !mdns_enabled {

src/smtp/send.rs

@@ -6,14 +6,10 @@ use super::Smtp;
 use crate::config::Config;
 use crate::context::Context;
 use crate::events::EventType;
+use crate::tools;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is split to chunks.
-// this does not affect MIME'e `To:` header.
-// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
-pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
-
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Envelope error: {}", _0)]
@@ -43,40 +39,30 @@ impl Smtp {
         }
 
         let message_len_bytes = message.len();
+        let recipients_display = recipients
+            .iter()
+            .map(|x| x.as_ref())
+            .collect::<Vec<&str>>()
+            .join(",");
 
-        let chunk_size = context
-            .get_configured_provider()
-            .await?
-            .and_then(|provider| provider.opt.max_smtp_rcpt_to)
-            .map_or(DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
+        let envelope =
+            Envelope::new(self.from.clone(), recipients.to_vec()).map_err(Error::Envelope)?;
+        let mail = SendableEmail::new(envelope, message);
 
-        for recipients_chunk in recipients.chunks(chunk_size) {
-            let recipients_display = recipients_chunk
-                .iter()
-                .map(|x| x.as_ref())
-                .collect::<Vec<&str>>()
-                .join(",");
+        if let Some(ref mut transport) = self.transport {
+            transport.send(mail).await.map_err(Error::SmtpSend)?;
 
-            let envelope = Envelope::new(self.from.clone(), recipients_chunk.to_vec())
-                .map_err(Error::Envelope)?;
-            let mail = SendableEmail::new(envelope, message);
-
-            if let Some(ref mut transport) = self.transport {
-                transport.send(mail).await.map_err(Error::SmtpSend)?;
-
-                let info_msg = format!(
-                    "Message len={message_len_bytes} was SMTP-sent to {recipients_display}"
-                );
-                info!(context, "{info_msg}.");
-                context.emit_event(EventType::SmtpMessageSent(info_msg));
-                self.last_success = Some(std::time::SystemTime::now());
-            } else {
-                warn!(
-                    context,
-                    "uh? SMTP has no transport, failed to send to {}", recipients_display
-                );
-                return Err(Error::NoTransport);
-            }
+            let info_msg =
+                format!("Message len={message_len_bytes} was SMTP-sent to {recipients_display}");
+            info!(context, "{info_msg}.");
+            context.emit_event(EventType::SmtpMessageSent(info_msg));
+            self.last_success = Some(tools::Time::now());
+        } else {
+            warn!(
+                context,
+                "uh? SMTP has no transport, failed to send to {}", recipients_display
+            );
+            return Err(Error::NoTransport);
         }
         Ok(())
     }

src/sql.rs

@@ -248,7 +248,7 @@ impl Sql {
                 msg.set_text(stock_str::delete_server_turned_off(context).await);
                 add_device_msg(context, None, Some(&mut msg)).await?;
                 context
-                    .set_config(Config::DeleteServerAfter, Some("0"))
+                    .set_config_internal(Config::DeleteServerAfter, Some("0"))
                     .await?;
             }
         }
@@ -259,12 +259,14 @@ impl Sql {
                 match blob.recode_to_avatar_size(context).await {
                     Ok(()) => {
                         context
-                            .set_config(Config::Selfavatar, Some(&avatar))
+                            .set_config_internal(Config::Selfavatar, Some(&avatar))
                             .await?
                     }
                     Err(e) => {
                         warn!(context, "Migrations can't recode avatar, removing. {:#}", e);
-                        context.set_config(Config::Selfavatar, None).await?
+                        context
+                            .set_config_internal(Config::Selfavatar, None)
+                            .await?
                     }
                 }
             }
@@ -572,22 +574,13 @@ impl Sql {
     pub async fn set_raw_config(&self, key: &str, value: Option<&str>) -> Result<()> {
         let mut lock = self.config_cache.write().await;
         if let Some(value) = value {
-            let exists = self
-                .exists("SELECT COUNT(*) FROM config WHERE keyname=?;", (key,))
-                .await?;
-
-            if exists {
-                self.execute("UPDATE config SET value=? WHERE keyname=?;", (value, key))
-                    .await?;
-            } else {
-                self.execute(
-                    "INSERT INTO config (keyname, value) VALUES (?, ?);",
-                    (key, value),
-                )
-                .await?;
-            }
+            self.execute(
+                "INSERT OR REPLACE INTO config (keyname, value) VALUES (?, ?)",
+                (key, value),
+            )
+            .await?;
         } else {
-            self.execute("DELETE FROM config WHERE keyname=?;", (key,))
+            self.execute("DELETE FROM config WHERE keyname=?", (key,))
                 .await?;
         }
         lock.insert(key.to_string(), value.map(|s| s.to_string()));
@@ -608,7 +601,7 @@ impl Sql {
 
         let mut lock = self.config_cache.write().await;
         let value = self
-            .query_get_value("SELECT value FROM config WHERE keyname=?;", (key,))
+            .query_get_value("SELECT value FROM config WHERE keyname=?", (key,))
             .await
             .context(format!("failed to fetch raw config: {key}"))?;
         lock.insert(key.to_string(), value.clone());
@@ -711,7 +704,7 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
     // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not
     // work out for whatever reason or are interrupted by the OS.
     if let Err(e) = context
-        .set_config(Config::LastHousekeeping, Some(&time().to_string()))
+        .set_config_internal(Config::LastHousekeeping, Some(&time().to_string()))
         .await
     {
         warn!(context, "Can't set config: {e:#}.");

src/sql/migrations.rs

@@ -1,11 +1,13 @@
 //! Migrations module.
 
 use anyhow::{Context as _, Result};
+use rusqlite::OptionalExtension;
 
 use crate::config::Config;
-use crate::constants::ShowEmails;
+use crate::constants::{self, ShowEmails};
 use crate::context::Context;
 use crate::imap;
+use crate::message::MsgId;
 use crate::provider::get_provider_by_domain;
 use crate::sql::Sql;
 use crate::tools::EmailAddress;
@@ -366,7 +368,7 @@ UPDATE chats SET protected=1, type=120 WHERE type=130;"#,
         if let Ok(addr) = context.get_primary_self_addr().await {
             if let Ok(domain) = EmailAddress::new(&addr).map(|email| email.domain) {
                 context
-                    .set_config(
+                    .set_config_internal(
                         Config::ConfiguredProvider,
                         get_provider_by_domain(&domain).map(|provider| provider.id),
                     )
@@ -785,6 +787,130 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
         .await?;
     }
 
+    if dbversion < 106 {
+        // Recreate `config` table with UNIQUE constraint on `keyname`.
+        sql.execute_migration(
+            "CREATE TABLE new_config (
+               id INTEGER PRIMARY KEY,
+               keyname TEXT UNIQUE,
+               value TEXT NOT NULL
+             );
+             INSERT OR IGNORE INTO new_config SELECT
+               id, keyname, value
+             FROM config;
+             DROP TABLE config;
+             ALTER TABLE new_config RENAME TO config;
+             CREATE INDEX config_index1 ON config (keyname);",
+            106,
+        )
+        .await?;
+    }
+
+    if dbversion < 107 {
+        sql.execute_migration(
+            "CREATE TABLE new_keypairs (
+               id INTEGER PRIMARY KEY AUTOINCREMENT,
+               private_key UNIQUE NOT NULL,
+               public_key UNIQUE NOT NULL
+             );
+             INSERT OR IGNORE INTO new_keypairs SELECT id, private_key, public_key FROM keypairs;
+
+             INSERT OR IGNORE
+             INTO config (keyname, value)
+             VALUES
+             ('key_id', (SELECT id FROM new_keypairs
+                         WHERE private_key=
+                           (SELECT private_key FROM keypairs
+                            WHERE addr=(SELECT value FROM config WHERE keyname='configured_addr')
+                            AND is_default=1)));
+
+             -- We do not drop the old `keypairs` table for now,
+             -- but move it to `old_keypairs`. We can remove it later
+             -- in next migrations. This may be needed for recovery
+             -- in case something is wrong with the migration.
+             ALTER TABLE keypairs RENAME TO old_keypairs;
+             ALTER TABLE new_keypairs RENAME TO keypairs;
+             ",
+            107,
+        )
+        .await?;
+    }
+
+    if dbversion < 108 {
+        let version = 108;
+        let chunk_size = context
+            .get_configured_provider()
+            .await?
+            .and_then(|provider| provider.opt.max_smtp_rcpt_to)
+            .map_or(constants::DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
+        sql.transaction(move |trans| {
+            Sql::set_db_version_trans(trans, version)?;
+            let id_max =
+                trans.query_row("SELECT IFNULL((SELECT MAX(id) FROM smtp), 0)", (), |row| {
+                    let id_max: i64 = row.get(0)?;
+                    Ok(id_max)
+                })?;
+            while let Some((id, rfc724_mid, mime, msg_id, recipients, retries)) = trans
+                .query_row(
+                    "SELECT id, rfc724_mid, mime, msg_id, recipients, retries FROM smtp \
+                    WHERE id<=? LIMIT 1",
+                    (id_max,),
+                    |row| {
+                        let id: i64 = row.get(0)?;
+                        let rfc724_mid: String = row.get(1)?;
+                        let mime: String = row.get(2)?;
+                        let msg_id: MsgId = row.get(3)?;
+                        let recipients: String = row.get(4)?;
+                        let retries: i64 = row.get(5)?;
+                        Ok((id, rfc724_mid, mime, msg_id, recipients, retries))
+                    },
+                )
+                .optional()?
+            {
+                trans.execute("DELETE FROM smtp WHERE id=?", (id,))?;
+                let recipients = recipients.split(' ').collect::<Vec<_>>();
+                for recipients in recipients.chunks(chunk_size) {
+                    let recipients = recipients.join(" ");
+                    trans.execute(
+                        "INSERT INTO smtp (rfc724_mid, mime, msg_id, recipients, retries) \
+                        VALUES (?, ?, ?, ?, ?)",
+                        (&rfc724_mid, &mime, msg_id, recipients, retries),
+                    )?;
+                }
+            }
+            Ok(())
+        })
+        .await
+        .with_context(|| format!("migration failed for version {version}"))?;
+
+        sql.set_db_version_in_cache(version).await?;
+    }
+
+    if dbversion < 109 {
+        sql.execute_migration(
+            r#"ALTER TABLE acpeerstates
+               ADD COLUMN backward_verified_key_id -- What we think the contact has as our verified key
+               INTEGER;
+               UPDATE acpeerstates
+               SET backward_verified_key_id=(SELECT value FROM config WHERE keyname='key_id')
+               WHERE verified_key IS NOT NULL
+               "#,
+            109,
+        )
+        .await?;
+    }
+
+    if dbversion < 110 {
+        sql.execute_migration(
+            "ALTER TABLE keypairs ADD COLUMN addr TEXT DEFAULT '' COLLATE NOCASE;
+            ALTER TABLE keypairs ADD COLUMN is_default INTEGER DEFAULT 0;
+            ALTER TABLE keypairs ADD COLUMN created INTEGER DEFAULT 0;
+            UPDATE keypairs SET addr=(SELECT value FROM config WHERE keyname='configured_addr'), is_default=1;",
+            110,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?
@@ -824,6 +950,12 @@ impl Sql {
         Ok(())
     }
 
+    async fn set_db_version_in_cache(&self, version: i32) -> Result<()> {
+        let mut lock = self.config_cache.write().await;
+        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
+        Ok(())
+    }
+
     async fn execute_migration(&self, query: &str, version: i32) -> Result<()> {
         self.transaction(move |transaction| {
             Self::set_db_version_trans(transaction, version)?;
@@ -834,10 +966,6 @@ impl Sql {
         .await
         .with_context(|| format!("execute_migration failed for version {version}"))?;
 
-        let mut lock = self.config_cache.write().await;
-        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
-        drop(lock);
-
-        Ok(())
+        self.set_db_version_in_cache(version).await
     }
 }

src/stock_str.rs

@@ -419,6 +419,16 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Member %1$s added."))]
     MsgAddMember = 173,
+
+    #[strum(props(
+        fallback = "⚠️ Your email provider %1$s requires end-to-end encryption which is not setup yet."
+    ))]
+    InvalidUnencryptedMail = 174,
+
+    #[strum(props(
+        fallback = "⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions."
+    ))]
+    CantDecryptOutgoingMsgs = 175,
 }
 
 impl StockMessage {
@@ -745,6 +755,11 @@ pub(crate) async fn cant_decrypt_msg_body(context: &Context) -> String {
     translated(context, StockMessage::CantDecryptMsgBody).await
 }
 
+/// Stock string:`Got outgoing message(s) encrypted for another setup...`.
+pub(crate) async fn cant_decrypt_outgoing_msgs(context: &Context) -> String {
+    translated(context, StockMessage::CantDecryptOutgoingMsgs).await
+}
+
 /// Stock string: `Fingerprints`.
 pub(crate) async fn finger_prints(context: &Context) -> String {
     translated(context, StockMessage::FingerPrints).await
@@ -1285,6 +1300,13 @@ pub(crate) async fn aeap_addr_changed(
         .replace3(new_addr)
 }
 
+/// Stock string: `⚠️ Your email provider %1$s requires end-to-end encryption which is not setup yet. Tap to learn more.`.
+pub(crate) async fn unencrypted_email(context: &Context, provider: &str) -> String {
+    translated(context, StockMessage::InvalidUnencryptedMail)
+        .await
+        .replace1(provider)
+}
+
 pub(crate) async fn aeap_explanation_and_link(
     context: &Context,
     old_addr: &str,

src/sync.rs

@@ -447,7 +447,7 @@ mod tests {
         )?;
         assert_eq!(sync_items.items.len(), 1);
         let SyncDataOrUnknown::SyncData(AlterChat { id, action }) =
-            &sync_items.items.get(0).unwrap().data
+            &sync_items.items.first().unwrap().data
         else {
             bail!("bad item");
         };
@@ -491,7 +491,7 @@ mod tests {
 
         assert_eq!(sync_items.items.len(), 1);
         if let SyncDataOrUnknown::SyncData(AddQrToken(token)) =
-            &sync_items.items.get(0).unwrap().data
+            &sync_items.items.first().unwrap().data
         {
             assert_eq!(token.invitenumber, "in");
             assert_eq!(token.auth, "yip");
@@ -514,7 +514,7 @@ mod tests {
     async fn test_execute_sync_items() -> Result<()> {
         let t = TestContext::new_alice().await;
 
-        assert!(!token::exists(&t, Namespace::Auth, "yip-auth").await);
+        assert!(!token::exists(&t, Namespace::Auth, "yip-auth").await?);
 
         let sync_items = t
             .parse_sync_items(
@@ -537,10 +537,10 @@ mod tests {
                 .await?
                 .is_none()
         );
-        assert!(token::exists(&t, Namespace::InviteNumber, "yip-in").await);
-        assert!(token::exists(&t, Namespace::Auth, "yip-auth").await);
-        assert!(!token::exists(&t, Namespace::Auth, "non-existent").await);
-        assert!(!token::exists(&t, Namespace::Auth, "directly deleted").await);
+        assert!(token::exists(&t, Namespace::InviteNumber, "yip-in").await?);
+        assert!(token::exists(&t, Namespace::Auth, "yip-auth").await?);
+        assert!(!token::exists(&t, Namespace::Auth, "non-existent").await?);
+        assert!(!token::exists(&t, Namespace::Auth, "directly deleted").await?);
 
         Ok(())
     }
@@ -577,13 +577,13 @@ mod tests {
         let alice2 = TestContext::new_alice().await;
         alice2.set_config_bool(Config::SyncMsgs, true).await?;
         alice2.recv_msg(&sent_msg).await;
-        assert!(token::exists(&alice2, token::Namespace::Auth, "testtoken").await);
+        assert!(token::exists(&alice2, token::Namespace::Auth, "testtoken").await?);
         assert_eq!(Chatlist::try_load(&alice2, 0, None, None).await?.len(), 0);
 
         // the same sync message sent to bob must not be executed
         let bob = TestContext::new_bob().await;
         bob.recv_msg(&sent_msg).await;
-        assert!(!token::exists(&bob, token::Namespace::Auth, "testtoken").await);
+        assert!(!token::exists(&bob, token::Namespace::Auth, "testtoken").await?);
 
         Ok(())
     }

src/test_utils.rs

@@ -1047,7 +1047,8 @@ fn print_logevent(logevent: &LogEvent) {
     }
 }
 
-/// Saves the other account's public key as verified.
+/// Saves the other account's public key as verified
+/// and peerstate as backwards verified.
 pub(crate) async fn mark_as_verified(this: &TestContext, other: &TestContext) {
     let mut peerstate = Peerstate::from_header(
         &EncryptHelper::new(other).await.unwrap().get_aheader(),
@@ -1063,6 +1064,7 @@ pub(crate) async fn mark_as_verified(this: &TestContext, other: &TestContext) {
 
     peerstate.verified_key = peerstate.public_key.clone();
     peerstate.verified_key_fingerprint = peerstate.public_key_fingerprint.clone();
+    peerstate.backward_verified_key_id = Some(this.get_config_i64(Config::KeyId).await.unwrap());
 
     peerstate.save_to_db(&this.sql).await.unwrap();
 }

src/token.rs

@@ -103,15 +103,15 @@ pub async fn lookup_or_new(
     token
 }
 
-pub async fn exists(context: &Context, namespace: Namespace, token: &str) -> bool {
-    context
+pub async fn exists(context: &Context, namespace: Namespace, token: &str) -> Result<bool> {
+    let exists = context
         .sql
         .exists(
             "SELECT COUNT(*) FROM tokens WHERE namespc=? AND token=?;",
             (namespace, token),
         )
-        .await
-        .unwrap_or_default()
+        .await?;
+    Ok(exists)
 }
 
 pub async fn delete(context: &Context, namespace: Namespace, token: &str) -> Result<()> {

src/tools.rs

@@ -9,6 +9,13 @@ use std::io::{Cursor, Write};
 use std::mem;
 use std::path::{Path, PathBuf};
 use std::str::from_utf8;
+// If a time value doesn't need to be sent to another host, saved to the db or otherwise used across
+// program restarts, a monotonically nondecreasing clock (`Instant`) should be used. But as
+// `Instant` may use `libc::clock_gettime(CLOCK_MONOTONIC)`, e.g. on Android, and does not advance
+// while being in deep sleep mode, we use `SystemTime` instead, but add an alias for it to document
+// why `Instant` isn't used in those places. Also this can help to switch to another clock impl if
+// we find any.
+pub use std::time::SystemTime as Time;
 use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, Context as _, Result};
@@ -482,6 +489,10 @@ pub(crate) fn time() -> i64 {
         .as_secs() as i64
 }
 
+pub(crate) fn time_elapsed(time: &Time) -> Duration {
+    time.elapsed().unwrap_or_default()
+}
+
 /// Struct containing all mailto information
 #[derive(Debug, Default, Eq, PartialEq)]
 pub struct MailTo {