chore(release): prepare for 1.142.6

If user has not set any settings manually
and provider is not configured,
default to strict TLS checks.

Bug was introduced in
<https://github.com/deltachat/deltachat-core-rust/pull/5854>
(commit 6b4532a08e)
and affects released core 1.142.4 and 1.142.5.

The problem only affects accounts configured
using these core versions with provider
not in the provider database or when using advanced settings.

.github/dependabot.yml

@@ -7,3 +7,10 @@ updates:
     commit-message:
       prefix: "chore(cargo)"
     open-pull-requests-limit: 50
+
+  # Keep GitHub Actions up to date.
+  # <https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot>
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.78.0
+      RUSTUP_TOOLCHAIN: 1.80.1
     steps:
       - uses: actions/checkout@v4
         with:
@@ -59,7 +59,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           show-progress: false
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - uses: EmbarkStudios/cargo-deny-action@v2
         with:
           arguments: --all-features --workspace
           command: check
@@ -95,11 +95,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.78.0
+            rust: 1.80.1
           - os: windows-latest
-            rust: 1.78.0
+            rust: 1.80.1
           - os: macos-latest
-            rust: 1.78.0
+            rust: 1.80.1
 
           # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest

.github/workflows/dependabot.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
+        uses: dependabot/fetch-metadata@v2.2.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Approve a PR

.github/workflows/node-docs.yml

@@ -31,7 +31,7 @@ jobs:
           mv docs js
 
       - name: Upload
-        uses: horochx/deploy-via-scp@v1.0.1
+        uses: horochx/deploy-via-scp@1.1.0
         with:
           user: ${{ secrets.USERNAME }}
           key: ${{ secrets.KEY }}

.github/workflows/upload-docs.yml

@@ -74,7 +74,7 @@ jobs:
           show-progress: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - name: Use Node.js
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v4
         with:
           node-version: '18'
       - name: npm install

CHANGELOG.md

@@ -1,5 +1,371 @@
 # Changelog
 
+## [1.142.6] - 2024-08-15
+
+### Fixes
+
+- Default to strict TLS checks if not configured.
+
+### Miscellaneous Tasks
+
+- deltachat-rpc-client: Fix ruff 0.6.0 warnings.
+
+## [1.142.5] - 2024-08-14
+
+### Fixes
+
+- Still try to create "INBOX.DeltaChat" if couldn't create "DeltaChat" ([#5870](https://github.com/deltachat/deltachat-core-rust/pull/5870)).
+- `store_seen_flags_on_imap`: Skip to next messages if couldn't select folder ([#5870](https://github.com/deltachat/deltachat-core-rust/pull/5870)).
+- Increase timeout for QR generation to 60s ([#5882](https://github.com/deltachat/deltachat-core-rust/pull/5882)).
+
+### Documentation
+
+- Document new `mdns_enabled` behavior (bots do not send MDNs by default).
+
+### CI
+
+- Configure Dependabot to update GitHub Actions.
+
+### Miscellaneous Tasks
+
+- cargo: Bump regex from 1.10.5 to 1.10.6.
+- cargo: Bump serde from 1.0.204 to 1.0.205.
+- deps: Bump horochx/deploy-via-scp from 1.0.1 to 1.1.0.
+- deps: Bump dependabot/fetch-metadata from 1.1.1 to 2.2.0.
+- deps: Bump actions/setup-node from 2 to 4.
+- Update provider database.
+
+## [1.142.4] - 2024-08-09
+
+### Build system
+
+- Downgrade Tokio to 1.38 to fix Android compilation.
+- Use `--locked` with `cargo install`.
+
+### Features / Changes
+
+- Add Config::FixIsChatmail.
+- Always move outgoing auto-generated messages to the mvbox.
+- Disable requesting MDNs for bots by default.
+- Allow using OAuth 2 with SOCKS5.
+- Allow autoconfig when SOCKS5 is enabled.
+- Update provider database.
+- cargo: Update iroh from 0.21 to 0.22 ([#5860](https://github.com/deltachat/deltachat-core-rust/pull/5860)).
+
+### CI
+
+- Update Rust to 1.80.1.
+- Update EmbarkStudios/cargo-deny-action.
+
+### Documentation
+
+- Point to active Header Protection draft
+
+### Refactor
+
+- Derive `Default` for `CertificateChecks`.
+- Merge imap_certificate_checks and smtp_certificate_checks.
+- Remove param_addr_urlencoded argument from get_autoconfig().
+- Pass address to moz_autoconfigure() instead of LoginParam.
+
+## [1.142.3] - 2024-08-04
+
+### Build system
+
+- cargo: Update rusqlite and libsqlite3-sys.
+- Fix cargo warnings about default-features
+- Do not disable "vendored" feature in the workspace.
+- cargo: Bump quick-xml from 0.35.0 to 0.36.1.
+- cargo: Bump uuid from 1.9.1 to 1.10.0.
+- cargo: Bump tokio from 1.38.0 to 1.39.2.
+- cargo: Bump env_logger from 0.11.3 to 0.11.5.
+- Remove sha2 dependency.
+- Remove `backtrace` dependency.
+- Remove direct "quinn" dependency.
+
+## [1.142.2] - 2024-08-02
+
+### Features / Changes
+
+- Try only the full email address if username is unspecified.
+- Sort DNS results by successful connection timestamp ([#5818](https://github.com/deltachat/deltachat-core-rust/pull/5818)).
+
+### Fixes
+
+- Await the tasks after aborting them.
+- Do not reset is_chatmail config on failed reconfiguration.
+- Fix compilation on iOS.
+- Reset configured_provider on reconfiguration.
+
+### Refactor
+
+- Don't update message state to `OutMdnRcvd` anymore.
+
+### Build system
+
+- Use workspace dependencies to make cargo-deny 0.15.1 happy.
+- cargo: Update bytemuck from 0.14.3 to 0.16.3.
+- cargo: Bump toml from 0.8.14 to 0.8.15.
+- cargo: Bump serde_json from 1.0.120 to 1.0.122.
+- cargo: Bump human-panic from 2.0.0 to 2.0.1.
+- cargo: Bump thiserror from 1.0.61 to 1.0.63.
+- cargo: Bump syn from 2.0.68 to 2.0.72.
+- cargo: Bump quoted_printable from 0.5.0 to 0.5.1.
+- cargo: Bump serde from 1.0.203 to 1.0.204.
+
+## [1.142.1] - 2024-07-30
+
+### Features / Changes
+
+- Do not reveal sender's language in read receipts ([#5802](https://github.com/deltachat/deltachat-core-rust/pull/5802)).
+- Try next DNS resolution result if TLS setup fails.
+- Report first error instead of the last on connection failure.
+
+### Fixes
+
+- smtp: Use DNS cache for implicit TLS connections.
+- Imex::import_backup: Unpack all blobs before importing a db ([#4307](https://github.com/deltachat/deltachat-core-rust/pull/4307)).
+- Import_backup_stream: Fix progress stucking at 0.
+- Sql::import: Detach backup db if any step of the import fails.
+- Imex::import_backup: Ignore errors from delete_and_reset_all_device_msgs().
+- Explicitly close the database on account removal.
+
+### Miscellaneous Tasks
+
+- cargo: Update time from 0.3.34 to 0.3.36.
+- cargo: Update iroh from 0.20.0 to 0.21.0.
+
+### Refactor
+
+- Add net/dns submodule.
+- Pass single ALPN around instead of ALPN list.
+- Replace {IMAP,SMTP,HTTP}_TIMEOUT with a single constant.
+- smtp: Unify SMTP connection setup between TLS and STARTTLS.
+- imap: Unify IMAP connection setup in Client::connect().
+- Move DNS resolution into IMAP and SMTP connect code.
+
+### CI
+
+- Update Rust to 1.80.0.
+
+## [1.142.0] - 2024-07-23
+
+### API-Changes
+
+- deltachat-jsonrpc: Add `pinned` property to `FullChat` and `BasicChat`.
+- deltachat-jsonrpc: Allow to set message quote text without referencing quoted message ([#5695](https://github.com/deltachat/deltachat-core-rust/pull/5695)).
+
+### Features / Changes
+
+- cargo: Update iroh from 0.17 to 0.20.
+- iroh: Pass direct addresses from Endpoint to Gossip.
+- New BACKUP2 transfer protocol.
+- Use `[...]` instead of `...` for protected subject.
+- Add email address and fingerprint to exported key file names ([#5694](https://github.com/deltachat/deltachat-core-rust/pull/5694)).
+- Request `imap` ALPN for IMAP TLS connections and `smtp` ALPN for SMTP TLS connections.
+- Limit the size of aggregated WebXDC update to 100 KiB ([#4825](https://github.com/deltachat/deltachat-core-rust/pull/4825)).
+- Don't create ad-hoc group on a member removal message ([#5618](https://github.com/deltachat/deltachat-core-rust/pull/5618)).
+- Don't unarchive a group on a member removal except SELF ([#5618](https://github.com/deltachat/deltachat-core-rust/pull/5618)).
+- Use custom DNS resolver for HTTP(S).
+- Promote fallback DNS results to cached on successful use.
+- Set summary thumbnail path for WebXDCs to "webxdc-icon://last-msg-id" ([#5782](https://github.com/deltachat/deltachat-core-rust/pull/5782)).
+- Do not show the address in invite QR code SVG.
+- Report better error from DcKey::from_asc() ([#5539](https://github.com/deltachat/deltachat-core-rust/pull/5539)).
+- Contact::create_ex: Don't send sync message if nothing changed ([#5705](https://github.com/deltachat/deltachat-core-rust/pull/5705)).
+
+### Fixes
+
+- `Message::set_quote`: Don't forget to remove `Param::ProtectQuote`.
+- Randomize avatar blob filenames to work around caching.
+- Correct copy-pasted DCACCOUNT parsing errors message.
+- Call `send_sync_msg()` only from the SMTP loop ([#5780](https://github.com/deltachat/deltachat-core-rust/pull/5780)).
+- Emit MsgsChanged if the number of unnoticed archived chats could decrease ([#5768](https://github.com/deltachat/deltachat-core-rust/pull/5768)).
+- Reject message with forged From even if no valid signatures are found.
+
+### Refactor
+
+- Move key transfer into its own submodule.
+- Move TempPathGuard into `tools` and use instead of `DeleteOnDrop`.
+- Return error from export_backup() without logging.
+- Reduce boilerplate for migration version increment.
+
+### Tests
+
+- Add test for `get_http_response` JSON-RPC call.
+
+### Build system
+
+- node: Pin node-gyp to version 10.1.
+
+### Miscellaneous Tasks
+
+- cargo: Update hashlink to remove allocator-api2 dependency.
+- cargo: Update openssl to v0.10.66.
+- deps: Bump openssl from 0.10.60 to 0.10.66 in /fuzz.
+- cargo: Update `image` crate to 0.25.2.
+
+## [1.141.2] - 2024-07-09
+
+### Features / Changes
+
+- Add `is_muted` config option.
+- Parse vcards exported by protonmail ([#5723](https://github.com/deltachat/deltachat-core-rust/pull/5723)).
+- Disable sending sync messages for bots ([#5705](https://github.com/deltachat/deltachat-core-rust/pull/5705)).
+
+### Fixes
+
+- Don't fail if going to send plaintext, but some peerstate is missing.
+- Correctly sanitize input everywhere ([#5697](https://github.com/deltachat/deltachat-core-rust/pull/5697)).
+- Do not try to register non-iOS tokens for heartbeats.
+- imap: Reset new_mail if folder is ignored.
+- Use and prefer Date from signed message part ([#5716](https://github.com/deltachat/deltachat-core-rust/pull/5716)).
+- Distinguish between database errors and no gossip topic.
+- MimeFactory::verified: Return true for self-chat.
+
+### Refactor
+
+- `MimeFactory::is_e2ee_guaranteed()`: always respect `Param::ForcePlaintext`.
+- Protect from reusing migration versions ([#5719](https://github.com/deltachat/deltachat-core-rust/pull/5719)).
+- Move `quota_needs_update` calculation to a separate function ([#5683](https://github.com/deltachat/deltachat-core-rust/pull/5683)).
+
+### Documentation
+
+- Document vCards in the specification ([#5724](https://github.com/deltachat/deltachat-core-rust/pull/5724))
+
+### Miscellaneous Tasks
+
+- cargo: Bump toml from 0.8.13 to 0.8.14.
+- cargo: Bump serde_json from 1.0.117 to 1.0.120.
+- cargo: Bump syn from 2.0.66 to 2.0.68.
+- cargo: Bump async-broadcast from 0.7.0 to 0.7.1.
+- cargo: Bump url from 2.5.0 to 2.5.2.
+- cargo: Bump log from 0.4.21 to 0.4.22.
+- cargo: Bump regex from 1.10.4 to 1.10.5.
+- cargo: Bump proptest from 1.4.0 to 1.5.0.
+- cargo: Bump uuid from 1.8.0 to 1.9.1.
+- cargo: Bump backtrace from 0.3.72 to 0.3.73.
+- cargo: Bump quick-xml from 0.31.0 to 0.35.0.
+- cargo: Update yerpc to 0.6.2.
+- cargo: Update rPGP from 0.11 to 0.13.
+
+## [1.141.1] - 2024-06-27
+
+### Fixes
+
+- Update quota if it's stale, not fresh ([#5683](https://github.com/deltachat/deltachat-core-rust/pull/5683)).
+- sql: Assign migration adding msgs.deleted a new number.
+
+### Refactor
+
+- mimefactory: Factor out header confidentiality policy ([#5715](https://github.com/deltachat/deltachat-core-rust/pull/5715)).
+- Improve logging during SMTP/IMAP configuration.
+
+## [1.141.0] - 2024-06-24
+
+### API-Changes
+
+- deltachat-jsonrpc: Add `get_chat_securejoin_qr_code()`.
+- api!(deltachat-rpc-client): make {Account,Chat}.get_qr_code() return no SVG
+  This is a breaking change, old method is renamed into `get_qr_code_svg()`.
+
+### Features / Changes
+
+- Prefer references to fully downloaded messages for chat assignment ([#5645](https://github.com/deltachat/deltachat-core-rust/pull/5645)).
+- Protect From name for verified chats and To names for encrypted chats ([#5166](https://github.com/deltachat/deltachat-core-rust/pull/5166)).
+- Display vCard contact name in the message summary.
+- Case-insensitive search for non-ASCII messages ([#5052](https://github.com/deltachat/deltachat-core-rust/pull/5052)).
+- Remove subject prefix from ad-hoc group names ([#5385](https://github.com/deltachat/deltachat-core-rust/pull/5385)).
+- Replace "Unnamed group" with "👥📧" to avoid translation.
+- Sync `Config::MvboxMove` across devices ([#5680](https://github.com/deltachat/deltachat-core-rust/pull/5680)).
+- Don't reveal profile data to a not yet verified contact ([#5166](https://github.com/deltachat/deltachat-core-rust/pull/5166)).
+- Don't reveal profile data in MDNs ([#5166](https://github.com/deltachat/deltachat-core-rust/pull/5166)).
+
+### Fixes
+
+- Fetch existing messages for bots as `InFresh` ([#4976](https://github.com/deltachat/deltachat-core-rust/pull/4976)).
+- Keep tombstones for two days before deleting ([#3685](https://github.com/deltachat/deltachat-core-rust/pull/3685)).
+- Housekeeping: Delete MDNs and webxdc status updates for tombstones.
+- Delete user-deleted messages on the server even if they show up on IMAP later.
+- Do not send sync messages if bcc_self is disabled.
+- Don't generate Config sync messages for unconfigured accounts.
+- Do not require the Message to render MDN.
+
+### CI
+
+- Update Rust to 1.79.0.
+
+### Documentation
+
+- Remove outdated documentation comment from `send_smtp_messages`.
+- Remove misleading configuration comment.
+
+### Miscellaneous Tasks
+
+- Update curve25519-dalek 4.1.x and suppress 3.2.0 warning.
+- Update provider database.
+
+### Refactor
+
+- Deduplicate dependency versions ([#5691](https://github.com/deltachat/deltachat-core-rust/pull/5691)).
+- Store public key instead of secret key for peer channels.
+
+### Tests
+
+- Image drafted as Viewtype::File is sent as is.
+- python: Set delete_server_after=1 ("delete immediately") for bots ([#4976](https://github.com/deltachat/deltachat-core-rust/pull/4976)).
+- deltachat-rpc-client: Test that webxdc realtime data is not reordered on the sender.
+- python: Wait for bot's DC_EVENT_IMAP_INBOX_IDLE before sending messages to it ([#5699](https://github.com/deltachat/deltachat-core-rust/pull/5699)).
+
+## [1.140.2] - 2024-06-07
+
+### API-Changes
+
+- jsonrpc: Add set_draft_vcard(.., msg_id, contacts).
+
+### Fixes
+
+- Allow fetch_existing_msgs for bots ([#4976](https://github.com/deltachat/deltachat-core-rust/pull/4976)).
+- Remove group member locally even if send_msg() fails ([#5508](https://github.com/deltachat/deltachat-core-rust/pull/5508)).
+- Revert member addition if the corresponding message couldn't be sent ([#5508](https://github.com/deltachat/deltachat-core-rust/pull/5508)).
+- @deltachat/stdio-rpc-server: Make local non-symlinked installation possible by using absolute paths for local dev version ([#5679](https://github.com/deltachat/deltachat-core-rust/pull/5679)).
+
+### Miscellaneous Tasks
+
+- cargo: Bump schemars from 0.8.19 to 0.8.21.
+- cargo: Bump backtrace from 0.3.71 to 0.3.72.
+
+### Refactor
+
+- @deltachat/stdio-rpc-server: Use old school require instead of the experimental json import ([#5628](https://github.com/deltachat/deltachat-core-rust/pull/5628)).
+
+### Tests
+
+- Set fetch_existing_msgs for bots ([#4976](https://github.com/deltachat/deltachat-core-rust/pull/4976)).
+- Don't leave protected group if some member's key is missing ([#5508](https://github.com/deltachat/deltachat-core-rust/pull/5508)).
+
+## [1.140.1] - 2024-06-05
+
+### Fixes
+
+- Retry sending MDNs on temporary error.
+- Set Config::IsChatmail in configure().
+- Do not miss new messages while expunging the folder.
+- Log messages with `info!` instead of `println!`.
+
+### Documentation
+
+- imap: Document why CLOSE is faster than EXPUNGE.
+
+### Refactor
+
+- imap: Make select_folder() accept non-optional folder.
+- Improve SMTP logs and errors.
+- Remove unused `select_folder::Error` variants.
+
+### Tests
+
+- deltachat-rpc-client: reenable `log_cli`.
+
 ## [1.140.0] - 2024-06-04
 
 ### Features / Changes
@@ -4371,3 +4737,15 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.139.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.139.4...v1.139.5
 [1.139.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.139.5...v1.139.6
 [1.140.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.139.6...v1.140.0
+[1.140.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.140.0...v1.140.1
+[1.140.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.140.1...v1.140.2
+[1.141.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.140.2...v1.141.0
+[1.141.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.141.0...v1.141.1
+[1.141.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.141.1...v1.141.2
+[1.142.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.141.2...v1.142.0
+[1.142.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.142.0...v1.142.1
+[1.142.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.142.1...v1.142.2
+[1.142.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.142.2...v1.142.3
+[1.142.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.142.3...v1.142.4
+[1.142.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.142.4...v1.142.5
+[1.142.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.142.5...v1.142.6

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.140.0"
+version = "1.142.6"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.77"
@@ -34,86 +34,83 @@ strip = true
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 deltachat-time = { path = "./deltachat-time" }
-deltachat-contact-tools = { path = "./deltachat-contact-tools" }
+deltachat-contact-tools = { workspace = true }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
-async-broadcast = "0.7.0"
-async-channel = "2.2.1"
+async-broadcast = "0.7.1"
+async-channel = { workspace = true }
 async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
-backtrace = "0.3"
-base64 = "0.22"
+base64 = { workspace = true }
 brotli = { version = "6", default-features=false, features = ["std"] }
-chrono = { workspace = true }
+chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 fast-socks5 = "0.9"
 fd-lock = "4"
-futures = "0.3"
-futures-lite = "2.3.0"
+futures = { workspace = true }
+futures-lite = { workspace = true }
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
 image = { version = "0.25.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 iroh_old = { version = "0.4.2", default-features = false, package = "iroh"}
-iroh-net = "0.17.0"
-iroh-gossip = { version = "0.17.0", features = ["net"] }
-quinn = "0.10.0"
+iroh-net = { version = "0.22.0", default-features = false }
+iroh-gossip = { version = "0.22.0", default-features = false, features = ["net"] }
 kamadak-exif = "0.5.3"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
+libc = { workspace = true }
 mailparse = "0.15"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
-num-traits = "0.2"
+num-traits = { workspace = true }
 once_cell = { workspace = true }
 percent-encoding = "2.3"
 parking_lot = "0.12"
-pgp = { version = "0.11", default-features = false }
+pgp = { version = "0.13", default-features = false }
 qrcodegen = "1.7.0"
-quick-xml = "0.31"
+quick-xml = "0.36"
 quoted_printable = "0.5"
-rand = "0.8"
+rand = { workspace = true }
 regex = { workspace = true }
-reqwest = { version = "0.11.27", features = ["json"] }
+reqwest = { version = "0.12.5", features = ["json"] }
 rusqlite = { workspace = true, features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-sanitize-filename = "0.5"
-serde_json = "1"
-serde = { version = "1.0", features = ["derive"] }
+sanitize-filename = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
 sha-1 = "0.10"
-sha2 = "0.10"
 smallvec = "1.13.2"
 strum = "0.26"
 strum_macros = "0.26"
 tagger = "4.3.4"
 textwrap = "0.16.1"
-thiserror = "1"
-tokio = { version = "1.37.0", features = ["fs", "rt-multi-thread", "macros"] }
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.15", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
-tokio-util = "0.7.9"
+tokio-util = { workspace = true }
 toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
+ansi_term = { workspace = true }
+anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = "2.3.0"
-log = "0.4"
+futures-lite = { workspace = true }
+log = { workspace = true }
 proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+tempfile = { workspace = true }
 testdir = "0.9.0"
-tokio = { version = "1.37.0", features = ["parking_lot", "rt-multi-thread", "macros"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 pretty_assertions = "1.3.0"
 
 [workspace]
@@ -159,10 +156,38 @@ harness = false
 
 [workspace.dependencies]
 anyhow = "1"
+ansi_term = "0.12.1"
+async-channel = "2.3.1"
+base64 = "0.22"
+chrono = { version = "0.4.38", default-features = false }
+deltachat-contact-tools = { path = "deltachat-contact-tools" }
+deltachat-jsonrpc = { path = "deltachat-jsonrpc" }
+deltachat = { path = "." }
+futures = "0.3.30"
+futures-lite = "2.3.0"
+libc = "0.2"
+log = "0.4"
+num-traits = "0.2"
 once_cell = "1.18.0"
+rand = "0.8"
 regex = "1.10"
-rusqlite = "0.31"
-chrono = { version = "0.4.38", default-features=false, features = ["alloc", "clock", "std"] }
+rusqlite = "0.32"
+sanitize-filename = "0.5"
+serde_json = "1"
+serde = "1.0"
+tempfile = "3.10.1"
+thiserror = "1"
+
+# 1.38 is the latest version before `mio` dependency update
+# that broke compilation with Android NDK r23c and r24.
+# Version 1.39.0 cannot be compiled using these NDKs,
+# see issue <https://github.com/tokio-rs/tokio/issues/6748>
+# for details.
+tokio = "~1.38.1"
+
+tokio-util = "0.7.11"
+tracing-subscriber = "0.3"
+yerpc = "0.6.2"
 
 [features]
 default = ["vendored"]
@@ -172,3 +197,6 @@ vendored = [
   "rusqlite/bundled-sqlcipher-vendored-openssl",
   "reqwest/native-tls-vendored"
 ]
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] }

README.md

@@ -30,13 +30,13 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run --locked -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Optionally, install `deltachat-repl` binary with
 ```
-$ cargo install --path deltachat-repl/
+$ cargo install --locked --path deltachat-repl/
 ```
 and run as
 ```

deltachat-contact-tools/Cargo.toml

@@ -12,7 +12,7 @@ anyhow = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true } # Needed in order to `impl rusqlite::types::ToSql for EmailAddress`. Could easily be put behind a feature.
-chrono = { workspace = true }
+chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.

deltachat-contact-tools/src/lib.rs

@@ -22,7 +22,8 @@
     clippy::bool_assert_comparison,
     clippy::manual_split_once,
     clippy::format_push_string,
-    clippy::bool_to_int_with_if
+    clippy::bool_to_int_with_if,
+    clippy::manual_range_contains
 )]
 
 use std::fmt;
@@ -35,10 +36,6 @@ use chrono::{DateTime, NaiveDateTime};
 use once_cell::sync::Lazy;
 use regex::Regex;
 
-// TODOs to clean up:
-// - Check if sanitizing is done correctly everywhere
-// - Apply lints everywhere (https://doc.rust-lang.org/cargo/reference/workspaces.html#the-lints-table)
-
 #[derive(Debug)]
 /// A Contact, as represented in a VCard.
 pub struct VcardContact {
@@ -115,7 +112,9 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
         // If `s` is `EMAIL;TYPE=work:alice@example.com` and `property` is `EMAIL`,
         // then `remainder` is now `;TYPE=work:alice@example.com`
 
-        // TODO this doesn't handle the case where there are quotes around a colon
+        // Note: This doesn't handle the case where there are quotes around a colon,
+        // like `NAME;Foo="Some quoted text: that contains a colon":value`.
+        // This could be improved in the future, but for now, the parsing is good enough.
         let (params, value) = remainder.split_once(':')?;
         // In the example from above, `params` is now `;TYPE=work`
         // and `value` is now `alice@example.com`
@@ -175,7 +174,15 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
         let mut photo = None;
         let mut datetime = None;
 
-        for line in lines.by_ref() {
+        for mut line in lines.by_ref() {
+            if let Some(remainder) = remove_prefix(line, "item1.") {
+                // Remove the group name, if the group is called "item1".
+                // If necessary, we can improve this to also remove groups that are called something different that "item1".
+                //
+                // Search "group name" at https://datatracker.ietf.org/doc/html/rfc6350 for more infos.
+                line = remainder;
+            }
+
             if let Some(email) = vcard_property(line, "email") {
                 addr.get_or_insert(email);
             } else if let Some(name) = vcard_property(line, "fn") {
@@ -183,6 +190,7 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
             } else if let Some(k) = remove_prefix(line, "KEY;PGP;ENCODING=BASE64:")
                 .or_else(|| remove_prefix(line, "KEY;TYPE=PGP;ENCODING=b:"))
                 .or_else(|| remove_prefix(line, "KEY:data:application/pgp-keys;base64,"))
+                .or_else(|| remove_prefix(line, "KEY;PREF=1:data:application/pgp-keys;base64,"))
             {
                 key.get_or_insert(k);
             } else if let Some(p) = remove_prefix(line, "PHOTO;JPEG;ENCODING=BASE64:")
@@ -263,27 +271,27 @@ impl rusqlite::types::ToSql for ContactAddress {
     }
 }
 
-/// Make the name and address
+/// Takes a name and an address and sanitizes them:
+/// - Extracts a name from the addr if the addr is in form "Alice <alice@example.org>"
+/// - Removes special characters from the name, see [`sanitize_name()`]
+/// - Removes the name if it is equal to the address by setting it to ""
 pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
     let (name, addr) = if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.is_empty() {
-                strip_rtlo_characters(captures.get(1).map_or("", |m| m.as_str()))
+                captures.get(1).map_or("", |m| m.as_str())
             } else {
-                strip_rtlo_characters(name)
+                name
             },
             captures
                 .get(2)
                 .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
-        (
-            strip_rtlo_characters(&normalize_name(name)),
-            addr.to_string(),
-        )
+        (name, addr.to_string())
     };
-    let mut name = normalize_name(&name);
+    let mut name = sanitize_name(name);
 
     // If the 'display name' is just the address, remove it:
     // Otherwise, the contact would sometimes be shown as "alice@example.com (alice@example.com)" (see `get_name_n_addr()`).
@@ -295,31 +303,77 @@ pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     (name, addr)
 }
 
-/// Normalize a name.
+/// Sanitizes a name.
 ///
-/// - Remove quotes (come from some bad MUA implementations)
-/// - Trims the resulting string
-///
-/// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
-pub fn normalize_name(full_name: &str) -> String {
-    let full_name = full_name.trim();
-    if full_name.is_empty() {
-        return full_name.into();
-    }
+/// - Removes newlines and trims the string
+/// - Removes quotes (come from some bad MUA implementations)
+/// - Removes potentially-malicious bidi characters
+pub fn sanitize_name(name: &str) -> String {
+    let name = sanitize_single_line(name);
 
-    match full_name.as_bytes() {
-        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
-            .get(1..full_name.len() - 1)
+    match name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => name
+            .get(1..name.len() - 1)
             .map_or("".to_string(), |s| s.trim().to_string()),
-        _ => full_name.to_string(),
+        _ => name.to_string(),
     }
 }
 
+/// Sanitizes user input
+///
+/// - Removes newlines and trims the string
+/// - Removes potentially-malicious bidi characters
+pub fn sanitize_single_line(input: &str) -> String {
+    sanitize_bidi_characters(input.replace(['\n', '\r'], " ").trim())
+}
+
 const RTLO_CHARACTERS: [char; 5] = ['\u{202A}', '\u{202B}', '\u{202C}', '\u{202D}', '\u{202E}'];
-/// This method strips all occurrences of the RTLO Unicode character.
-/// [Why is this needed](https://github.com/deltachat/deltachat-core-rust/issues/3479)?
-pub fn strip_rtlo_characters(input_str: &str) -> String {
-    input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "")
+const ISOLATE_CHARACTERS: [char; 3] = ['\u{2066}', '\u{2067}', '\u{2068}'];
+const POP_ISOLATE_CHARACTER: char = '\u{2069}';
+/// Some control unicode characters can influence whether adjacent text is shown from
+/// left to right or from right to left.
+///
+/// Since user input is not supposed to influence how adjacent text looks,
+/// this function removes some of these characters.
+///
+/// Also see https://github.com/deltachat/deltachat-core-rust/issues/3479.
+pub fn sanitize_bidi_characters(input_str: &str) -> String {
+    // RTLO_CHARACTERS are apparently rarely used in practice.
+    // They can impact all following text, so, better remove them all:
+    let input_str = input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "");
+
+    // If the ISOLATE characters are not ended with a POP DIRECTIONAL ISOLATE character,
+    // we regard the input as potentially malicious and simply remove all ISOLATE characters.
+    // See https://en.wikipedia.org/wiki/Bidirectional_text#Unicode_bidi_support
+    // and https://www.w3.org/International/questions/qa-bidi-unicode-controls.en
+    // for an explanation about ISOLATE characters.
+    fn isolate_characters_are_valid(input_str: &str) -> bool {
+        let mut isolate_character_nesting: i32 = 0;
+        for char in input_str.chars() {
+            if ISOLATE_CHARACTERS.contains(&char) {
+                isolate_character_nesting += 1;
+            } else if char == POP_ISOLATE_CHARACTER {
+                isolate_character_nesting -= 1;
+            }
+
+            // According to Wikipedia, 125 levels are allowed:
+            // https://en.wikipedia.org/wiki/Unicode_control_characters
+            // (although, in practice, we could also significantly lower this number)
+            if isolate_character_nesting < 0 || isolate_character_nesting > 125 {
+                return false;
+            }
+        }
+        isolate_character_nesting == 0
+    }
+
+    if isolate_characters_are_valid(&input_str) {
+        input_str
+    } else {
+        input_str.replace(
+            |char| ISOLATE_CHARACTERS.contains(&char) || POP_ISOLATE_CHARACTER == char,
+            "",
+        )
+    }
 }
 
 /// Returns false if addr is an invalid address, otherwise true.
@@ -668,4 +722,89 @@ END:VCARD
             assert_eq!(contacts[0].profile_image.as_deref().unwrap(), "/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAL8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==");
         }
     }
+
+    #[test]
+    fn test_protonmail_vcard() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN;PREF=1:Alice Wonderland
+UID:proton-web-03747582-328d-38dc-5ddd-000000000000
+ITEM1.EMAIL;PREF=1:alice@example.org
+ITEM1.KEY;PREF=1:data:application/pgp-keys;base64,aaaaaaaaaaaaaaaaaaaaaaaaa
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ITEM1.KEY;PREF=2:data:application/pgp-keys;base64,bbbbbbbbbbbbbbbbbbbbbbbbb
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+ITEM1.X-PM-ENCRYPT:true
+ITEM1.X-PM-SIGN:true
+END:VCARD",
+        );
+
+        assert_eq!(contacts.len(), 1);
+        assert_eq!(&contacts[0].addr, "alice@example.org");
+        assert_eq!(&contacts[0].authname, "Alice Wonderland");
+        assert_eq!(contacts[0].key.as_ref().unwrap(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+        assert!(contacts[0].timestamp.is_err());
+        assert_eq!(contacts[0].profile_image, None);
+    }
+
+    #[test]
+    fn test_sanitize_name() {
+        assert_eq!(&sanitize_name(" hello world   "), "hello world");
+        assert_eq!(&sanitize_name("<"), "<");
+        assert_eq!(&sanitize_name(">"), ">");
+        assert_eq!(&sanitize_name("'"), "'");
+        assert_eq!(&sanitize_name("\""), "\"");
+    }
+
+    #[test]
+    fn test_sanitize_single_line() {
+        assert_eq!(sanitize_single_line("Hi\naiae "), "Hi aiae");
+        assert_eq!(sanitize_single_line("\r\nahte\n\r"), "ahte");
+    }
+
+    #[test]
+    fn test_sanitize_bidi_characters() {
+        // Legit inputs:
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat\u{2069}"),
+            "Tes\u{2067}ting Delta Chat\u{2069}"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"),
+            "Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"),
+            "Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"
+        );
+
+        // Potentially-malicious inputs:
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{202C}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Testing Delta Chat\u{2069}"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2069}ting Delta Chat\u{2067}"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2068}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+    }
 }

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.140.0"
+version = "1.142.6"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -14,21 +14,21 @@ name = "deltachat"
 crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
-deltachat = { path = "../", default-features = false }
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
-libc = "0.2"
+deltachat = { workspace = true, default-features = false }
+deltachat-jsonrpc = { workspace = true, optional = true }
+libc = { workspace = true }
 human-panic = { version = "2", default-features = false }
-num-traits = "0.2"
-serde_json = "1.0"
-tokio = { version = "1.37.0", features = ["rt-multi-thread"] }
-anyhow = "1"
-thiserror = "1"
-rand = "0.8"
-once_cell = "1.18.0"
-yerpc = { version = "0.5.1", features = ["anyhow_expose"] }
+num-traits = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+anyhow = { workspace = true }
+thiserror = { workspace = true }
+rand = { workspace = true }
+once_cell = { workspace = true }
+yerpc = { workspace = true, features = ["anyhow_expose"] }
 
 [features]
 default = ["vendored"]
-vendored = ["deltachat/vendored"]
+vendored = ["deltachat/vendored", "deltachat-jsonrpc/vendored"]
 jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/deltachat.h

@@ -409,7 +409,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `socks5_user` = SOCKS5 proxy username
  * - `socks5_password` = SOCKS5 proxy password
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
- * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
+ * - `smtp_certificate_checks` = deprecated option, should be set to the same value as `imap_certificate_checks` but ignored by the new core
  * - `displayname`  = Own name to use when sending messages. MUAs are allowed to spread this way e.g. using CC, defaults to empty
  * - `selfstatus`   = Own status to display, e.g. in e-mail footers, defaults to empty
  * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
@@ -420,7 +420,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    and also recoded to a reasonable size.
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
- *                    1=send and request read receipts (default)
+ *                    1=send and request read receipts
+ *                    default=send and request read receipts, only send but not reuqest if `bot` is set
  * - `bcc_self`     = 0=do not send a copy of outgoing messages to self (default),
  *                    1=send a copy of outgoing messages to self.
  *                    Sending messages to self is needed for a proper multi-account setup,
@@ -481,8 +482,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `bot`          = Set to "1" if this is a bot.
  *                    Prevents adding the "Device messages" and "Saved messages" chats,
  *                    adds Auto-Submitted header to outgoing messages,
- *                    accepts contact requests automatically (calling dc_accept_chat() is not needed for bots)
- *                    and does not cut large incoming text messages.
+ *                    accepts contact requests automatically (calling dc_accept_chat() is not needed),
+ *                    does not cut large incoming text messages,
+ *                    handles existing messages the same way as new ones if `fetch_existing_msgs=1`.
  * - `last_msg_id` = database ID of the last message processed by the bot.
  *                   This ID and IDs below it are guaranteed not to be returned
  *                   by dc_get_next_msgs() and dc_wait_next_msgs().
@@ -493,8 +495,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                   For most bots calling `dc_markseen_msgs()` is the
  *                   recommended way to update this value
  *                   even for self-sent messages.
- * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
- *                    0=do not fetch existing messages on configure.
+ * - `fetch_existing_msgs` = 0=do not fetch existing messages on configure (default),
+ *                    1=fetch most recent existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
  * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
  *                    0=use IMAP IDLE if the server supports it.
@@ -518,6 +520,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    1=After the key changed, `dc_chat_can_send()` returns false and `dc_chat_is_protection_broken()` returns true
  *                    until `dc_accept_chat()` is called.
  * - `is_chatmail` = 1 if the the server is a chatmail server, 0 otherwise.
+ * - `is_muted`     = Whether a context is muted by the user.
+ *                    Muted contexts should not sound, vibrate or show notifications.
+ *                    In contrast to `dc_set_chat_mute_duration()`,
+ *                    fresh message and badge counters are not changed by this setting,
+ *                    but should be tuned down where appropriate.
  * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
  *                    The prefix should be followed by the system and maybe subsystem,
  *                    e.g. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
@@ -2498,6 +2505,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
 #define         DC_QR_BACKUP                 251
+#define         DC_QR_BACKUP2                252
 #define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
@@ -2544,6 +2552,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr() and then dc_configure().
  *
  * - DC_QR_BACKUP:
+ * - DC_QR_BACKUP2:
  *   ask the user if they want to set up a new device.
  *   If so, pass the qr-code to dc_receive_backup().
  *
@@ -6642,12 +6651,16 @@ void dc_event_unref(dc_event_t* event);
 /// "Message opened"
 ///
 /// Used in subjects of outgoing read receipts.
+///
+/// @deprecated Deprecated 2024-07-26
 #define DC_STR_READRCPT                   31
 
 /// "The message '%1$s' you sent was displayed on the screen of the recipient."
 ///
 /// Used as message text of outgoing read receipts.
 /// - %1$s will be replaced by the subject of the displayed message
+///
+/// @deprecated Deprecated 2024-06-23
 #define DC_STR_READRCPT_MAILBODY          32
 
 /// @deprecated Deprecated, this string is no longer needed.
@@ -7366,7 +7379,7 @@ void dc_event_unref(dc_event_t* event);
 /// Used as info message.
 #define DC_STR_SECUREJOIN_WAIT_TIMEOUT 191
 
-/// "Contact"
+/// "Contact". Deprecated, currently unused.
 #define DC_STR_CONTACT 200
 
 /**

deltachat-ffi/src/lib.rs

@@ -4364,7 +4364,7 @@ pub unsafe extern "C" fn dc_backup_provider_wait(provider: *mut dc_backup_provid
     let ctx = &*ffi_provider.context;
     let provider = &mut ffi_provider.provider;
     block_on(provider)
-        .context("Failed to await BackupProvider")
+        .context("Failed to await backup provider")
         .log_err(ctx)
         .set_last_error(ctx)
         .ok();
@@ -4418,7 +4418,7 @@ trait ResultExt<T, E> {
     /// Like `log_err()`, but:
     /// - returns the default value instead of an Err value.
     /// - emits an error instead of a warning for an [Err] result. This means
-    /// that the error will be shown to the user in a small pop-up.
+    ///   that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
 }
 

deltachat-ffi/src/lot.rs

@@ -50,6 +50,7 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
                 Qr::Backup { .. } => None,
+                Qr::Backup2 { .. } => None,
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
                 Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
@@ -102,6 +103,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
                 Qr::Account { .. } => LotState::QrAccount,
                 Qr::Backup { .. } => LotState::QrBackup,
+                Qr::Backup2 { .. } => LotState::QrBackup2,
                 Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
                 Qr::Addr { .. } => LotState::QrAddr,
                 Qr::Url { .. } => LotState::QrUrl,
@@ -127,6 +129,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::Backup { .. } => Default::default(),
+                Qr::Backup2 { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
                 Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
@@ -177,6 +180,8 @@ pub enum LotState {
 
     QrBackup = 251,
 
+    QrBackup2 = 252,
+
     /// text1=domain, text2=instance pattern
     QrWebrtcInstance = 260,
 

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.140.0"
+version = "1.142.6"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -13,30 +13,30 @@ path = "src/webserver.rs"
 required-features = ["webserver"]
 
 [dependencies]
-anyhow = "1"
-deltachat = { path = ".." }
-deltachat-contact-tools = { path = "../deltachat-contact-tools" }
-num-traits = "0.2"
-schemars = "0.8.19"
-serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.10.1"
-log = "0.4"
-async-channel = { version = "2.2.1" }
-futures = { version = "0.3.30" }
-serde_json = "1"
-yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+anyhow = { workspace = true }
+deltachat = { workspace = true }
+deltachat-contact-tools = { workspace = true }
+num-traits = { workspace = true }
+schemars = "0.8.21"
+serde = { workspace = true, features = ["derive"] }
+tempfile = { workspace = true }
+log = { workspace = true }
+async-channel = { workspace = true }
+futures = { workspace = true }
+serde_json = { workspace = true }
+yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
-tokio = { version = "1.37.0" }
-sanitize-filename = "0.5"
+tokio = { workspace = true }
+sanitize-filename = { workspace = true }
 walkdir = "2.5.0"
-base64 = "0.22"
+base64 = { workspace = true }
 
 # optional dependencies
 axum = { version = "0.7", optional = true, features = ["ws"] }
-env_logger = { version = "0.11.3", optional = true }
+env_logger = { version = "0.11.5", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.37.0", features = ["full", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["full", "rt-multi-thread"] }
 
 
 [features]

deltachat-jsonrpc/src/api.rs

@@ -707,7 +707,22 @@ impl CommandApi {
         ChatId::new(chat_id).get_encryption_info(&ctx).await
     }
 
-    /// Get QR code (text and SVG) that will offer an Setup-Contact or Verified-Group invitation.
+    /// Get QR code text that will offer a [SecureJoin](https://securejoin.delta.chat/) invitation.
+    ///
+    /// If `chat_id` is a group chat ID, SecureJoin QR code for the group is returned.
+    /// If `chat_id` is unset, setup contact QR code is returned.
+    async fn get_chat_securejoin_qr_code(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+    ) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let chat = chat_id.map(ChatId::new);
+        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
+        Ok(qr)
+    }
+
+    /// Get QR code (text and SVG) that will offer a Setup-Contact or Verified-Group invitation.
     /// The QR code is compatible to the OPENPGP4FPR format
     /// so that a basic fingerprint comparison also works e.g. with OpenKeychain.
     ///
@@ -729,10 +744,9 @@ impl CommandApi {
     ) -> Result<(String, String)> {
         let ctx = self.get_context(account_id).await?;
         let chat = chat_id.map(ChatId::new);
-        Ok((
-            securejoin::get_securejoin_qr(&ctx, chat).await?,
-            get_securejoin_qr_svg(&ctx, chat).await?,
-        ))
+        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
+        let svg = get_securejoin_qr_svg(&ctx, chat).await?;
+        Ok((qr, svg))
     }
 
     /// Continue a Setup-Contact or Verified-Group-Invite protocol
@@ -1476,6 +1490,20 @@ impl CommandApi {
         deltachat::contact::make_vcard(&ctx, &contacts).await
     }
 
+    /// Sets vCard containing the given contacts to the message draft.
+    async fn set_draft_vcard(
+        &self,
+        account_id: u32,
+        msg_id: u32,
+        contacts: Vec<u32>,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts: Vec<_> = contacts.iter().map(|&c| ContactId::new(c)).collect();
+        let mut msg = Message::load_from_db(&ctx, MsgId::new(msg_id)).await?;
+        msg.make_vcard(&ctx, &contacts).await?;
+        msg.get_chat_id().set_draft(&ctx, Some(&mut msg)).await
+    }
+
     // ---------------------------------------------
     //                   chat
     // ---------------------------------------------
@@ -1644,10 +1672,10 @@ impl CommandApi {
     ///
     /// This call will block until the QR code is ready,
     /// even if there is no concurrent call to [`CommandApi::provide_backup`],
-    /// but will fail after 10 seconds to avoid deadlocks.
+    /// but will fail after 60 seconds to avoid deadlocks.
     async fn get_backup_qr(&self, account_id: u32) -> Result<String> {
         let qr = tokio::time::timeout(
-            Duration::from_secs(10),
+            Duration::from_secs(60),
             self.inner_get_backup_qr(account_id),
         )
         .await
@@ -1663,13 +1691,13 @@ impl CommandApi {
     ///
     /// This call will block until the QR code is ready,
     /// even if there is no concurrent call to [`CommandApi::provide_backup`],
-    /// but will fail after 10 seconds to avoid deadlocks.
+    /// but will fail after 60 seconds to avoid deadlocks.
     ///
     /// Returns the QR code rendered as an SVG image.
     async fn get_backup_qr_svg(&self, account_id: u32) -> Result<String> {
         let ctx = self.get_context(account_id).await?;
         let qr = tokio::time::timeout(
-            Duration::from_secs(10),
+            Duration::from_secs(60),
             self.inner_get_backup_qr(account_id),
         )
         .await

deltachat-jsonrpc/src/api/types/chat.rs

@@ -32,6 +32,7 @@ pub struct FullChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
+    pinned: bool,
     // subtitle  - will be moved to frontend because it uses translation functions
     chat_type: u32,
     is_unpromoted: bool,
@@ -104,6 +105,7 @@ impl FullChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),
@@ -153,6 +155,7 @@ pub struct BasicChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
+    pinned: bool,
     chat_type: u32,
     is_unpromoted: bool,
     is_self_talk: bool,
@@ -180,6 +183,7 @@ impl BasicChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),

deltachat-jsonrpc/src/api/types/message.rs

@@ -577,7 +577,9 @@ pub struct MessageData {
     pub file: Option<String>,
     pub location: Option<(f64, f64)>,
     pub override_sender_name: Option<String>,
+    /// Quoted message id. Takes preference over `quoted_text` (see below).
     pub quoted_message_id: Option<u32>,
+    pub quoted_text: Option<String>,
 }
 
 impl MessageData {
@@ -613,6 +615,9 @@ impl MessageData {
                     ),
                 )
                 .await?;
+        } else if let Some(text) = self.quoted_text {
+            let protect = false;
+            message.set_quote_text(Some((text, protect)));
         }
         Ok(message)
     }

deltachat-jsonrpc/src/api/types/qr.rs

@@ -35,6 +35,11 @@ pub enum QrObject {
     Backup {
         ticket: String,
     },
+    Backup2 {
+        auth_token: String,
+
+        node_addr: String,
+    },
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
@@ -132,6 +137,14 @@ impl From<Qr> for QrObject {
             Qr::Backup { ticket } => QrObject::Backup {
                 ticket: ticket.to_string(),
             },
+            Qr::Backup2 {
+                ref node_addr,
+                auth_token,
+            } => QrObject::Backup2 {
+                node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
+
+                auth_token,
+            },
             Qr::WebrtcInstance {
                 domain,
                 instance_pattern,

deltachat-jsonrpc/typescript/package.json

@@ -3,7 +3,7 @@
   "dependencies": {
     "@deltachat/tiny-emitter": "3.0.0",
     "isomorphic-ws": "^4.0.1",
-    "yerpc": "^0.4.3"
+    "yerpc": "^0.6.2"
   },
   "devDependencies": {
     "@types/chai": "^4.2.21",
@@ -58,5 +58,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.140.0"
+  "version": "1.142.6"
 }

deltachat-repl/Cargo.toml

@@ -1,20 +1,20 @@
 [package]
 name = "deltachat-repl"
-version = "1.140.0"
+version = "1.142.6"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [dependencies]
-ansi_term = "0.12.1"
-anyhow = "1"
-deltachat = { path = "..", features = ["internals"]}
+ansi_term = { workspace = true }
+anyhow = { workspace = true }
+deltachat = { workspace = true, features = ["internals"]}
 dirs = "5"
-log = "0.4.21"
-rusqlite = "0.31"
+log = { workspace = true }
+rusqlite = { workspace = true }
 rustyline = "14"
-tokio = { version = "1.37.0", features = ["fs", "rt-multi-thread", "macros"] }
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
 
 [features]
 default = ["vendored"]

deltachat-repl/src/cmdline.rs

@@ -339,7 +339,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  receive-backup <qr>\n\
                  export-keys\n\
                  import-keys\n\
-                 export-setup\n\
                  poke [<eml-file>|<folder>|<addr> <key-file>]\n\
                  reset <flags>\n\
                  stop\n\
@@ -504,17 +503,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "import-keys" => {
             imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
         }
-        "export-setup" => {
-            let setup_code = create_setup_code(&context);
-            let file_name = blobdir.join("autocrypt-setup-message.html");
-            let file_content = render_setup_file(&context, &setup_code).await?;
-            fs::write(&file_name, file_content).await?;
-            println!(
-                "Setup message written to: {}\nSetup code: {}",
-                file_name.display(),
-                &setup_code,
-            );
-        }
         "poke" => {
             ensure!(poke_spec(&context, Some(arg1)).await, "Poke failed");
         }

deltachat-repl/src/main.rs

@@ -152,7 +152,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 14] = [
+const IMEX_COMMANDS: [&str; 13] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -163,7 +163,6 @@ const IMEX_COMMANDS: [&str; 14] = [
     "receive-backup",
     "export-keys",
     "import-keys",
-    "export-setup",
     "poke",
     "reset",
     "stop",

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.140.0"
+version = "1.142.6"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -250,12 +250,16 @@ class Account:
         """
         return Chat(self, self._rpc.secure_join(self.id, qrdata))
 
-    def get_qr_code(self) -> tuple[str, str]:
-        """Get Setup-Contact QR Code text and SVG data.
+    def get_qr_code(self) -> str:
+        """Get Setup-Contact QR Code text.
 
-        this data needs to be transferred to another Delta Chat account
+        This data needs to be transferred to another Delta Chat account
         in a second channel, typically used by mobiles with QRcode-show + scan UX.
         """
+        return self._rpc.get_chat_securejoin_qr_code(self.id, None)
+
+    def get_qr_code_svg(self) -> tuple[str, str]:
+        """Get Setup-Contact QR code text and SVG."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.id, None)
 
     def get_message_by_id(self, msg_id: int) -> Message:

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -96,7 +96,11 @@ class Chat:
         """Return encryption info for this chat."""
         return self._rpc.get_chat_encryption_info(self.account.id, self.id)
 
-    def get_qr_code(self) -> tuple[str, str]:
+    def get_qr_code(self) -> str:
+        """Get Join-Group QR code text."""
+        return self._rpc.get_chat_securejoin_qr_code(self.account.id, self.id)
+
+    def get_qr_code_svg(self) -> tuple[str, str]:
         """Get Join-Group QR code text and SVG data."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.account.id, self.id)
 

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -114,13 +114,13 @@ class ACFactory:
         return to_client.run_until(lambda e: e.kind == EventType.INCOMING_MSG)
 
 
-@pytest.fixture()
+@pytest.fixture
 def rpc(tmp_path) -> AsyncGenerator:
     rpc_server = Rpc(accounts_dir=str(tmp_path / "accounts"))
     with rpc_server:
         yield rpc_server
 
 
-@pytest.fixture()
+@pytest.fixture
 def acfactory(rpc) -> AsyncGenerator:
     return ACFactory(DeltaChat(rpc))

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -210,6 +210,7 @@ def test_multidevice_sync_chat(acfactory: ACFactory) -> None:
     alice_second_device.clear_all_events()
     alice_chat_bob.pin()
     wait_for_chatlist_specific_item(alice_second_device, alice_chat_bob.id)
+    assert alice_second_device.get_chat_by_id(alice_chat_bob.id).get_basic_snapshot().pinned
 
     alice_second_device.clear_all_events()
     alice_chat_bob.mute()

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -12,10 +12,11 @@ import threading
 import time
 
 import pytest
+
 from deltachat_rpc_client import EventType
 
 
-@pytest.fixture()
+@pytest.fixture
 def path_to_webxdc(request):
     p = request.path.parent.parent.parent.joinpath("test-data/webxdc/chess.xdc")
     assert p.exists()
@@ -68,7 +69,7 @@ def wait_receive_realtime_data(msg_data_list):
         if event.kind == EventType.WEBXDC_REALTIME_DATA:
             for i, (msg, data) in enumerate(msg_data_list):
                 if msg.id == event.msg_id:
-                    assert data == event.data
+                    assert list(data) == event.data
                     log(f"msg {msg.id}: got correct realtime data {data}")
                     del msg_data_list[i]
                     break
@@ -184,3 +185,26 @@ def test_no_duplicate_messages(acfactory, path_to_webxdc):
         if event.kind == EventType.WEBXDC_REALTIME_DATA:
             assert int(bytes(event.data).decode()) > n
             break
+
+
+def test_no_reordering(acfactory, path_to_webxdc):
+    """Test that sending a lot of realtime messages does not result in reordering."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, b"hello")
+    wait_receive_realtime_data([(ac2_webxdc_msg, b"hello")])
+
+    for i in range(200):
+        ac1_webxdc_msg.send_webxdc_realtime_data([i])
+
+    for i in range(200):
+        while 1:
+            event = ac2.wait_for_event()
+            if event.kind == EventType.WEBXDC_REALTIME_DATA and bytes(event.data) != b"hello":
+                if event.data[0] == i:
+                    break
+                pytest.fail("Reordering detected")

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,13 +1,14 @@
 import logging
 
 import pytest
+
 from deltachat_rpc_client import Chat, EventType, SpecialContactId
 
 
 def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
 
     alice.wait_for_securejoin_inviter_success()
@@ -30,13 +31,35 @@ def test_qr_setup_contact(acfactory, tmp_path) -> None:
     bob2.export_self_keys(tmp_path)
 
     logging.info("Bob imports a key")
-    bob.import_self_keys(tmp_path / "private-key-default.asc")
+    bob.import_self_keys(tmp_path)
 
     assert bob.get_config("key_id") == "2"
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert not bob_contact_alice_snapshot.is_verified
 
 
+def test_qr_setup_contact_svg(acfactory) -> None:
+    alice = acfactory.new_configured_account()
+    _, _, domain = alice.get_config("addr").rpartition("@")
+
+    _qr_code, svg = alice.get_qr_code_svg()
+
+    # Test that email address is in SVG
+    # when we have no display name.
+    # Check only the domain name, because
+    # long address may be split over multiple lines
+    # and not matched.
+    assert domain in svg
+
+    alice.set_config("displayname", "Alice")
+
+    # Test that display name is used
+    # in SVG and no address is visible.
+    _qr_code, svg = alice.get_qr_code_svg()
+    assert domain not in svg
+    assert "Alice" in svg
+
+
 @pytest.mark.parametrize("protect", [True, False])
 def test_qr_securejoin(acfactory, protect):
     alice, bob = acfactory.get_online_accounts(2)
@@ -46,7 +69,7 @@ def test_qr_securejoin(acfactory, protect):
     assert alice_chat.get_basic_snapshot().is_protected == protect
 
     logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
 
     # Check that at least some of the handshake messages are deleted.
@@ -91,7 +114,7 @@ def test_qr_securejoin_contact_request(acfactory) -> None:
 
     alice_chat = alice.create_group("Verified group", protect=True)
     logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
     while True:
         event = bob.wait_for_event()
@@ -106,7 +129,7 @@ def test_qr_readreceipt(acfactory) -> None:
     alice, bob, charlie = acfactory.get_online_accounts(3)
 
     logging.info("Bob and Charlie setup contact with Alice")
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
 
     bob.secure_join(qr_code)
     charlie.secure_join(qr_code)
@@ -168,13 +191,13 @@ def test_setup_contact_resetup(acfactory) -> None:
     """Tests that setup contact works after Alice resets the device and changes the key."""
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     alice = acfactory.resetup_account(alice)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -188,7 +211,7 @@ def test_verified_group_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -205,7 +228,7 @@ def test_verified_group_recovery(acfactory) -> None:
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
+    qr_code = ac3.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -252,7 +275,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -269,7 +292,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
+    qr_code = ac3.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -336,7 +359,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
     ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
 
     logging.info("ac3: verify with ac2")
-    qr_code, _svg = ac2.get_qr_code()
+    qr_code = ac2.get_qr_code()
     ac3.secure_join(qr_code)
     ac2.wait_for_securejoin_inviter_success()
 
@@ -346,7 +369,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
 
     logging.info("ac1: create verified group that ac2 fully joins")
     ch1 = ac1.create_group("Group", protect=True)
-    qr_code, _svg = ch1.get_qr_code()
+    qr_code = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
 
@@ -359,7 +382,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
-    qr_code, _svg = ch1.get_qr_code()
+    qr_code = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.remove()
     logging.info("ac2 now has pending bobstate but ac1 is shutoff")
@@ -381,7 +404,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
-    qr_code, _svg = vg.get_qr_code()
+    qr_code = vg.get_qr_code()
     ac4.secure_join(qr_code)
     ac3.wait_for_securejoin_inviter_success()
     while 1:
@@ -402,7 +425,7 @@ def test_qr_new_group_unblocked(acfactory):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_chat = ac1.create_group("Group for joining", protect=True)
-    qr_code, _svg = ac1_chat.get_qr_code()
+    qr_code = ac1_chat.get_qr_code()
     ac2.secure_join(qr_code)
 
     ac1.wait_for_securejoin_inviter_success()
@@ -425,7 +448,7 @@ def test_aeap_flow_verified(acfactory):
     logging.info("ac1: create verified-group QR, ac2 scans and joins")
     chat = ac1.create_group("hello", protect=True)
     assert chat.get_basic_snapshot().is_protected
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     logging.info("ac2: start QR-code based join-group protocol")
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
@@ -464,12 +487,12 @@ def test_gossip_verification(acfactory) -> None:
     alice, bob, carol = acfactory.get_online_accounts(3)
 
     # Bob verifies Alice.
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     # Bob verifies Carol.
-    qr_code, _svg = carol.get_qr_code()
+    qr_code = carol.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -520,16 +543,16 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     ac3_chat = ac3.create_group("Verified group", protect=True)
 
     # ac1 joins ac3 group.
-    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac3_qr_code = ac3_chat.get_qr_code()
     ac1.secure_join(ac3_qr_code)
     ac1.wait_for_securejoin_joiner_success()
 
     # ac1 waits for member added message and creates a QR code.
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+    ac1_qr_code = snapshot.chat.get_qr_code()
 
     # ac2 verifies ac1
-    qr_code, _svg = ac1.get_qr_code()
+    qr_code = ac1.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -589,7 +612,7 @@ def test_withdraw_securejoin_qr(acfactory):
     assert alice_chat.get_basic_snapshot().is_protected
     logging.info("Bob joins verified group")
 
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob_chat = bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 

deltachat-rpc-client/tests/test_something.py

@@ -1,12 +1,15 @@
+import base64
 import concurrent.futures
 import json
 import logging
 import os
+import socket
 import subprocess
 import time
 from unittest.mock import MagicMock
 
 import pytest
+
 from deltachat_rpc_client import Contact, EventType, Message, events
 from deltachat_rpc_client.const import DownloadState, MessageState
 from deltachat_rpc_client.direct_imap import DirectImap
@@ -68,6 +71,18 @@ def test_configure_starttls(acfactory) -> None:
     assert account.is_configured()
 
 
+def test_configure_ip(acfactory) -> None:
+    account = acfactory.new_preconfigured_account()
+
+    domain = account.get_config("addr").rsplit("@")[-1]
+    ip_address = socket.gethostbyname(domain)
+
+    # This should fail TLS check.
+    account.set_config("mail_server", ip_address)
+    with pytest.raises(JsonRpcError):
+        account.configure()
+
+
 def test_account(acfactory) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
@@ -103,12 +118,12 @@ def test_account(acfactory) -> None:
     assert alice.get_chatlist(snapshot=True)
     assert alice.get_qr_code()
     assert alice.get_fresh_messages()
-    assert alice.get_next_messages()
 
     # Test sending empty message.
     assert len(bob.wait_next_messages()) == 0
     alice_chat_bob.send_text("")
     messages = bob.wait_next_messages()
+    assert bob.get_next_messages() == messages
     assert len(messages) == 1
     message = messages[0]
     snapshot = message.get_snapshot()
@@ -613,3 +628,10 @@ def test_markseen_contact_request(acfactory, tmp_path):
         if event.kind == EventType.MSGS_NOTICED:
             break
     assert message2.get_snapshot().state == MessageState.IN_SEEN
+
+
+def test_get_http_response(acfactory):
+    alice = acfactory.new_configured_account()
+    http_response = alice._rpc.get_http_response(alice.id, "https://example.org")
+    assert http_response["mimetype"] == "text/html"
+    assert b"<title>Example Domain</title>" in base64.b64decode((http_response["blob"] + "==").encode())

deltachat-rpc-client/tox.ini

@@ -28,5 +28,5 @@ commands =
 
 [pytest]
 timeout = 300
-#log_cli = true
+log_cli = true
 log_level = debug

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.140.0"
+version = "1.142.6"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -10,18 +10,18 @@ keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
 
 [dependencies]
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
-deltachat = { path = "..", default-features = false }
+deltachat-jsonrpc = { workspace = true }
+deltachat = { workspace = true }
 
-anyhow = "1"
-futures-lite = "2.3.0"
-log = "0.4"
-serde_json = "1"
-serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.37.0", features = ["io-std"] }
-tokio-util = "0.7.9"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+anyhow = { workspace = true }
+futures-lite = { workspace = true }
+log = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+tokio = { workspace = true, features = ["io-std"] }
+tokio-util = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
 
 [features]
 default = ["vendored"]

deltachat-rpc-server/npm-package/README.md

@@ -7,7 +7,7 @@ This simplifies cross-compilation and even reduces binary size (no CFFI layer an
 
 ## Usage
 
-> The **minimum** nodejs version for this package is `20.11`
+> The **minimum** nodejs version for this package is `16`
 
 ```
 npm i @deltachat/stdio-rpc-server @deltachat/jsonrpc-client

deltachat-rpc-server/npm-package/index.js

@@ -11,9 +11,6 @@ import {
   NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR,
 } from "./src/errors.js";
 
-// Because this is not compiled by typescript, esm needs this stuff (` with { type: "json" };`,
-// nodejs still complains about it being experimental, but deno also uses it, so treefit bets taht it will become standard)
-import package_json from "./package.json" with { type: "json" };
 import { createRequire } from "node:module";
 
 function findRPCServerInNodeModules() {
@@ -25,7 +22,12 @@ function findRPCServerInNodeModules() {
     return resolve(package_name);
   } catch (error) {
     console.debug("findRpcServerInNodeModules", error);
-    if (Object.keys(package_json.optionalDependencies).includes(package_name)) {
+    const require = createRequire(import.meta.url);
+    if (
+      Object.keys(require("./package.json").optionalDependencies).includes(
+        package_name
+      )
+    ) {
       throw new Error(NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name));
     } else {
       throw new Error(NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR());

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "1.140.0"
+  "version": "1.142.6"
 }

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -55,9 +55,10 @@ for (const { folder_name, package_name } of platform_package_names) {
 }
 
 if (is_local) {
-  package_json.peerDependencies["@deltachat/jsonrpc-client"] = 'file:../../deltachat-jsonrpc/typescript'
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] =
+    `file:${join(expected_cwd, "/../../deltachat-jsonrpc/typescript")}`;
 } else {
-  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*"
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*";
 }
 
 await fs.writeFile("./package.json", JSON.stringify(package_json, null, 4));

deny.toml

@@ -15,6 +15,10 @@ ignore = [
 
     # Unmaintained encoding
     "RUSTSEC-2021-0153",
+
+    # Problem in curve25519-dalek 3.2.0 used by iroh 0.4.
+    # curve25519-dalek 4.1.3 has the problem fixed.
+    "RUSTSEC-2024-0344",
 ]
 
 [bans]
@@ -50,9 +54,9 @@ skip = [
      { name = "fastrand", version = "1.9.0" },
      { name = "futures-lite", version = "1.13.0" },
      { name = "getrandom", version = "<0.2" },
+     { name = "h2", version = "0.3.26" },
      { name = "http-body", version = "0.4.6" },
      { name = "http", version = "0.2.12" },
-     { name = "hyper-rustls", version = "0.24.2" },
      { name = "hyper", version = "0.14.28" },
      { name = "idna", version = "0.4.0" },
      { name = "netlink-packet-core", version = "0.5.0" },
@@ -70,7 +74,6 @@ skip = [
      { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
-     { name = "reqwest", version = "0.11.27" },
      { name = "ring", version = "0.16.20" },
      { name = "rustls-pemfile", version = "1.0.4" },
      { name = "rustls", version = "0.21.11" },
@@ -82,6 +85,7 @@ skip = [
      { name = "spki", version = "0.6.0" },
      { name = "ssh-encoding", version = "0.1.0" },
      { name = "ssh-key", version = "0.5.1" },
+     { name = "strsim", version = "0.10.0" },
      { name = "sync_wrapper", version = "0.1.2" },
      { name = "synstructure", version = "0.12.6" },
      { name = "syn", version = "1.0.109" },

node/constants.js

@@ -128,6 +128,7 @@ module.exports = {
   DC_QR_ASK_VERIFYCONTACT: 200,
   DC_QR_ASK_VERIFYGROUP: 202,
   DC_QR_BACKUP: 251,
+  DC_QR_BACKUP2: 252,
   DC_QR_ERROR: 400,
   DC_QR_FPR_MISMATCH: 220,
   DC_QR_FPR_OK: 210,

node/lib/constants.ts

@@ -128,6 +128,7 @@ export enum C {
   DC_QR_ASK_VERIFYCONTACT = 200,
   DC_QR_ASK_VERIFYGROUP = 202,
   DC_QR_BACKUP = 251,
+  DC_QR_BACKUP2 = 252,
   DC_QR_ERROR = 400,
   DC_QR_FPR_MISMATCH = 220,
   DC_QR_FPR_OK = 210,

package.json

@@ -11,7 +11,7 @@
     "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
     "mocha": "^8.2.1",
-    "node-gyp": "^10.0.0",
+    "node-gyp": "~10.1.0",
     "prebuildify": "^5.0.1",
     "prebuildify-ci": "^1.0.5",
     "prettier": "^3.0.3",
@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.140.0"
+  "version": "1.142.6"
 }

python/examples/test_examples.py

@@ -44,11 +44,6 @@ def test_group_tracking_plugin(acfactory, lp):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
 
-    botproc.fnmatch_lines(
-        """
-        *ac_configure_completed*
-    """,
-    )
     ac1.add_account_plugin(FFIEventLogger(ac1))
     ac2.add_account_plugin(FFIEventLogger(ac2))
 

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.140.0"
+version = "1.142.6"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.7"

python/src/deltachat/testplugin.py

@@ -552,6 +552,15 @@ class ACFactory:
 
         bot_cfg = self.get_next_liveconfig()
         bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
+        self._acsetup.start_configure(bot_ac)
+        self.wait_configured(bot_ac)
+        bot_ac.start_io()
+        # Wait for DC_EVENT_IMAP_INBOX_IDLE so that all emails appeared in the bot's Inbox later are
+        # considered new and not existing ones, and thus processed by the bot.
+        print(bot_ac._logid, "waiting for inbox IDLE to become ready")
+        bot_ac._evtracker.wait_idle_inbox_ready()
+        bot_ac.stop_io()
+        self._acsetup._account2state[bot_ac] = self._acsetup.IDLEREADY
 
         # Forget ac as it will be opened by the bot subprocess
         # but keep something in the list to not confuse account generation

python/tests/test_0_complex_or_slow.py

@@ -1,4 +1,5 @@
 import sys
+import time
 
 import pytest
 import deltachat as dc
@@ -675,3 +676,17 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     assert msg_in.chat == chat2_offl
     assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
     assert ac2_offl_ac1_contact.is_verified()
+
+
+def test_deleted_msgs_dont_reappear(acfactory):
+    ac1 = acfactory.new_online_configuring_account()
+    acfactory.bring_accounts_online()
+    ac1.set_config("bcc_self", "1")
+    chat = ac1.get_self_contact().create_chat()
+    msg = chat.send_text("hello")
+    ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+    ac1.delete_messages([msg])
+    ac1._evtracker.get_matching("DC_EVENT_MSG_DELETED")
+    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+    time.sleep(5)
+    assert len(chat.get_messages()) == 0

python/tests/test_1_online.py

@@ -484,6 +484,16 @@ def test_move_works_on_self_sent(acfactory):
     ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
 
+def test_move_sync_msgs(acfactory):
+    ac1 = acfactory.new_online_configuring_account(bcc_self=True, sync_msgs=True, fix_is_chatmail=True)
+    acfactory.bring_accounts_online()
+
+    ac1.set_config("displayname", "Alice")
+    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+    ac1.set_config("displayname", "Bob")
+    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+
+
 def test_forward_messages(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     chat = ac1.create_chat(ac2)
@@ -1562,8 +1572,6 @@ def test_import_export_online_all(acfactory, tmp_path, data, lp):
 
         # check progress events for import
         assert imex_tracker.wait_progress(1, progress_upper_limit=249)
-        assert imex_tracker.wait_progress(500, progress_upper_limit=749)
-        assert imex_tracker.wait_progress(750, progress_upper_limit=999)
         assert imex_tracker.wait_progress(1000)
 
     assert_account_is_proper(ac1)

release-date.in

@@ -1 +1 @@
-2024-06-04
+2024-08-15

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.78.0
+RUST_VERSION=1.80.1
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/create-provider-data-rs.py

@@ -149,7 +149,7 @@ def process_data(data, file):
     oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None"
 
     provider = ""
-    before_login_hint = cleanstr(data.get("before_login_hint", ""))
+    before_login_hint = cleanstr(data.get("before_login_hint", "") or "")
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
     if (not has_imap and not has_smtp) or (has_imap and has_smtp):
         provider += (

scripts/make-rpc-testenv.sh

@@ -3,4 +3,4 @@ set -euo pipefail
 
 tox -c deltachat-rpc-client -e py --devenv venv
 venv/bin/pip install --upgrade pip
-cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+cargo install --locked --path deltachat-rpc-server/ --root "$PWD/venv" --debug

scripts/run-rpc-test.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
-cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
+cargo install --locked --path deltachat-rpc-server/ --root "$PWD/venv" --debug
 PATH="$PWD/venv/bin:$PATH" tox -c deltachat-rpc-client

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=2f3db24107e4802c2df0aa0a40f0e144006c0a9b
+REV=05c1b2029da74718e4bdc3799a46e29c4f794dc7
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.34.0
+Version: 0.35.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -22,7 +22,13 @@ to implement typical messenger functions.
 - [Locations](#locations)
     - [User locations](#user-locations)
     - [Points of interest](#points-of-interest)
+- [Stickers](#stickers)
+- [Voice messages](#voice-messages)
+- [Reactions](#reactions)
+- [Attaching a contact to a message](#attaching-a-contact-to-a-message)
+- [Transitioning to a new e-mail address (AEAP)](#transitioning-to-a-new-e-mail-address-aeap)
 - [Miscellaneous](#miscellaneous)
+    - [Sync messages](#sync-messages)
 
 
 # Encryption
@@ -461,6 +467,58 @@ As an extension to RFC 9078, it is allowed to send empty reaction message,
 in which case all previously sent reactions are retracted.
 
 
+# Attaching a contact to a message
+
+Messengers MAY allow the user to attach a contact to a message
+in order to share it with the chat partner.
+The contact MUST be sent as a [vCard](https://datatracker.ietf.org/doc/html/rfc6350).
+
+The vCard MUST contain `EMAIL`,
+`FN` (display name),
+and `VERSION` (which version of the vCard standard you're using).
+If available, it SHOULD contain
+`REV` (current timestamp),
+`PHOTO` (avatar), and
+`KEY` (OpenPGP public key,
+in binary format,
+encoded with vanilla base64;
+note that this is different from the OpenPGP 'ASCII Armor' format).
+
+Example vCard:
+
+```
+BEGIN:VCARD
+VERSION:4.0
+EMAIL:alice@example.org
+FN:Alice Wonderland
+KEY:data:application/pgp-keys;base64,[Base64-data]
+PHOTO:data:image/jpeg;base64,[image in Base64]
+REV:20240418T184242Z
+END:VCARD
+```
+
+It is fine if messengers do include a full vCard parser
+and e.g. simply search for the line starting with `EMAIL`
+in order to get the email address.
+
+
+# Transitioning to a new e-mail address (AEAP)
+
+When receiving a message:
+- If the key exists, but belongs to another address
+- AND there is a `Chat-Version` header
+- AND the message is signed correctly
+- AND the From address is (also) in the encrypted (and therefore signed) headers 
+- AND the message timestamp is newer than the contact's `lastseen`
+  (to prevent changing the address back when messages arrive out of order)
+  (this condition is not that important
+  since we will have eventual consistency even without it):
+
+  Replace the contact in _all_ groups,
+  possibly deduplicate the members list,
+  and add a system message to all of these chats.
+
+
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -484,21 +542,4 @@ We define the effective date of a message
 as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
-
-# Transitioning to a new e-mail address (AEAP)
-
-When receiving a message:
-- If the key exists, but belongs to another address
-- AND there is a `Chat-Version` header
-- AND the message is signed correctly
-- AND the From address is (also) in the encrypted (and therefore signed) headers 
-- AND the message timestamp is newer than the contact's `lastseen`
-  (to prevent changing the address back when messages arrive out of order)
-  (this condition is not that important
-  since we will have eventual consistency even without it):
-
-  Replace the contact in _all_ groups,
-  possibly deduplicate the members list,
-  and add a system message to all of these chats.
-
 Copyright © 2017-2021 Delta Chat contributors.

src/accounts.rs

@@ -166,6 +166,19 @@ impl Accounts {
             .remove(&id)
             .with_context(|| format!("no account with id {id}"))?;
         ctx.stop_io().await;
+
+        // Explicitly close the database
+        // to make sure the database file is closed
+        // and can be removed on Windows.
+        // If some spawned task tries to use the database afterwards,
+        // it will fail.
+        //
+        // Previously `stop_io()` aborted the tasks without awaiting them
+        // and this resulted in keeping `Context` clones inside
+        // `Future`s that were not dropped. This bug is fixed now,
+        // but explicitly closing the database ensures that file is freed
+        // even if not all `Context` references are dropped.
+        ctx.sql.close().await;
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id) {

src/blob.rs

@@ -12,7 +12,7 @@ use anyhow::{format_err, Context as _, Result};
 use base64::Engine as _;
 use futures::StreamExt;
 use image::codecs::jpeg::JpegEncoder;
-use image::io::Reader as ImageReader;
+use image::ImageReader;
 use image::{DynamicImage, GenericImage, GenericImageView, ImageFormat, Pixel, Rgba};
 use num_traits::FromPrimitive;
 use tokio::io::AsyncWriteExt;
@@ -1101,32 +1101,34 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_1() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "jpg",
-            true, // has Exif
-            1000,
-            1000,
-            0,
-            1000,
-            1000,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 1000,
+            original_height: 1000,
+            compressed_width: 1000,
+            compressed_height: 1000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
             bytes,
-            "jpg",
-            true, // has Exif
-            1000,
-            1000,
-            0,
-            1000,
-            1000,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 1000,
+            original_height: 1000,
+            compressed_width: 1000,
+            compressed_height: 1000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1135,18 +1137,20 @@ mod tests {
     async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        let img_rotated = SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "jpg",
-            true, // has Exif
-            2000,
-            1800,
-            270,
-            1800,
-            2000,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 2000,
+            original_height: 1800,
+            orientation: 270,
+            compressed_width: 1800,
+            compressed_height: 2000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
         assert_correct_rotation(&img_rotated);
@@ -1155,18 +1159,18 @@ mod tests {
         img_rotated.write_to(&mut buf, ImageFormat::Jpeg).unwrap();
         let bytes = buf.into_inner();
 
-        let img_rotated = send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
-            &bytes,
-            "jpg",
-            false, // no Exif
-            1800,
-            2000,
-            0,
-            1800,
-            2000,
-        )
+        let img_rotated = SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
+            bytes: &bytes,
+            extension: "jpg",
+            original_width: 1800,
+            original_height: 2000,
+            compressed_width: 1800,
+            compressed_height: 2000,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
         assert_correct_rotation(&img_rotated);
@@ -1176,64 +1180,80 @@ mod tests {
     async fn test_recode_image_balanced_png() {
         let bytes = include_bytes!("../test-data/image/screenshot.png");
 
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            1920,
-            1080,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
 
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            constants::WORSE_IMAGE_SIZE,
-            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: constants::WORSE_IMAGE_SIZE,
+            compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
 
-        send_image_check_mediaquality(
-            Viewtype::File,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::File,
+            media_quality_config: "1",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            1920,
-            1080,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            ..Default::default()
+        }
+        .test()
+        .await
+        .unwrap();
+
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::File,
+            media_quality_config: "1",
+            bytes,
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            set_draft: true,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
 
         // This will be sent as Image, see [`BlobObject::maybe_sticker`] for explanation.
-        send_image_check_mediaquality(
-            Viewtype::Sticker,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Sticker,
+            media_quality_config: "0",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            1920,
-            1080,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: 1920,
+            compressed_height: 1080,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1244,18 +1264,18 @@ mod tests {
     async fn test_recode_image_rgba_png_to_jpeg() {
         let bytes = include_bytes!("../test-data/image/screenshot-rgba.png");
 
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "1",
             bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            constants::WORSE_IMAGE_SIZE,
-            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-        )
+            extension: "png",
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: constants::WORSE_IMAGE_SIZE,
+            compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1263,18 +1283,19 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_huge_jpg() {
         let bytes = include_bytes!("../test-data/image/screenshot.jpg");
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("0"),
+        SendImageCheckMediaquality {
+            viewtype: Viewtype::Image,
+            media_quality_config: "0",
             bytes,
-            "jpg",
-            true, // has Exif
-            1920,
-            1080,
-            0,
-            constants::BALANCED_IMAGE_SIZE,
-            constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
-        )
+            extension: "jpg",
+            has_exif: true,
+            original_width: 1920,
+            original_height: 1080,
+            compressed_width: constants::BALANCED_IMAGE_SIZE,
+            compressed_height: constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
+            ..Default::default()
+        }
+        .test()
         .await
         .unwrap();
     }
@@ -1296,71 +1317,93 @@ mod tests {
         assert_eq!(luma, 0);
     }
 
-    #[allow(clippy::too_many_arguments)]
-    async fn send_image_check_mediaquality(
-        viewtype: Viewtype,
-        media_quality_config: Option<&str>,
-        bytes: &[u8],
-        extension: &str,
-        has_exif: bool,
-        original_width: u32,
-        original_height: u32,
-        orientation: i32,
-        compressed_width: u32,
-        compressed_height: u32,
-    ) -> anyhow::Result<DynamicImage> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-        alice
-            .set_config(Config::MediaQuality, media_quality_config)
-            .await?;
-        let file = alice.get_blobdir().join("file").with_extension(extension);
+    #[derive(Default)]
+    struct SendImageCheckMediaquality<'a> {
+        pub(crate) viewtype: Viewtype,
+        pub(crate) media_quality_config: &'a str,
+        pub(crate) bytes: &'a [u8],
+        pub(crate) extension: &'a str,
+        pub(crate) has_exif: bool,
+        pub(crate) original_width: u32,
+        pub(crate) original_height: u32,
+        pub(crate) orientation: i32,
+        pub(crate) compressed_width: u32,
+        pub(crate) compressed_height: u32,
+        pub(crate) set_draft: bool,
+    }
 
-        fs::write(&file, &bytes)
-            .await
-            .context("failed to write file")?;
-        check_image_size(&file, original_width, original_height);
+    impl SendImageCheckMediaquality<'_> {
+        pub(crate) async fn test(self) -> anyhow::Result<DynamicImage> {
+            let viewtype = self.viewtype;
+            let media_quality_config = self.media_quality_config;
+            let bytes = self.bytes;
+            let extension = self.extension;
+            let has_exif = self.has_exif;
+            let original_width = self.original_width;
+            let original_height = self.original_height;
+            let orientation = self.orientation;
+            let compressed_width = self.compressed_width;
+            let compressed_height = self.compressed_height;
+            let set_draft = self.set_draft;
 
-        let (_, exif) = image_metadata(&std::fs::File::open(&file)?)?;
-        if has_exif {
-            let exif = exif.unwrap();
-            assert_eq!(exif_orientation(&exif, &alice), orientation);
-        } else {
+            let alice = TestContext::new_alice().await;
+            let bob = TestContext::new_bob().await;
+            alice
+                .set_config(Config::MediaQuality, Some(media_quality_config))
+                .await?;
+            let file = alice.get_blobdir().join("file").with_extension(extension);
+
+            fs::write(&file, &bytes)
+                .await
+                .context("failed to write file")?;
+            check_image_size(&file, original_width, original_height);
+
+            let (_, exif) = image_metadata(&std::fs::File::open(&file)?)?;
+            if has_exif {
+                let exif = exif.unwrap();
+                assert_eq!(exif_orientation(&exif, &alice), orientation);
+            } else {
+                assert!(exif.is_none());
+            }
+
+            let mut msg = Message::new(viewtype);
+            msg.set_file(file.to_str().unwrap(), None);
+            let chat = alice.create_chat(&bob).await;
+            if set_draft {
+                chat.id.set_draft(&alice, Some(&mut msg)).await.unwrap();
+                msg = chat.id.get_draft(&alice).await.unwrap().unwrap();
+                assert_eq!(msg.get_viewtype(), Viewtype::File);
+            }
+            let sent = alice.send_msg(chat.id, &mut msg).await;
+            let alice_msg = alice.get_last_msg().await;
+            assert_eq!(alice_msg.get_width() as u32, compressed_width);
+            assert_eq!(alice_msg.get_height() as u32, compressed_height);
+            let file_saved = alice
+                .get_blobdir()
+                .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
+            alice_msg.save_file(&alice, &file_saved).await?;
+            check_image_size(file_saved, compressed_width, compressed_height);
+
+            let bob_msg = bob.recv_msg(&sent).await;
+            assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
+            assert_eq!(bob_msg.get_width() as u32, compressed_width);
+            assert_eq!(bob_msg.get_height() as u32, compressed_height);
+            let file_saved = bob
+                .get_blobdir()
+                .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
+            bob_msg.save_file(&bob, &file_saved).await?;
+            if viewtype == Viewtype::File {
+                assert_eq!(file_saved.extension().unwrap(), extension);
+                let bytes1 = fs::read(&file_saved).await?;
+                assert_eq!(&bytes1, bytes);
+            }
+
+            let (_, exif) = image_metadata(&std::fs::File::open(&file_saved)?)?;
             assert!(exif.is_none());
+
+            let img = check_image_size(file_saved, compressed_width, compressed_height);
+            Ok(img)
         }
-
-        let mut msg = Message::new(viewtype);
-        msg.set_file(file.to_str().unwrap(), None);
-        let chat = alice.create_chat(&bob).await;
-        let sent = alice.send_msg(chat.id, &mut msg).await;
-        let alice_msg = alice.get_last_msg().await;
-        assert_eq!(alice_msg.get_width() as u32, compressed_width);
-        assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let file_saved = alice
-            .get_blobdir()
-            .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
-        alice_msg.save_file(&alice, &file_saved).await?;
-        check_image_size(file_saved, compressed_width, compressed_height);
-
-        let bob_msg = bob.recv_msg(&sent).await;
-        assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
-        assert_eq!(bob_msg.get_width() as u32, compressed_width);
-        assert_eq!(bob_msg.get_height() as u32, compressed_height);
-        let file_saved = bob
-            .get_blobdir()
-            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
-        bob_msg.save_file(&bob, &file_saved).await?;
-        if viewtype == Viewtype::File {
-            assert_eq!(file_saved.extension().unwrap(), extension);
-            let bytes1 = fs::read(&file_saved).await?;
-            assert_eq!(&bytes1, bytes);
-        }
-
-        let (_, exif) = image_metadata(&std::fs::File::open(&file_saved)?)?;
-        assert!(exif.is_none());
-
-        let img = check_image_size(file_saved, compressed_width, compressed_height);
-        Ok(img)
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

src/chat.rs

@@ -8,7 +8,7 @@ use std::str::FromStr;
 use std::time::Duration;
 
 use anyhow::{anyhow, bail, ensure, Context as _, Result};
-use deltachat_contact_tools::{strip_rtlo_characters, ContactAddress};
+use deltachat_contact_tools::{sanitize_bidi_characters, sanitize_single_line, ContactAddress};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
 use strum_macros::EnumIter;
@@ -46,10 +46,10 @@ use crate::stock_str;
 use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     buf_compress, create_id, create_outgoing_rfc724_mid, create_smeared_timestamp,
-    create_smeared_timestamps, get_abs_path, gm2local_offset, improve_single_line_input,
-    smeared_time, time, IsNoneOrEmpty, SystemTime,
+    create_smeared_timestamps, get_abs_path, gm2local_offset, smeared_time, time, IsNoneOrEmpty,
+    SystemTime,
 };
-use crate::webxdc::WEBXDC_SUFFIX;
+use crate::webxdc::StatusUpdateSerial;
 
 /// An chat item, such as a message or a marker.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -322,7 +322,7 @@ impl ChatId {
         param: Option<String>,
         timestamp: i64,
     ) -> Result<Self> {
-        let grpname = strip_rtlo_characters(grpname);
+        let grpname = sanitize_single_line(grpname);
         let timestamp = cmp::min(timestamp, smeared_time(context));
         let row_id =
             context.sql.insert(
@@ -894,8 +894,20 @@ impl ChatId {
                     .await?
                     .context("no file stored in params")?;
                 msg.param.set(Param::File, blob.as_name());
-                if blob.suffix() == Some(WEBXDC_SUFFIX) {
-                    msg.viewtype = Viewtype::Webxdc;
+                if msg.viewtype == Viewtype::File {
+                    if let Some((better_type, _)) =
+                        message::guess_msgtype_from_suffix(&blob.to_abs_path())
+                            // We do not do an automatic conversion to other viewtypes here so that
+                            // users can send images as "files" to preserve the original quality
+                            // (usually we compress images). The remaining conversions are done by
+                            // `prepare_msg_blob()` later.
+                            .filter(|&(vt, _)| vt == Viewtype::Webxdc || vt == Viewtype::Vcard)
+                    {
+                        msg.viewtype = better_type;
+                    }
+                }
+                if msg.viewtype == Viewtype::Vcard {
+                    msg.try_set_vcard(context, &blob.to_abs_path()).await?;
                 }
             }
         }
@@ -916,12 +928,13 @@ impl ChatId {
                         .sql
                         .execute(
                             "UPDATE msgs
-                            SET timestamp=?,type=?,txt=?, param=?,mime_in_reply_to=?
+                            SET timestamp=?,type=?,txt=?,txt_normalized=?,param=?,mime_in_reply_to=?
                             WHERE id=?;",
                             (
                                 time(),
                                 msg.viewtype,
                                 &msg.text,
+                                message::normalize_text(&msg.text),
                                 msg.param.to_string(),
                                 msg.in_reply_to.as_deref().unwrap_or_default(),
                                 msg.id,
@@ -945,10 +958,11 @@ impl ChatId {
                  type,
                  state,
                  txt,
+                 txt_normalized,
                  param,
                  hidden,
                  mime_in_reply_to)
-         VALUES (?,?,?, ?,?,?,?,?,?);",
+         VALUES (?,?,?,?,?,?,?,?,?,?);",
                 (
                     self,
                     ContactId::SELF,
@@ -956,6 +970,7 @@ impl ChatId {
                     msg.viewtype,
                     MessageState::OutDraft,
                     &msg.text,
+                    message::normalize_text(&msg.text),
                     msg.param.to_string(),
                     1,
                     msg.in_reply_to.as_deref().unwrap_or_default(),
@@ -1920,7 +1935,7 @@ impl Chat {
             self.param.remove(Param::Unpromoted);
             self.update_param(context).await?;
             // send_sync_msg() is called (usually) a moment later at send_msg_to_smtp()
-            // when the group-creation message is actually sent though SMTP -
+            // when the group creation message is actually sent through SMTP --
             // this makes sure, the other devices are aware of grpid that is used in the sync-message.
             context
                 .sync_qr_code_tokens(Some(self.id))
@@ -2064,7 +2079,7 @@ impl Chat {
                 .execute(
                     "UPDATE msgs
                      SET rfc724_mid=?, chat_id=?, from_id=?, to_id=?, timestamp=?, type=?,
-                         state=?, txt=?, subject=?, param=?,
+                         state=?, txt=?, txt_normalized=?, subject=?, param=?,
                          hidden=?, mime_in_reply_to=?, mime_references=?, mime_modified=?,
                          mime_headers=?, mime_compressed=1, location_id=?, ephemeral_timer=?,
                          ephemeral_timestamp=?
@@ -2078,6 +2093,7 @@ impl Chat {
                         msg.viewtype,
                         msg.state,
                         msg.text,
+                        message::normalize_text(&msg.text),
                         &msg.subject,
                         msg.param.to_string(),
                         msg.hidden,
@@ -2106,6 +2122,7 @@ impl Chat {
                         type,
                         state,
                         txt,
+                        txt_normalized,
                         subject,
                         param,
                         hidden,
@@ -2117,7 +2134,7 @@ impl Chat {
                         location_id,
                         ephemeral_timer,
                         ephemeral_timestamp)
-                        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?,?);",
+                        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1,?,?,?);",
                     params_slice![
                         msg.rfc724_mid,
                         msg.chat_id,
@@ -2127,6 +2144,7 @@ impl Chat {
                         msg.viewtype,
                         msg.state,
                         msg.text,
+                        message::normalize_text(&msg.text),
                         &msg.subject,
                         msg.param.to_string(),
                         msg.hidden,
@@ -2222,7 +2240,7 @@ pub(crate) async fn sync(context: &Context, id: SyncId, action: SyncAction) -> R
     context
         .add_sync_item(SyncData::AlterChat { id, action })
         .await?;
-    context.send_sync_msg().await?;
+    context.scheduler.interrupt_smtp().await;
     Ok(())
 }
 
@@ -2649,6 +2667,10 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
                 .await?;
         }
 
+        if msg.viewtype == Viewtype::Vcard {
+            msg.try_set_vcard(context, &blob.to_abs_path()).await?;
+        }
+
         let mut maybe_sticker = msg.viewtype == Viewtype::Sticker;
         if !send_as_is
             && (msg.viewtype == Viewtype::Image
@@ -2837,7 +2859,7 @@ pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message
 async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
     // protect all system messages against RTLO attacks
     if msg.is_system_message() {
-        msg.text = strip_rtlo_characters(&msg.text);
+        msg.text = sanitize_bidi_characters(&msg.text);
     }
 
     if !prepare_send_msg(context, chat_id, msg).await?.is_empty() {
@@ -2887,7 +2909,7 @@ async fn prepare_send_msg(
 /// The caller has to interrupt SMTP loop or otherwise process new rows.
 pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result<Vec<i64>> {
     let needs_encryption = msg.param.get_bool(Param::GuaranteeE2ee).unwrap_or_default();
-    let mimefactory = MimeFactory::from_msg(context, msg).await?;
+    let mimefactory = MimeFactory::from_msg(context, msg.clone()).await?;
     let attach_selfavatar = mimefactory.attach_selfavatar;
     let mut recipients = mimefactory.recipients();
 
@@ -3246,35 +3268,25 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()>
             context.emit_event(EventType::MsgsNoticed(chat_id_in_archive));
             chatlist_events::emit_chatlist_item_changed(context, chat_id_in_archive);
         }
-        chatlist_events::emit_chatlist_item_changed(context, DC_CHAT_ID_ARCHIVED_LINK);
-    } else {
-        let exists = context
-            .sql
-            .exists(
-                "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;",
-                (MessageState::InFresh, chat_id),
-            )
-            .await?;
-        if !exists {
-            return Ok(());
-        }
-
-        context
-            .sql
-            .execute(
-                "UPDATE msgs
-                SET state=?
-              WHERE state=?
-                AND hidden=0
-                AND chat_id=?;",
-                (MessageState::InNoticed, MessageState::InFresh, chat_id),
-            )
-            .await?;
+    } else if context
+        .sql
+        .execute(
+            "UPDATE msgs
+            SET state=?
+          WHERE state=?
+            AND hidden=0
+            AND chat_id=?;",
+            (MessageState::InNoticed, MessageState::InFresh, chat_id),
+        )
+        .await?
+        == 0
+    {
+        return Ok(());
     }
 
     context.emit_event(EventType::MsgsNoticed(chat_id));
     chatlist_events::emit_chatlist_item_changed(context, chat_id);
-
+    context.on_archived_chats_maybe_noticed();
     Ok(())
 }
 
@@ -3337,6 +3349,7 @@ pub(crate) async fn mark_old_messages_as_noticed(
             context,
             "Marking chats as noticed because there are newer outgoing messages: {changed_chats:?}."
         );
+        context.on_archived_chats_maybe_noticed();
     }
 
     for c in changed_chats {
@@ -3482,7 +3495,7 @@ pub async fn create_group_chat(
     protect: ProtectionStatus,
     chat_name: &str,
 ) -> Result<ChatId> {
-    let chat_name = improve_single_line_input(chat_name);
+    let chat_name = sanitize_single_line(chat_name);
     ensure!(!chat_name.is_empty(), "Invalid chat name");
 
     let grpid = create_id();
@@ -3716,12 +3729,14 @@ pub(crate) async fn add_contact_to_chat_ex(
     if from_handshake && chat.param.get_int(Param::Unpromoted).unwrap_or_default() == 1 {
         chat.param.remove(Param::Unpromoted);
         chat.update_param(context).await?;
-        let _ = context
+        if context
             .sync_qr_code_tokens(Some(chat_id))
             .await
             .log_err(context)
             .is_ok()
-            && context.send_sync_msg().await.log_err(context).is_ok();
+        {
+            context.scheduler.interrupt_smtp().await;
+        }
     }
 
     if context.is_self_addr(contact.get_addr()).await? {
@@ -3760,7 +3775,10 @@ pub(crate) async fn add_contact_to_chat_ex(
         msg.param.set_cmd(SystemMessage::MemberAddedToGroup);
         msg.param.set(Param::Arg, contact_addr);
         msg.param.set_int(Param::Arg2, from_handshake.into());
-        msg.id = send_msg(context, chat_id, &mut msg).await?;
+        if let Err(e) = send_msg(context, chat_id, &mut msg).await {
+            remove_from_chat_contacts_table(context, chat_id, contact_id).await?;
+            return Err(e);
+        }
         sync = Nosync;
     }
     context.emit_event(EventType::ChatModified(chat_id));
@@ -3916,8 +3934,7 @@ pub async fn remove_contact_from_chat(
             if let Some(contact) = Contact::get_by_id_optional(context, contact_id).await? {
                 if chat.typ == Chattype::Group && chat.is_promoted() {
                     msg.viewtype = Viewtype::Text;
-                    if contact.id == ContactId::SELF {
-                        set_group_explicitly_left(context, &chat.grpid).await?;
+                    if contact_id == ContactId::SELF {
                         msg.text = stock_str::msg_group_left_local(context, ContactId::SELF).await;
                     } else {
                         msg.text = stock_str::msg_del_member_local(
@@ -3929,17 +3946,24 @@ pub async fn remove_contact_from_chat(
                     }
                     msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup);
                     msg.param.set(Param::Arg, contact.get_addr().to_lowercase());
-                    msg.id = send_msg(context, chat_id, &mut msg).await?;
+                    let res = send_msg(context, chat_id, &mut msg).await;
+                    if contact_id == ContactId::SELF {
+                        res?;
+                        set_group_explicitly_left(context, &chat.grpid).await?;
+                    } else if let Err(e) = res {
+                        warn!(context, "remove_contact_from_chat({chat_id}, {contact_id}): send_msg() failed: {e:#}.");
+                    }
                 } else {
                     sync = Sync;
                 }
             }
             // we remove the member from the chat after constructing the
             // to-be-send message. If between send_msg() and here the
-            // process dies the user will have to re-do the action.  It's
-            // better than the other way round: you removed
-            // someone from DB but no peer or device gets to know about it and
-            // group membership is thus different on different devices.
+            // process dies, the user will be able to redo the action. It's better than the other
+            // way round: you removed someone from DB but no peer or device gets to know about it
+            // and group membership is thus different on different devices. But if send_msg()
+            // failed, we still remove the member locally, otherwise it would be impossible to
+            // remove a member with missing key from a protected group.
             // Note also that sending a message needs all recipients
             // in order to correctly determine encryption so if we
             // removed it first, it would complicate the
@@ -3987,7 +4011,7 @@ async fn rename_ex(
     chat_id: ChatId,
     new_name: &str,
 ) -> Result<()> {
-    let new_name = improve_single_line_input(new_name);
+    let new_name = sanitize_single_line(new_name);
     /* the function only sets the names of group chats; normal chats get their names from the contacts */
     let mut success = false;
 
@@ -4018,7 +4042,7 @@ async fn rename_ex(
             if chat.is_promoted()
                 && !chat.is_mailing_list()
                 && chat.typ != Chattype::Broadcast
-                && improve_single_line_input(&chat.name) != new_name
+                && sanitize_single_line(&chat.name) != new_name
             {
                 msg.viewtype = Viewtype::Text;
                 msg.text =
@@ -4242,9 +4266,39 @@ pub async fn resend_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
         msg.timestamp_sort = create_smeared_timestamp(context);
         // note(treefit): only matters if it is the last message in chat (but probably to expensive to check, debounce also solves it)
         chatlist_events::emit_chatlist_item_changed(context, msg.chat_id);
-        if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
-            context.scheduler.interrupt_smtp().await;
+        if create_send_msg_jobs(context, &mut msg).await?.is_empty() {
+            continue;
         }
+        if msg.viewtype == Viewtype::Webxdc {
+            let conn_fn = |conn: &mut rusqlite::Connection| {
+                let range = conn.query_row(
+                    "SELECT IFNULL(min(id), 1), IFNULL(max(id), 0) \
+                     FROM msgs_status_updates WHERE msg_id=?",
+                    (msg.id,),
+                    |row| {
+                        let min_id: StatusUpdateSerial = row.get(0)?;
+                        let max_id: StatusUpdateSerial = row.get(1)?;
+                        Ok((min_id, max_id))
+                    },
+                )?;
+                if range.0 > range.1 {
+                    return Ok(());
+                };
+                // `first_serial` must be decreased, otherwise if `Context::flush_status_updates()`
+                // runs in parallel, it would miss the race and instead of resending just remove the
+                // updates thinking that they have been already sent.
+                conn.execute(
+                    "INSERT INTO smtp_status_updates (msg_id, first_serial, last_serial, descr) \
+                     VALUES(?, ?, ?, '') \
+                     ON CONFLICT(msg_id) \
+                     DO UPDATE SET first_serial=min(first_serial - 1, excluded.first_serial)",
+                    (msg.id, range.0, range.1),
+                )?;
+                Ok(())
+            };
+            context.sql.call_write(conn_fn).await?;
+        }
+        context.scheduler.interrupt_smtp().await;
     }
     Ok(())
 }
@@ -4346,9 +4400,10 @@ pub async fn add_device_msg_with_importance(
             timestamp_rcvd,
             type,state,
             txt,
+            txt_normalized,
             param,
             rfc724_mid)
-            VALUES (?,?,?,?,?,?,?,?,?,?,?);",
+            VALUES (?,?,?,?,?,?,?,?,?,?,?,?);",
                 (
                     chat_id,
                     ContactId::DEVICE,
@@ -4359,6 +4414,7 @@ pub async fn add_device_msg_with_importance(
                     msg.viewtype,
                     state,
                     &msg.text,
+                    message::normalize_text(&msg.text),
                     msg.param.to_string(),
                     rfc724_mid,
                 ),
@@ -4462,8 +4518,8 @@ pub(crate) async fn add_info_msg_with_cmd(
 
     let row_id =
     context.sql.insert(
-        "INSERT INTO msgs (chat_id,from_id,to_id,timestamp,timestamp_sent,timestamp_rcvd,type,state,txt,rfc724_mid,ephemeral_timer, param,mime_in_reply_to)
-        VALUES (?,?,?, ?,?,?,?,?, ?,?,?, ?,?);",
+        "INSERT INTO msgs (chat_id,from_id,to_id,timestamp,timestamp_sent,timestamp_rcvd,type,state,txt,txt_normalized,rfc724_mid,ephemeral_timer,param,mime_in_reply_to)
+        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
         (
             chat_id,
             from_id.unwrap_or(ContactId::INFO),
@@ -4474,6 +4530,7 @@ pub(crate) async fn add_info_msg_with_cmd(
             Viewtype::Text,
             MessageState::InNoticed,
             text,
+            message::normalize_text(text),
             rfc724_mid,
             ephemeral_timer,
             param.to_string(),
@@ -4518,8 +4575,8 @@ pub(crate) async fn update_msg_text_and_timestamp(
     context
         .sql
         .execute(
-            "UPDATE msgs SET txt=?, timestamp=? WHERE id=?;",
-            (text, timestamp, msg_id),
+            "UPDATE msgs SET txt=?, txt_normalized=?, timestamp=? WHERE id=?;",
+            (text, message::normalize_text(text), timestamp, msg_id),
         )
         .await?;
     context.emit_msgs_changed(chat_id, msg_id);
@@ -4611,7 +4668,7 @@ impl Context {
                     .0
             }
             SyncId::Msgids(msgids) => {
-                let msg = message::get_latest_by_rfc724_mids(self, msgids)
+                let msg = message::get_by_rfc724_mids(self, msgids)
                     .await?
                     .with_context(|| format!("No message found for Message-IDs {msgids:?}"))?;
                 ChatId::lookup_by_message(&msg)
@@ -4631,6 +4688,14 @@ impl Context {
             SyncAction::SetContacts(addrs) => set_contacts_by_addrs(self, chat_id, addrs).await,
         }
     }
+
+    /// Emits the appropriate `MsgsChanged` event. Should be called if the number of unnoticed
+    /// archived chats could decrease. In general we don't want to make an extra db query to know if
+    /// a noticied chat is archived. Emitting events should be cheap, a false-positive `MsgsChanged`
+    /// is ok.
+    pub(crate) fn on_archived_chats_maybe_noticed(&self) {
+        self.emit_msgs_changed(DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0));
+    }
 }
 
 #[cfg(test)]
@@ -5023,6 +5088,7 @@ mod tests {
 
         // Bob leaves the chat.
         remove_contact_from_chat(&bob, bob_chat_id, ContactId::SELF).await?;
+        bob.pop_sent_msg().await;
 
         // Bob receives a msg about Alice adding Claire to the group.
         bob.recv_msg(&alice_sent_add_msg).await;
@@ -5075,6 +5141,7 @@ mod tests {
         let sent_msg = alice.pop_sent_msg().await;
         bob.recv_msg(&sent_msg).await;
         remove_contact_from_chat(&bob, bob_chat_id, bob_fiona_contact_id).await?;
+        bob.pop_sent_msg().await;
 
         // This doesn't add Fiona back because Bob just removed them.
         let sent_msg = alice.send_text(alice_chat_id, "Welcome, Fiona!").await;
@@ -5800,7 +5867,27 @@ mod tests {
         assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2);
 
         // mark one of the archived+muted chats as noticed: check that the archive-link counter is changed as well
+        t.evtracker.clear_events();
         marknoticed_chat(&t, claire_chat_id).await?;
+        let ev = t
+            .evtracker
+            .get_matching(|ev| {
+                matches!(
+                    ev,
+                    EventType::MsgsChanged {
+                        chat_id: DC_CHAT_ID_ARCHIVED_LINK,
+                        ..
+                    }
+                )
+            })
+            .await;
+        assert_eq!(
+            ev,
+            EventType::MsgsChanged {
+                chat_id: DC_CHAT_ID_ARCHIVED_LINK,
+                msg_id: MsgId::new(0),
+            }
+        );
         assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2);
         assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0);
         assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1);

src/chatlist.rs

@@ -82,11 +82,13 @@ impl Chatlist {
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
     /// - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
     ///   is added as needed.
+    ///
     /// `query`: An optional query for filtering the list. Only chats matching this query
-    ///     are returned. When `is:unread` is contained in the query, the chatlist is
-    ///     filtered such that only chats with unread messages show up.
+    /// are returned. When `is:unread` is contained in the query, the chatlist is
+    /// filtered such that only chats with unread messages show up.
+    ///
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
-    ///     are returned.
+    /// are returned.
     pub async fn try_load(
         context: &Context,
         listflags: usize,

src/config.rs

@@ -6,21 +6,21 @@ use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
 use base64::Engine as _;
-use deltachat_contact_tools::addr_cmp;
+use deltachat_contact_tools::{addr_cmp, sanitize_single_line};
 use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumString};
 use tokio::fs;
 
 use crate::blob::BlobObject;
-use crate::constants::{self, DC_VERSION_STR};
+use crate::constants;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 use crate::sync::{self, Sync::*, SyncData};
-use crate::tools::{get_abs_path, improve_single_line_input};
+use crate::tools::get_abs_path;
 
 /// The available configuration keys.
 #[derive(
@@ -59,7 +59,10 @@ pub enum Config {
     /// IMAP server security (e.g. TLS, STARTTLS).
     MailSecurity,
 
-    /// How to check IMAP server TLS certificates.
+    /// How to check TLS certificates.
+    ///
+    /// "IMAP" in the name is for compatibility,
+    /// this actually applies to both IMAP and SMTP connections.
     ImapCertificateChecks,
 
     /// SMTP server hostname.
@@ -77,7 +80,9 @@ pub enum Config {
     /// SMTP server security (e.g. TLS, STARTTLS).
     SendSecurity,
 
-    /// How to check SMTP server TLS certificates.
+    /// Deprecated option for backwards compatibilty.
+    ///
+    /// Certificate checks for SMTP are actually controlled by `imap_certificate_checks` config.
     SmtpCertificateChecks,
 
     /// Whether to use OAuth 2.
@@ -124,14 +129,14 @@ pub enum Config {
 
     /// True if Message Delivery Notifications (read receipts) should
     /// be sent and requested.
-    #[strum(props(default = "1"))]
     MdnsEnabled,
 
     /// True if "Sent" folder should be watched for changes.
     #[strum(props(default = "0"))]
     SentboxWatch,
 
-    /// True if chat messages should be moved to a separate folder.
+    /// True if chat messages should be moved to a separate folder. Auto-sent messages like sync
+    /// ones are moved there anyway.
     #[strum(props(default = "1"))]
     MvboxMove,
 
@@ -209,7 +214,12 @@ pub enum Config {
     /// Configured IMAP server security (e.g. TLS, STARTTLS).
     ConfiguredMailSecurity,
 
-    /// How to check IMAP server TLS certificates.
+    /// Configured TLS certificate checks.
+    /// This option is saved on successful configuration
+    /// and should not be modified manually.
+    ///
+    /// This actually applies to both IMAP and SMTP connections,
+    /// but has "IMAP" in the name for backwards compatibility.
     ConfiguredImapCertificateChecks,
 
     /// Configured SMTP server hostname.
@@ -224,7 +234,9 @@ pub enum Config {
     /// Configured SMTP server port.
     ConfiguredSendPort,
 
-    /// How to check SMTP server TLS certificates.
+    /// Deprecated, stored for backwards compatibility.
+    ///
+    /// ConfiguredImapCertificateChecks is actually used.
     ConfiguredSmtpCertificateChecks,
 
     /// Whether OAuth 2 is used with configured provider.
@@ -257,6 +269,12 @@ pub enum Config {
     /// True if account is a chatmail account.
     IsChatmail,
 
+    /// True if `IsChatmail` mustn't be autoconfigured. For tests.
+    FixIsChatmail,
+
+    /// True if account is muted.
+    IsMuted,
+
     /// All secondary self addresses separated by spaces
     /// (`addr1@example.org addr2@example.org addr3@example.org`)
     SecondaryAddrs,
@@ -314,7 +332,8 @@ pub enum Config {
     #[strum(props(default = "0"))]
     DownloadLimit,
 
-    /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set.
+    /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set
+    /// and `Bot` unset.
     #[strum(props(default = "1"))]
     SyncMsgs,
 
@@ -378,14 +397,11 @@ impl Config {
     /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
     /// mustn't be controlled by other devices.
     pub(crate) fn is_synced(&self) -> bool {
-        // We don't restart IO from the synchronisation code, so this is to be on the safe side.
-        if self.needs_io_restart() {
-            return false;
-        }
         matches!(
             self,
             Self::Displayname
                 | Self::MdnsEnabled
+                | Self::MvboxMove
                 | Self::ShowEmails
                 | Self::Selfavatar
                 | Self::Selfstatus,
@@ -394,10 +410,7 @@ impl Config {
 
     /// Whether the config option needs an IO scheduler restart to take effect.
     pub(crate) fn needs_io_restart(&self) -> bool {
-        matches!(
-            self,
-            Config::MvboxMove | Config::OnlyFetchMvbox | Config::SentboxWatch
-        )
+        matches!(self, Config::OnlyFetchMvbox | Config::SentboxWatch)
     }
 }
 
@@ -423,7 +436,7 @@ impl Context {
                         .into_owned()
                 })
             }
-            Config::SysVersion => Some((*DC_VERSION_STR).clone()),
+            Config::SysVersion => Some((*constants::DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{RECOMMENDED_FILE_SIZE}")),
             Config::SysConfigKeys => Some(get_config_keys_string()),
             _ => self.sql.get_raw_config(key.as_ref()).await?,
@@ -481,7 +494,8 @@ impl Context {
     /// Returns true if movebox ("DeltaChat" folder) should be watched.
     pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
         Ok(self.get_config_bool(Config::MvboxMove).await?
-            || self.get_config_bool(Config::OnlyFetchMvbox).await?)
+            || self.get_config_bool(Config::OnlyFetchMvbox).await?
+            || !self.get_config_bool(Config::IsChatmail).await?)
     }
 
     /// Returns true if sentbox ("Sent" folder) should be watched.
@@ -493,6 +507,29 @@ impl Context {
                 .is_some())
     }
 
+    /// Returns true if sync messages should be sent.
+    pub(crate) async fn should_send_sync_msgs(&self) -> Result<bool> {
+        Ok(self.get_config_bool(Config::SyncMsgs).await?
+            && self.get_config_bool(Config::BccSelf).await?
+            && !self.get_config_bool(Config::Bot).await?)
+    }
+
+    /// Returns whether MDNs should be requested.
+    pub(crate) async fn should_request_mdns(&self) -> Result<bool> {
+        match self.get_config_bool_opt(Config::MdnsEnabled).await? {
+            Some(val) => Ok(val),
+            None => Ok(!self.get_config_bool(Config::Bot).await?),
+        }
+    }
+
+    /// Returns whether MDNs should be sent.
+    pub(crate) async fn should_send_mdns(&self) -> Result<bool> {
+        Ok(self
+            .get_config_bool_opt(Config::MdnsEnabled)
+            .await?
+            .unwrap_or(true))
+    }
+
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
@@ -600,7 +637,7 @@ impl Context {
         mut value: Option<&str>,
     ) -> Result<()> {
         Self::check_config(key, value)?;
-        let sync = sync == Sync && key.is_synced();
+        let sync = sync == Sync && key.is_synced() && self.is_configured().await?;
         let better_value;
 
         match key {
@@ -639,7 +676,7 @@ impl Context {
             }
             Config::Displayname => {
                 if let Some(v) = value {
-                    better_value = improve_single_line_input(v);
+                    better_value = sanitize_single_line(v);
                     value = Some(&better_value);
                 }
                 self.sql.set_raw_config(key.as_ref(), value).await?;
@@ -677,7 +714,7 @@ impl Context {
         {
             return Ok(());
         }
-        Box::pin(self.send_sync_msg()).await.log_err(self).ok();
+        self.scheduler.interrupt_smtp().await;
         Ok(())
     }
 
@@ -942,6 +979,17 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mdns_default_behaviour() -> Result<()> {
+        let t = &TestContext::new_alice().await;
+        assert!(t.should_request_mdns().await?);
+        assert!(t.should_send_mdns().await?);
+        t.set_config_bool(Config::Bot, true).await?;
+        assert!(!t.should_request_mdns().await?);
+        assert!(t.should_send_mdns().await?);
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sync() -> Result<()> {
         let alice0 = TestContext::new_alice().await;
@@ -968,20 +1016,16 @@ mod tests {
         // Reset to default. Test that it's not synced because defaults may differ across client
         // versions.
         alice0.set_config(Config::MdnsEnabled, None).await?;
-        assert_eq!(alice0.get_config_bool(Config::MdnsEnabled).await?, true);
         alice0.set_config_bool(Config::MdnsEnabled, false).await?;
         sync(&alice0, &alice1).await;
         assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
 
-        let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
-        alice0
-            .set_config_bool(Config::ShowEmails, !show_emails)
-            .await?;
-        sync(&alice0, &alice1).await;
-        assert_eq!(
-            alice1.get_config_bool(Config::ShowEmails).await?,
-            !show_emails
-        );
+        for key in [Config::ShowEmails, Config::MvboxMove] {
+            let val = alice0.get_config_bool(key).await?;
+            alice0.set_config_bool(key, !val).await?;
+            sync(&alice0, &alice1).await;
+            assert_eq!(alice1.get_config_bool(key).await?, !val);
+        }
 
         // `Config::SyncMsgs` mustn't be synced.
         alice0.set_config_bool(Config::SyncMsgs, false).await?;
@@ -1046,7 +1090,8 @@ mod tests {
 
         let status = "Synced via usual message";
         alice0.set_config(Config::Selfstatus, Some(status)).await?;
-        alice0.pop_sent_msg().await; // Sync message
+        alice0.send_sync_msg().await?;
+        alice0.pop_sent_msg().await;
         let status1 = "Synced via sync message";
         alice1.set_config(Config::Selfstatus, Some(status1)).await?;
         tcm.send_recv(alice0, alice1, "hi Alice!").await;
@@ -1069,7 +1114,8 @@ mod tests {
         alice0
             .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
             .await?;
-        alice0.pop_sent_msg().await; // Sync message
+        alice0.send_sync_msg().await?;
+        alice0.pop_sent_msg().await;
         let file = alice1.dir.path().join("avatar.jpg");
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         tokio::fs::write(&file, bytes).await?;

src/configure.rs

@@ -27,7 +27,7 @@ use crate::config::{self, Config};
 use crate::context::Context;
 use crate::imap::{session::Session as ImapSession, Imap};
 use crate::log::LogExt;
-use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
+use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
@@ -113,10 +113,6 @@ impl Context {
         let mut param = LoginParam::load_candidate_params(self).await?;
         let old_addr = self.get_config(Config::ConfiguredAddr).await?;
 
-        // Reset our knowledge about whether the server is a chatmail server.
-        // We will update it when we connect to IMAP.
-        self.set_config_internal(Config::IsChatmail, None).await?;
-
         let success = configure(self, &mut param).await;
         self.set_config_internal(Config::NotifyAboutWrongPw, None)
             .await?;
@@ -193,10 +189,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     // Step 1: Load the parameters and check email-address and password
 
-    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
-    // socks5 requests, this can work with socks5 too.  OAuth is always set either for both
-    // IMAP and SMTP or not at all.
-    if param.imap.oauth2 && !socks5_enabled {
+    // OAuth is always set either for both IMAP and SMTP or not at all.
+    if param.imap.oauth2 {
         // the used oauth2 addr may differ, check this.
         // if get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -216,7 +210,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     let parsed = EmailAddress::new(&param.addr).context("Bad email-address")?;
     let param_domain = parsed.domain;
-    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
 
     // Step 2: Autoconfig
     progress!(ctx, 200);
@@ -267,7 +260,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                                         }
                                     }
                                 },
-                                strict_tls: Some(provider.opt.strict_tls),
                             })
                             .collect();
 
@@ -282,19 +274,14 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         } else {
             // Try receiving autoconfig
             info!(ctx, "no offline autoconfig found");
-            param_autoconfig = if socks5_enabled {
-                // Currently we can't do http requests through socks5, to not leak
-                // the ip, just don't do online autoconfig
-                info!(ctx, "socks5 enabled, skipping autoconfig");
-                None
-            } else {
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
-            }
+            param_autoconfig = get_autoconfig(ctx, param, &param_domain).await;
         }
     } else {
         param_autoconfig = None;
     }
 
+    let strict_tls = param.strict_tls();
+
     progress!(ctx, 500);
 
     let mut servers = param_autoconfig.unwrap_or_default();
@@ -308,7 +295,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.imap.port,
             socket: param.imap.security,
             username: param.imap.user.clone(),
-            strict_tls: None,
         })
     }
     if !servers
@@ -321,24 +307,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             port: param.smtp.port,
             socket: param.smtp.security,
             username: param.smtp.user.clone(),
-            strict_tls: None,
         })
     }
 
-    // respect certificate setting from function parameters
-    for server in &mut servers {
-        let certificate_checks = match server.protocol {
-            Protocol::Imap => param.imap.certificate_checks,
-            Protocol::Smtp => param.smtp.certificate_checks,
-        };
-        server.strict_tls = match certificate_checks {
-            CertificateChecks::AcceptInvalidCertificates
-            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
-            CertificateChecks::Strict => Some(true),
-            CertificateChecks::Automatic => server.strict_tls,
-        };
-    }
-
     let servers = expand_param_vector(servers, &param.addr, &param_domain);
 
     progress!(ctx, 550);
@@ -354,9 +325,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::Smtp)
         .cloned()
         .collect();
-    let provider_strict_tls = param
-        .provider
-        .map_or(socks5_config.is_some(), |provider| provider.opt.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -366,18 +334,13 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.server.clone_from(&smtp_server.hostname);
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
-            smtp_param.certificate_checks = match smtp_server.strict_tls {
-                Some(true) => CertificateChecks::Strict,
-                Some(false) => CertificateChecks::AcceptInvalidCertificates,
-                None => CertificateChecks::Automatic,
-            };
 
             match try_smtp_one_param(
                 &context_smtp,
                 &smtp_param,
                 &socks5_config,
                 &smtp_addr,
-                provider_strict_tls,
+                strict_tls,
                 &mut smtp,
             )
             .await
@@ -413,18 +376,13 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.server.clone_from(&imap_server.hostname);
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
-        param.imap.certificate_checks = match imap_server.strict_tls {
-            Some(true) => CertificateChecks::Strict,
-            Some(false) => CertificateChecks::AcceptInvalidCertificates,
-            None => CertificateChecks::Automatic,
-        };
 
         match try_imap_one_param(
             ctx,
             &param.imap,
             &param.socks5_config,
             &param.addr,
-            provider_strict_tls,
+            strict_tls,
         )
         .await
         {
@@ -458,7 +416,22 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    if imap_session.is_chatmail() {
+    let is_chatmail = match ctx.get_config_bool(Config::FixIsChatmail).await? {
+        false => {
+            let is_chatmail = imap_session.is_chatmail();
+            ctx.set_config(
+                Config::IsChatmail,
+                Some(match is_chatmail {
+                    false => "0",
+                    true => "1",
+                }),
+            )
+            .await?;
+            is_chatmail
+        }
+        true => ctx.get_config_bool(Config::IsChatmail).await?,
+    };
+    if is_chatmail {
         ctx.set_config(Config::SentboxWatch, None).await?;
         ctx.set_config(Config::MvboxMove, Some("0")).await?;
         ctx.set_config(Config::OnlyFetchMvbox, None).await?;
@@ -466,8 +439,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         ctx.set_config(Config::E2eeEnabled, Some("1")).await?;
     }
 
-    let create_mvbox = ctx.should_watch_mvbox().await?;
-
+    let create_mvbox = !is_chatmail;
     imap.configure_folders(ctx, &mut imap_session, create_mvbox)
         .await?;
 
@@ -512,20 +484,21 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
 /// Retrieve available autoconfigurations.
 ///
-/// A Search configurations from the domain used in the email-address, prefer encrypted
-/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
+/// A. Search configurations from the domain used in the email-address
+/// B. If we have no configuration yet, search configuration in Thunderbird's central database
 async fn get_autoconfig(
     ctx: &Context,
     param: &LoginParam,
     param_domain: &str,
-    param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
+    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
+
     if let Ok(res) = moz_autoconfigure(
         ctx,
         &format!(
             "https://autoconfig.{param_domain}/mail/config-v1.1.xml?emailaddress={param_addr_urlencoded}"
         ),
-        param,
+        &param.addr,
     )
     .await
     {
@@ -540,7 +513,7 @@ async fn get_autoconfig(
             "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
             &param_domain, &param_addr_urlencoded
         ),
-        param,
+        &param.addr,
     )
     .await
     {
@@ -576,7 +549,7 @@ async fn get_autoconfig(
     if let Ok(res) = moz_autoconfigure(
         ctx,
         &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
-        param,
+        &param.addr,
     )
     .await
     {
@@ -591,15 +564,15 @@ async fn try_imap_one_param(
     param: &ServerLoginParam,
     socks5_config: &Option<Socks5Config>,
     addr: &str,
-    provider_strict_tls: bool,
+    strict_tls: bool,
 ) -> Result<(Imap, ImapSession), ConfigurationError> {
     let inf = format!(
-        "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
+        "imap: {}@{}:{} security={} strict_tls={} oauth2={} socks5_config={}",
         param.user,
         param.server,
         param.port,
         param.security,
-        param.certificate_checks,
+        strict_tls,
         param.oauth2,
         if let Some(socks5_config) = socks5_config {
             socks5_config.to_string()
@@ -611,7 +584,7 @@ async fn try_imap_one_param(
 
     let (_s, r) = async_channel::bounded(1);
 
-    let mut imap = match Imap::new(param, socks5_config.clone(), addr, provider_strict_tls, r) {
+    let mut imap = match Imap::new(param, socks5_config.clone(), addr, strict_tls, r) {
         Err(err) => {
             info!(context, "failure: {:#}", err);
             return Err(ConfigurationError {
@@ -624,14 +597,14 @@ async fn try_imap_one_param(
 
     match imap.connect(context).await {
         Err(err) => {
-            info!(context, "failure: {:#}", err);
+            info!(context, "IMAP failure: {err:#}.");
             Err(ConfigurationError {
                 config: inf,
                 msg: format!("{err:#}"),
             })
         }
         Ok(session) => {
-            info!(context, "success: {}", inf);
+            info!(context, "IMAP success: {inf}.");
             Ok((imap, session))
         }
     }
@@ -642,16 +615,16 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     socks5_config: &Option<Socks5Config>,
     addr: &str,
-    provider_strict_tls: bool,
+    strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
+        "smtp: {}@{}:{} security={} strict_tls={} oauth2={} socks5_config={}",
         param.user,
         param.server,
         param.port,
         param.security,
-        param.certificate_checks,
+        strict_tls,
         param.oauth2,
         if let Some(socks5_config) = socks5_config {
             socks5_config.to_string()
@@ -662,16 +635,16 @@ async fn try_smtp_one_param(
     info!(context, "Trying: {}", inf);
 
     if let Err(err) = smtp
-        .connect(context, param, socks5_config, addr, provider_strict_tls)
+        .connect(context, param, socks5_config, addr, strict_tls)
         .await
     {
-        info!(context, "failure: {}", err);
+        info!(context, "SMTP failure: {err:#}.");
         Err(ConfigurationError {
             config: inf,
             msg: format!("{err:#}"),
         })
     } else {
-        info!(context, "success: {}", inf);
+        info!(context, "SMTP success: {inf}.");
         smtp.disconnect();
         Ok(())
     }
@@ -729,7 +702,7 @@ pub enum Error {
 
     #[error("XML error at position {position}: {error}")]
     InvalidXml {
-        position: usize,
+        position: u64,
         #[source]
         error: quick_xml::Error,
     },

src/configure/auto_mozilla.rs

@@ -9,7 +9,6 @@ use quick_xml::events::{BytesStart, Event};
 
 use super::{Error, ServerParams};
 use crate::context::Context;
-use crate::login_param::LoginParam;
 use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
@@ -80,7 +79,7 @@ fn parse_server<B: BufRead>(
         })
         .map(|typ| {
             typ.unwrap()
-                .decode_and_unescape_value(reader)
+                .decode_and_unescape_value(reader.decoder())
                 .unwrap_or_default()
                 .to_lowercase()
         })
@@ -191,7 +190,7 @@ fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoco
     };
 
     let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+    reader.config_mut().trim_text(true);
 
     let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
         position: reader.buffer_position(),
@@ -248,7 +247,6 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
                 hostname: server.hostname,
                 port: server.port,
                 username: server.username,
-                strict_tls: None,
             })
         })
         .collect();
@@ -258,11 +256,11 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
-    param_in: &LoginParam,
+    addr: &str,
 ) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_serverparams(&param_in.addr, &xml_raw);
+    let res = parse_serverparams(addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,

src/configure/auto_outlook.rs

@@ -162,7 +162,7 @@ fn parse_xml_reader<B: BufRead>(
 
 fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+    reader.config_mut().trim_text(true);
 
     parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
         position: reader.buffer_position(),
@@ -187,7 +187,6 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
                 hostname: protocol.server,
                 port: protocol.port,
                 username: String::new(),
-                strict_tls: None,
             })
         })
         .collect()

src/configure/server_params.rs

@@ -22,31 +22,18 @@ pub(crate) struct ServerParams {
 
     /// Username, empty if unknown.
     pub username: String,
-
-    /// Whether TLS certificates should be strictly checked or not, `None` for automatic.
-    pub strict_tls: Option<bool>,
 }
 
 impl ServerParams {
     fn expand_usernames(self, addr: &str) -> Vec<ServerParams> {
-        let mut res = Vec::new();
-
         if self.username.is_empty() {
-            res.push(Self {
+            vec![Self {
                 username: addr.to_string(),
                 ..self.clone()
-            });
-
-            if let Some(at) = addr.find('@') {
-                res.push(Self {
-                    username: addr.split_at(at).0.to_string(),
-                    ..self
-                });
-            }
+            }]
         } else {
-            res.push(self)
+            vec![self]
         }
-        res
     }
 
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
@@ -135,14 +122,6 @@ impl ServerParams {
             vec![self]
         }
     }
-
-    fn expand_strict_tls(self) -> Vec<ServerParams> {
-        vec![Self {
-            // Strict if not set by the user or provider database.
-            strict_tls: Some(self.strict_tls.unwrap_or(true)),
-            ..self
-        }]
-    }
 }
 
 /// Expands vector of `ServerParams`, replacing placeholders with
@@ -155,9 +134,7 @@ pub(crate) fn expand_param_vector(
     v.into_iter()
         // The order of expansion is important.
         //
-        // Ports are expanded the last, so they are changed the first.  Username is only changed if
-        // default value (address with domain) didn't work for all available hosts and ports.
-        .flat_map(|params| params.expand_strict_tls().into_iter())
+        // Ports are expanded the last, so they are changed the first.
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
         .flat_map(|params| params.expand_ports().into_iter())
@@ -177,7 +154,6 @@ mod tests {
                 port: 0,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -191,7 +167,6 @@ mod tests {
                 port: 993,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true)
             }],
         );
 
@@ -202,7 +177,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
-                strict_tls: None,
             }],
             "foobar@example.net",
             "example.net",
@@ -217,7 +191,6 @@ mod tests {
                     port: 123,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true),
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
@@ -225,12 +198,10 @@ mod tests {
                     port: 123,
                     socket: Socket::Starttls,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
             ],
         );
 
-        // Test that strict_tls is not expanded for plaintext connections.
         let v = expand_param_vector(
             vec![ServerParams {
                 protocol: Protocol::Smtp,
@@ -238,7 +209,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Plain,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -251,7 +221,6 @@ mod tests {
                 port: 123,
                 socket: Socket::Plain,
                 username: "foobar".to_string(),
-                strict_tls: Some(true)
             }],
         );
 
@@ -263,7 +232,6 @@ mod tests {
                 port: 10480,
                 socket: Socket::Ssl,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -277,7 +245,6 @@ mod tests {
                     port: 10480,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
                 ServerParams {
                     protocol: Protocol::Imap,
@@ -285,7 +252,6 @@ mod tests {
                     port: 10480,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
                 ServerParams {
                     protocol: Protocol::Imap,
@@ -293,7 +259,6 @@ mod tests {
                     port: 10480,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 }
             ],
         );
@@ -307,7 +272,6 @@ mod tests {
                 port: 0,
                 socket: Socket::Automatic,
                 username: "foobar".to_string(),
-                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",
@@ -321,7 +285,6 @@ mod tests {
                     port: 465,
                     socket: Socket::Ssl,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
                 },
                 ServerParams {
                     protocol: Protocol::Smtp,
@@ -329,7 +292,45 @@ mod tests {
                     port: 587,
                     socket: Socket::Starttls,
                     username: "foobar".to_string(),
-                    strict_tls: Some(true)
+                },
+            ],
+        );
+
+        // Test that email address is used as the default username.
+        // We do not try other usernames
+        // such as the local part of the address
+        // as this is very uncommon configuration
+        // and not worth doubling the number of candidates to try.
+        // If such configuration is used, email provider
+        // should provide XML autoconfig or
+        // be added to the provider database as an exception.
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Imap,
+                hostname: "example.net".to_string(),
+                port: 0,
+                socket: Socket::Automatic,
+                username: "".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 993,
+                    socket: Socket::Ssl,
+                    username: "foobar@example.net".to_string(),
+                },
+                ServerParams {
+                    protocol: Protocol::Imap,
+                    hostname: "example.net".to_string(),
+                    port: 143,
+                    socket: Socket::Starttls,
+                    username: "foobar@example.net".to_string(),
                 },
             ],
         );

src/constants.rs

@@ -209,7 +209,7 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // Key for the folder configuration version (see below).
 pub(crate) const DC_FOLDERS_CONFIGURED_KEY: &str = "folders_configured";
 // this value can be increased if the folder configuration is changed and must be redone on next program start
-pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
+pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 5;
 
 // If more recipients are needed in SMTP's `RCPT TO:` header, the recipient list is split into
 // chunks. This does not affect MIME's `To:` header. Can be overwritten by setting

src/contact.rs

@@ -11,7 +11,7 @@ use async_channel::{self as channel, Receiver, Sender};
 use base64::Engine as _;
 pub use deltachat_contact_tools::may_be_valid_addr;
 use deltachat_contact_tools::{
-    self as contact_tools, addr_cmp, addr_normalize, sanitize_name_and_addr, strip_rtlo_characters,
+    self as contact_tools, addr_cmp, addr_normalize, sanitize_name, sanitize_name_and_addr,
     ContactAddress, VcardContact,
 };
 use deltachat_derive::{FromSql, ToSql};
@@ -37,9 +37,7 @@ use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::sql::{self, params_iter};
 use crate::sync::{self, Sync::*};
-use crate::tools::{
-    duration_to_str, get_abs_path, improve_single_line_input, smeared_time, time, SystemTime,
-};
+use crate::tools::{duration_to_str, get_abs_path, smeared_time, time, SystemTime};
 use crate::{chat, chatlist_events, stock_str};
 
 /// Time during which a contact is considered as seen recently.
@@ -626,9 +624,7 @@ impl Contact {
         name: &str,
         addr: &str,
     ) -> Result<ContactId> {
-        let name = improve_single_line_input(name);
-
-        let (name, addr) = sanitize_name_and_addr(&name, addr);
+        let (name, addr) = sanitize_name_and_addr(name, addr);
         let addr = ContactAddress::new(&addr)?;
 
         let (contact_id, sth_modified) =
@@ -646,7 +642,7 @@ impl Contact {
             set_blocked(context, Nosync, contact_id, false).await?;
         }
 
-        if sync.into() {
+        if sync.into() && sth_modified != Modifier::None {
             chat::sync(
                 context,
                 chat::SyncId::ContactAddr(addr.to_string()),
@@ -751,7 +747,7 @@ impl Contact {
     /// - "name": name passed as function argument, belonging to the given origin
     /// - "row_name": current name used in the database, typically set to "name"
     /// - "row_authname": name as authorized from a contact, set only through a From-header
-    /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
+    ///   Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
     ///
     /// Returns the contact_id and a `Modifier` value indicating if a modification occurred.
     pub(crate) async fn add_or_lookup(
@@ -769,7 +765,7 @@ impl Contact {
             return Ok((ContactId::SELF, sth_modified));
         }
 
-        let mut name = strip_rtlo_characters(name);
+        let mut name = sanitize_name(name);
         #[allow(clippy::collapsible_if)]
         if origin <= Origin::OutgoingTo {
             // The user may accidentally have written to a "noreply" address with another MUA:
@@ -1001,7 +997,7 @@ impl Contact {
     /// - if the flag DC_GCL_ADD_SELF is set, SELF is added to the list unless filtered by other parameters
     /// - if the flag DC_GCL_VERIFIED_ONLY is set, only verified contacts are returned.
     ///   if DC_GCL_VERIFIED_ONLY is not set, verified and unverified contacts are returned.
-    /// `query` is a string to filter the list.
+    ///   `query` is a string to filter the list.
     pub async fn get_all(
         context: &Context,
         listflags: u32,
@@ -1917,14 +1913,19 @@ impl RecentlySeenLoop {
             .unwrap();
     }
 
-    pub(crate) fn abort(self) {
+    pub(crate) async fn abort(self) {
         self.handle.abort();
+
+        // Await aborted task to ensure the `Future` is dropped
+        // with all resources moved inside such as the `Context`
+        // reference to `InnerContext`.
+        self.handle.await.ok();
     }
 }
 
 #[cfg(test)]
 mod tests {
-    use deltachat_contact_tools::{may_be_valid_addr, normalize_name};
+    use deltachat_contact_tools::may_be_valid_addr;
 
     use super::*;
     use crate::chat::{get_chat_contacts, send_text_msg, Chat};
@@ -1963,15 +1964,6 @@ mod tests {
         assert_eq!(may_be_valid_addr("user@domain.tld."), false);
     }
 
-    #[test]
-    fn test_normalize_name() {
-        assert_eq!(&normalize_name(" hello world   "), "hello world");
-        assert_eq!(&normalize_name("<"), "<");
-        assert_eq!(&normalize_name(">"), ">");
-        assert_eq!(&normalize_name("'"), "'");
-        assert_eq!(&normalize_name("\""), "\"");
-    }
-
     #[test]
     fn test_normalize_addr() {
         assert_eq!(addr_normalize("mailto:john@doe.com"), "john@doe.com");

src/context.rs

@@ -541,18 +541,10 @@ impl Context {
         }
 
         // update quota (to send warning if full) - but only check it once in a while
-        let quota_needs_update = {
-            let quota = self.quota.read().await;
-            quota
-                .as_ref()
-                .filter(|quota| {
-                    time_elapsed(&quota.modified)
-                        > Duration::from_secs(DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT)
-                })
-                .is_none()
-        };
-
-        if quota_needs_update {
+        if self
+            .quota_needs_update(DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT)
+            .await
+        {
             if let Err(err) = self.update_recent_quota(&mut session).await {
                 warn!(self, "Failed to update quota: {err:#}.");
             }
@@ -822,6 +814,16 @@ impl Context {
         }
 
         res.insert("is_chatmail", self.is_chatmail().await?.to_string());
+        res.insert(
+            "fix_is_chatmail",
+            self.get_config_bool(Config::FixIsChatmail)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "is_muted",
+            self.get_config_bool(Config::IsMuted).await?.to_string(),
+        );
 
         if let Some(metadata) = &*self.metadata.read().await {
             if let Some(comment) = &metadata.comment {
@@ -1259,12 +1261,12 @@ impl Context {
         Ok(list)
     }
 
-    /// Searches for messages containing the query string.
+    /// Searches for messages containing the query string case-insensitively.
     ///
     /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
     /// is `None` this searches messages from all chats.
     pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: &str) -> Result<Vec<MsgId>> {
-        let real_query = query.trim();
+        let real_query = query.trim().to_lowercase();
         if real_query.is_empty() {
             return Ok(Vec::new());
         }
@@ -1280,7 +1282,7 @@ impl Context {
                  WHERE m.chat_id=?
                    AND m.hidden=0
                    AND ct.blocked=0
-                   AND txt LIKE ?
+                   AND IFNULL(txt_normalized, txt) LIKE ?
                  ORDER BY m.timestamp,m.id;",
                     (chat_id, str_like_in_text),
                     |row| row.get::<_, MsgId>("id"),
@@ -1316,7 +1318,7 @@ impl Context {
                    AND m.hidden=0
                    AND c.blocked!=1
                    AND ct.blocked=0
-                   AND m.txt LIKE ?
+                   AND IFNULL(txt_normalized, txt) LIKE ?
                  ORDER BY m.id DESC LIMIT 1000",
                     (str_like_in_text,),
                     |row| row.get::<_, MsgId>("id"),
@@ -1346,7 +1348,7 @@ impl Context {
         Ok(sentbox.as_deref() == Some(folder_name))
     }
 
-    /// Returns true if given folder name is the name of the "Delta Chat" folder.
+    /// Returns true if given folder name is the name of the "DeltaChat" folder.
     pub async fn is_mvbox(&self, folder_name: &str) -> Result<bool> {
         let mvbox = self.get_config(Config::ConfiguredMvboxFolder).await?;
         Ok(mvbox.as_deref() == Some(folder_name))
@@ -1558,6 +1560,22 @@ mod tests {
         assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_muted_context() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+        t.set_config(Config::IsMuted, Some("1")).await?;
+        let chat = t.create_chat_with_contact("", "bob@g.it").await;
+        receive_msg(&t, &chat).await;
+
+        // muted contexts should still show dimmed badge counters eg. in the sidebars,
+        // (same as muted chats show dimmed badge counters in the chatlist)
+        // therefore the fresh messages count should not be affected.
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
@@ -1721,6 +1739,8 @@ mod tests {
         msg2.set_text("barbaz".to_string());
         send_msg(&alice, chat.id, &mut msg2).await?;
 
+        alice.send_text(chat.id, "Δ-Chat").await;
+
         // Global search with a part of text finds the message.
         let res = alice.search_msgs(None, "ob").await?;
         assert_eq!(res.len(), 1);
@@ -1733,6 +1753,12 @@ mod tests {
         assert_eq!(res.first(), Some(&msg2.id));
         assert_eq!(res.get(1), Some(&msg1.id));
 
+        // Search is case-insensitive.
+        for chat_id in [None, Some(chat.id)] {
+            let res = alice.search_msgs(chat_id, "δ-chat").await?;
+            assert_eq!(res.len(), 1);
+        }
+
         // Global search with longer text does not find any message.
         let res = alice.search_msgs(None, "foobarbaz").await?;
         assert!(res.is_empty());

src/dehtml.rs

@@ -129,7 +129,7 @@ fn dehtml_quick_xml(buf: &str) -> (String, String) {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
-    reader.check_end_names(false);
+    reader.config_mut().check_end_names = false;
 
     let mut buf = Vec::new();
 
@@ -299,7 +299,7 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
                 })
             {
                 let href = href
-                    .decode_and_unescape_value(reader)
+                    .decode_and_unescape_value(reader.decoder())
                     .unwrap_or_default()
                     .to_string();
 
@@ -348,7 +348,7 @@ fn maybe_push_tag(
 fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
     event.attributes().any(|r| {
         r.map(|a| {
-            a.decode_and_unescape_value(reader)
+            a.decode_and_unescape_value(reader.decoder())
                 .map(|v| v == name)
                 .unwrap_or(false)
         })
@@ -457,7 +457,7 @@ mod tests {
 
     #[test]
     fn test_dehtml_parse_href() {
-        let html = "<a href=url>text</a";
+        let html = "<a href=url>text</a>";
         let plain = dehtml(html).unwrap().text;
 
         assert_eq!(plain, "[text](url)");

src/download.rs

@@ -184,7 +184,7 @@ impl Session {
             bail!("Attempt to fetch UID 0");
         }
 
-        self.select_folder(context, folder).await?;
+        self.select_with_uidvalidity(context, folder).await?;
 
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);

src/ephemeral.rs

@@ -447,7 +447,7 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
                 for (msg_id, chat_id, viewtype, location_id) in rows {
                     transaction.execute(
                         "UPDATE msgs
-                     SET chat_id=?, txt='', subject='', txt_raw='',
+                     SET chat_id=?, txt='', txt_normalized=NULL, subject='', txt_raw='',
                          mime_headers='', from_id=0, to_id=0, param=''
                      WHERE id=?",
                         (DC_CHAT_ID_TRASH, msg_id),
@@ -1024,7 +1024,7 @@ mod tests {
         t.send_text(self_chat.id, "Saved message, which we delete manually")
             .await;
         let msg = t.get_last_msg_in(self_chat.id).await;
-        msg.id.trash(&t).await?;
+        msg.id.trash(&t, false).await?;
         check_msg_is_deleted(&t, &self_chat, msg.id).await;
 
         self_chat
@@ -1304,7 +1304,7 @@ mod tests {
         let msg = alice.get_last_msg().await;
 
         // Message is deleted when its timer expires.
-        msg.id.trash(&alice).await?;
+        msg.id.trash(&alice, false).await?;
 
         // Message with Message-ID <third@example.com>, referencing <first@example.com> and
         // <second@example.com>, is received.  The message <second@example.come> is not in the

src/headerdef.rs

@@ -11,6 +11,7 @@ pub enum HeaderDef {
     Date,
     From_,
     To,
+    AutoSubmitted,
 
     /// Carbon copy.
     Cc,

src/imap.rs

@@ -16,7 +16,7 @@ use std::{
 use anyhow::{bail, format_err, Context as _, Result};
 use async_channel::Receiver;
 use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
-use deltachat_contact_tools::{normalize_name, ContactAddress};
+use deltachat_contact_tools::ContactAddress;
 use futures::{FutureExt as _, StreamExt, TryStreamExt};
 use futures_lite::FutureExt;
 use num_traits::FromPrimitive;
@@ -32,11 +32,10 @@ use crate::contact::{Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
+use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::{self, Message, MessageState, MessengerMessage, MsgId, Viewtype};
 use crate::mimeparser;
 use crate::oauth2::get_oauth2_access_token;
-use crate::provider::Socket;
 use crate::receive_imf::{
     from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner, ReceivedMsg,
 };
@@ -232,20 +231,13 @@ impl Imap {
         lp: &ServerLoginParam,
         socks5_config: Option<Socks5Config>,
         addr: &str,
-        provider_strict_tls: bool,
+        strict_tls: bool,
         idle_interrupt_receiver: Receiver<()>,
     ) -> Result<Self> {
         if lp.server.is_empty() || lp.user.is_empty() || lp.password.is_empty() {
             bail!("Incomplete IMAP connection parameters");
         }
 
-        let strict_tls = match lp.certificate_checks {
-            CertificateChecks::Automatic => provider_strict_tls,
-            CertificateChecks::Strict => true,
-            CertificateChecks::AcceptInvalidCertificates
-            | CertificateChecks::AcceptInvalidCertificates2 => false,
-        };
-
         let imap = Imap {
             idle_interrupt_receiver,
             addr: addr.to_string(),
@@ -273,17 +265,11 @@ impl Imap {
         }
 
         let param = LoginParam::load_configured_params(context).await?;
-        // the trailing underscore is correct
-
         let imap = Self::new(
             &param.imap,
             param.socks5_config.clone(),
             &param.addr,
-            param
-                .provider
-                .map_or(param.socks5_config.is_some(), |provider| {
-                    provider.opt.strict_tls
-                }),
+            param.strict_tls(),
             idle_interrupt_receiver,
         )?;
         Ok(imap)
@@ -313,7 +299,7 @@ impl Imap {
         if !ratelimit_duration.is_zero() {
             warn!(
                 context,
-                "IMAP got rate limited, waiting for {} until can connect",
+                "IMAP got rate limited, waiting for {} until can connect.",
                 duration_to_str(ratelimit_duration),
             );
             let interrupted = async {
@@ -342,52 +328,16 @@ impl Imap {
         );
         self.conn_backoff_ms = max(BACKOFF_MIN_MS, self.conn_backoff_ms);
 
-        let connection_res: Result<Client> =
-            if self.lp.security == Socket::Starttls || self.lp.security == Socket::Plain {
-                let imap_server: &str = self.lp.server.as_ref();
-                let imap_port = self.lp.port;
+        let connection_res = Client::connect(
+            context,
+            self.lp.server.as_ref(),
+            self.lp.port,
+            self.strict_tls,
+            self.socks5_config.clone(),
+            self.lp.security,
+        )
+        .await;
 
-                if let Some(socks5_config) = &self.socks5_config {
-                    if self.lp.security == Socket::Starttls {
-                        Client::connect_starttls_socks5(
-                            context,
-                            imap_server,
-                            imap_port,
-                            socks5_config.clone(),
-                            self.strict_tls,
-                        )
-                        .await
-                    } else {
-                        Client::connect_insecure_socks5(
-                            context,
-                            imap_server,
-                            imap_port,
-                            socks5_config.clone(),
-                        )
-                        .await
-                    }
-                } else if self.lp.security == Socket::Starttls {
-                    Client::connect_starttls(context, imap_server, imap_port, self.strict_tls).await
-                } else {
-                    Client::connect_insecure(context, imap_server, imap_port).await
-                }
-            } else {
-                let imap_server: &str = self.lp.server.as_ref();
-                let imap_port = self.lp.port;
-
-                if let Some(socks5_config) = &self.socks5_config {
-                    Client::connect_secure_socks5(
-                        context,
-                        imap_server,
-                        imap_port,
-                        self.strict_tls,
-                        socks5_config.clone(),
-                    )
-                    .await
-                } else {
-                    Client::connect_secure(context, imap_server, imap_port, self.strict_tls).await
-                }
-            };
         let client = connection_res?;
         self.conn_backoff_ms = BACKOFF_MIN_MS;
         self.ratelimit.send();
@@ -483,7 +433,11 @@ impl Imap {
             .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
             .await?;
         if folders_configured.unwrap_or_default() < constants::DC_FOLDERS_CONFIGURED_VERSION {
-            let create_mvbox = true;
+            let is_chatmail = match context.get_config_bool(Config::FixIsChatmail).await? {
+                false => session.is_chatmail(),
+                true => context.get_config_bool(Config::IsChatmail).await?,
+            };
+            let create_mvbox = !is_chatmail || context.get_config_bool(Config::MvboxMove).await?;
             self.configure_folders(context, &mut session, create_mvbox)
                 .await?;
         }
@@ -540,18 +494,20 @@ impl Imap {
     ) -> Result<bool> {
         if should_ignore_folder(context, folder, folder_meaning).await? {
             info!(context, "Not fetching from {folder:?}.");
+            session.new_mail = false;
             return Ok(false);
         }
 
-        let new_emails = session
+        session
             .select_with_uidvalidity(context, folder)
             .await
             .with_context(|| format!("Failed to select folder {folder:?}"))?;
 
-        if !new_emails && !fetch_existing_msgs {
+        if !session.new_mail && !fetch_existing_msgs {
             info!(context, "No new emails in folder {folder:?}.");
             return Ok(false);
         }
+        session.new_mail = false;
 
         let uid_validity = get_uidvalidity(context, folder).await?;
         let old_uid_next = get_uid_next(context, folder).await?;
@@ -598,20 +554,26 @@ impl Imap {
             // in the `INBOX.DeltaChat` folder again.
             let _target;
             let target = if let Some(message_id) = &message_id {
-                let is_dup = if let Some((_, ts_sent_old)) =
-                    message::rfc724_mid_exists(context, message_id).await?
-                {
+                let msg_info =
+                    message::rfc724_mid_exists_ex(context, message_id, "deleted=1").await?;
+                let delete = if let Some((_, _, true)) = msg_info {
+                    info!(context, "Deleting locally deleted message {message_id}.");
+                    true
+                } else if let Some((_, ts_sent_old, _)) = msg_info {
                     let is_chat_msg = headers.get_header_value(HeaderDef::ChatVersion).is_some();
                     let ts_sent = headers
                         .get_header_value(HeaderDef::Date)
                         .and_then(|v| mailparse::dateparse(&v).ok())
                         .unwrap_or_default();
-                    is_dup_msg(is_chat_msg, ts_sent, ts_sent_old)
+                    let is_dup = is_dup_msg(is_chat_msg, ts_sent, ts_sent_old);
+                    if is_dup {
+                        info!(context, "Deleting duplicate message {message_id}.");
+                    }
+                    is_dup
                 } else {
                     false
                 };
-                if is_dup {
-                    info!(context, "Deleting duplicate message {message_id}.");
+                if delete {
                     &delete_target
                 } else if context
                     .sql
@@ -765,10 +727,6 @@ impl Imap {
         context: &Context,
         session: &mut Session,
     ) -> Result<()> {
-        if context.get_config_bool(Config::Bot).await? {
-            return Ok(()); // Bots don't want those messages
-        }
-
         add_all_recipients_as_contacts(context, session, Config::ConfiguredSentboxFolder)
             .await
             .context("failed to get recipients from the sentbox")?;
@@ -838,7 +796,7 @@ impl Session {
         // Collect pairs of UID and Message-ID.
         let mut msgs = BTreeMap::new();
 
-        self.select_folder(context, folder).await?;
+        self.select_with_uidvalidity(context, folder).await?;
 
         let mut list = self
             .uid_fetch("1:*", RFC724MID_UID)
@@ -1039,7 +997,7 @@ impl Session {
             // MOVE/DELETE operations. This does not result in multiple SELECT commands
             // being sent because `select_folder()` does nothing if the folder is already
             // selected.
-            self.select_folder(context, folder).await?;
+            self.select_with_uidvalidity(context, folder).await?;
 
             // Empty target folder name means messages should be deleted.
             if target.is_empty() {
@@ -1087,18 +1045,12 @@ impl Session {
             .await?;
 
         for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
-            self.select_folder(context, &folder)
-                .await
-                .context("failed to select folder")?;
-
-            if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
+            if let Err(err) = self.select_with_uidvalidity(context, &folder).await {
+                warn!(context, "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}.");
+            } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
                 warn!(
                     context,
-                    "Cannot mark messages {} in folder {} as seen, will retry later: {}.",
-                    uid_set,
-                    folder,
-                    err
-                );
+                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}.");
             } else {
                 info!(
                     context,
@@ -1131,7 +1083,7 @@ impl Session {
             return Ok(());
         }
 
-        self.select_folder(context, folder)
+        self.select_with_uidvalidity(context, folder)
             .await
             .context("failed to select folder")?;
 
@@ -1197,6 +1149,9 @@ impl Session {
         set_modseq(context, folder, highest_modseq)
             .await
             .with_context(|| format!("failed to set MODSEQ for folder {folder}"))?;
+        if !updated_chat_ids.is_empty() {
+            context.on_archived_chats_maybe_noticed();
+        }
         for updated_chat_id in updated_chat_ids {
             context.emit_event(EventType::MsgsNoticed(updated_chat_id));
             chatlist_events::emit_chatlist_item_changed(context, updated_chat_id);
@@ -1519,7 +1474,7 @@ impl Session {
         } else if !context.push_subscriber.heartbeat_subscribed().await {
             let context = context.clone();
             // Subscribe for heartbeat notifications.
-            tokio::spawn(async move { context.push_subscriber.subscribe().await });
+            tokio::spawn(async move { context.push_subscriber.subscribe(&context).await });
         }
 
         Ok(())
@@ -1549,8 +1504,8 @@ impl Session {
 
     /// Attempts to configure mvbox.
     ///
-    /// Tries to find any folder in the given list of `folders`. If none is found, tries to create
-    /// `folders[0]`. This method does not use LIST command to ensure that
+    /// Tries to find any folder examining `folders` in the order they go. If none is found, tries
+    /// to create any folder in the same order. This method does not use LIST command to ensure that
     /// configuration works even if mailbox lookup is forbidden via Access Control List (see
     /// <https://datatracker.ietf.org/doc/html/rfc4314>).
     ///
@@ -1584,16 +1539,17 @@ impl Session {
         if !create_mvbox {
             return Ok(None);
         }
-        let Some(folder) = folders.first() else {
-            return Ok(None);
-        };
-        match self.select_with_uidvalidity(context, folder).await {
-            Ok(_) => {
-                info!(context, "MVBOX-folder {} created.", folder);
-                return Ok(Some(folder));
-            }
-            Err(err) => {
-                warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
+        // Some servers require namespace-style folder names like "INBOX.DeltaChat", so we try all
+        // the variants here.
+        for folder in folders {
+            match self.select_with_uidvalidity(context, folder).await {
+                Ok(_) => {
+                    info!(context, "MVBOX-folder {} created.", folder);
+                    return Ok(Some(folder));
+                }
+                Err(err) => {
+                    warn!(context, "Cannot create MVBOX-folder {:?}: {}", folder, err);
+                }
             }
         }
         Ok(None)
@@ -1841,6 +1797,20 @@ async fn needs_move_to_mvbox(
     context: &Context,
     headers: &[mailparse::MailHeader<'_>],
 ) -> Result<bool> {
+    let has_chat_version = headers.get_header_value(HeaderDef::ChatVersion).is_some();
+    if !context.get_config_bool(Config::IsChatmail).await?
+        && has_chat_version
+        && headers
+            .get_header_value(HeaderDef::AutoSubmitted)
+            .filter(|val| val.to_ascii_lowercase() == "auto-generated")
+            .is_some()
+    {
+        if let Some(from) = mimeparser::get_from(headers) {
+            if context.is_self_addr(&from.addr).await? {
+                return Ok(true);
+            }
+        }
+    }
     if !context.get_config_bool(Config::MvboxMove).await? {
         return Ok(false);
     }
@@ -1854,7 +1824,7 @@ async fn needs_move_to_mvbox(
         return Ok(false);
     }
 
-    if headers.get_header_value(HeaderDef::ChatVersion).is_some() {
+    if has_chat_version {
         Ok(true)
     } else if let Some(parent) = get_prefetch_parent_message(context, headers).await? {
         match parent.is_dc_message {
@@ -2421,12 +2391,6 @@ async fn add_all_recipients_as_contacts(
 
     let mut any_modified = false;
     for recipient in recipients {
-        let display_name_normalized = recipient
-            .display_name
-            .as_ref()
-            .map(|s| normalize_name(s))
-            .unwrap_or_default();
-
         let recipient_addr = match ContactAddress::new(&recipient.addr) {
             Err(err) => {
                 warn!(
@@ -2442,7 +2406,7 @@ async fn add_all_recipients_as_contacts(
 
         let (_, modified) = Contact::add_or_lookup(
             context,
-            &display_name_normalized,
+            &recipient.display_name.unwrap_or_default(),
             &recipient_addr,
             Origin::OutgoingTo,
         )

src/imap/client.rs

@@ -1,24 +1,23 @@
-use std::{
-    ops::{Deref, DerefMut},
-    time::Duration,
-};
+use std::net::SocketAddr;
+use std::ops::{Deref, DerefMut};
 
-use anyhow::{Context as _, Result};
+use anyhow::{bail, format_err, Context as _, Result};
 use async_imap::Client as ImapClient;
 use async_imap::Session as ImapSession;
+use fast_socks5::client::Socks5Stream;
 use tokio::io::BufWriter;
 
 use super::capabilities::Capabilities;
 use super::session::Session;
 use crate::context::Context;
-use crate::net::connect_tcp;
+use crate::net::dns::{lookup_host_with_cache, update_connect_timestamp};
 use crate::net::session::SessionStream;
 use crate::net::tls::wrap_tls;
+use crate::net::update_connection_history;
+use crate::net::{connect_tcp_inner, connect_tls_inner};
+use crate::provider::Socket;
 use crate::socks::Socks5Config;
-use fast_socks5::client::Socks5Stream;
-
-/// IMAP connection, write and read timeout.
-pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(60);
+use crate::tools::time;
 
 #[derive(Debug)]
 pub(crate) struct Client {
@@ -98,14 +97,67 @@ impl Client {
         Ok(Session::new(session, capabilities))
     }
 
-    pub async fn connect_secure(
+    pub async fn connect(
         context: &Context,
-        hostname: &str,
+        host: &str,
         port: u16,
         strict_tls: bool,
+        socks5_config: Option<Socks5Config>,
+        security: Socket,
     ) -> Result<Self> {
-        let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?;
-        let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?;
+        if let Some(socks5_config) = socks5_config {
+            let client = match security {
+                Socket::Automatic => bail!("IMAP port security is not configured"),
+                Socket::Ssl => {
+                    Client::connect_secure_socks5(context, host, port, strict_tls, socks5_config)
+                        .await?
+                }
+                Socket::Starttls => {
+                    Client::connect_starttls_socks5(context, host, port, socks5_config, strict_tls)
+                        .await?
+                }
+                Socket::Plain => {
+                    Client::connect_insecure_socks5(context, host, port, socks5_config).await?
+                }
+            };
+            Ok(client)
+        } else {
+            let mut first_error = None;
+            let load_cache =
+                strict_tls && (security == Socket::Ssl || security == Socket::Starttls);
+            for resolved_addr in
+                lookup_host_with_cache(context, host, port, "imap", load_cache).await?
+            {
+                let res = match security {
+                    Socket::Automatic => bail!("IMAP port security is not configured"),
+                    Socket::Ssl => Client::connect_secure(resolved_addr, host, strict_tls).await,
+                    Socket::Starttls => {
+                        Client::connect_starttls(resolved_addr, host, strict_tls).await
+                    }
+                    Socket::Plain => Client::connect_insecure(resolved_addr).await,
+                };
+                match res {
+                    Ok(client) => {
+                        let ip_addr = resolved_addr.ip().to_string();
+                        if load_cache {
+                            update_connect_timestamp(context, host, &ip_addr).await?;
+                        }
+                        update_connection_history(context, "imap", host, port, &ip_addr, time())
+                            .await?;
+                        return Ok(client);
+                    }
+                    Err(err) => {
+                        warn!(context, "Failed to connect to {resolved_addr}: {err:#}.");
+                        first_error.get_or_insert(err);
+                    }
+                }
+            }
+            Err(first_error.unwrap_or_else(|| format_err!("no DNS resolution results for {host}")))
+        }
+    }
+
+    async fn connect_secure(addr: SocketAddr, hostname: &str, strict_tls: bool) -> Result<Self> {
+        let tls_stream = connect_tls_inner(addr, hostname, strict_tls, "imap").await?;
         let buffered_stream = BufWriter::new(tls_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = Client::new(session_stream);
@@ -116,8 +168,8 @@ impl Client {
         Ok(client)
     }
 
-    pub async fn connect_insecure(context: &Context, hostname: &str, port: u16) -> Result<Self> {
-        let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, false).await?;
+    async fn connect_insecure(addr: SocketAddr) -> Result<Self> {
+        let tcp_stream = connect_tcp_inner(addr).await?;
         let buffered_stream = BufWriter::new(tcp_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = Client::new(session_stream);
@@ -128,17 +180,12 @@ impl Client {
         Ok(client)
     }
 
-    pub async fn connect_starttls(
-        context: &Context,
-        hostname: &str,
-        port: u16,
-        strict_tls: bool,
-    ) -> Result<Self> {
-        let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?;
+    async fn connect_starttls(addr: SocketAddr, host: &str, strict_tls: bool) -> Result<Self> {
+        let tcp_stream = connect_tcp_inner(addr).await?;
 
         // Run STARTTLS command and convert the client back into a stream.
         let buffered_tcp_stream = BufWriter::new(tcp_stream);
-        let mut client = ImapClient::new(buffered_tcp_stream);
+        let mut client = async_imap::Client::new(buffered_tcp_stream);
         let _greeting = client
             .read_response()
             .await
@@ -150,7 +197,7 @@ impl Client {
         let buffered_tcp_stream = client.into_inner();
         let tcp_stream = buffered_tcp_stream.into_inner();
 
-        let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream)
+        let tls_stream = wrap_tls(strict_tls, host, "imap", tcp_stream)
             .await
             .context("STARTTLS upgrade failed")?;
 
@@ -160,7 +207,7 @@ impl Client {
         Ok(client)
     }
 
-    pub async fn connect_secure_socks5(
+    async fn connect_secure_socks5(
         context: &Context,
         domain: &str,
         port: u16,
@@ -168,9 +215,9 @@ impl Client {
         socks5_config: Socks5Config,
     ) -> Result<Self> {
         let socks5_stream = socks5_config
-            .connect(context, domain, port, IMAP_TIMEOUT, strict_tls)
+            .connect(context, domain, port, strict_tls)
             .await?;
-        let tls_stream = wrap_tls(strict_tls, domain, socks5_stream).await?;
+        let tls_stream = wrap_tls(strict_tls, domain, "imap", socks5_stream).await?;
         let buffered_stream = BufWriter::new(tls_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = Client::new(session_stream);
@@ -181,15 +228,13 @@ impl Client {
         Ok(client)
     }
 
-    pub async fn connect_insecure_socks5(
+    async fn connect_insecure_socks5(
         context: &Context,
         domain: &str,
         port: u16,
         socks5_config: Socks5Config,
     ) -> Result<Self> {
-        let socks5_stream = socks5_config
-            .connect(context, domain, port, IMAP_TIMEOUT, false)
-            .await?;
+        let socks5_stream = socks5_config.connect(context, domain, port, false).await?;
         let buffered_stream = BufWriter::new(socks5_stream);
         let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
         let mut client = Client::new(session_stream);
@@ -200,7 +245,7 @@ impl Client {
         Ok(client)
     }
 
-    pub async fn connect_starttls_socks5(
+    async fn connect_starttls_socks5(
         context: &Context,
         hostname: &str,
         port: u16,
@@ -208,7 +253,7 @@ impl Client {
         strict_tls: bool,
     ) -> Result<Self> {
         let socks5_stream = socks5_config
-            .connect(context, hostname, port, IMAP_TIMEOUT, strict_tls)
+            .connect(context, hostname, port, strict_tls)
             .await?;
 
         // Run STARTTLS command and convert the client back into a stream.
@@ -225,7 +270,7 @@ impl Client {
         let buffered_socks5_stream = client.into_inner();
         let socks5_stream: Socks5Stream<_> = buffered_socks5_stream.into_inner();
 
-        let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream)
+        let tls_stream = wrap_tls(strict_tls, hostname, "imap", socks5_stream)
             .await
             .context("STARTTLS upgrade failed")?;
         let buffered_stream = BufWriter::new(tls_stream);

src/imap/idle.rs

@@ -9,7 +9,8 @@ use tokio::time::timeout;
 use super::session::Session;
 use super::Imap;
 use crate::context::Context;
-use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
+use crate::imap::FolderMeaning;
+use crate::net::TIMEOUT;
 use crate::tools::{self, time_elapsed};
 
 /// Timeout after which IDLE is finished
@@ -29,9 +30,13 @@ impl Session {
     ) -> Result<Self> {
         use futures::future::FutureExt;
 
-        self.select_folder(context, folder).await?;
+        self.select_with_uidvalidity(context, folder).await?;
 
         if self.server_sent_unsolicited_exists(context)? {
+            self.new_mail = true;
+        }
+
+        if self.new_mail {
             return Ok(self);
         }
 
@@ -47,7 +52,7 @@ impl Session {
 
         // At this point IDLE command was sent and we received a "+ idling" response. We will now
         // read from the stream without getting any data for up to `IDLE_TIMEOUT`. If we don't
-        // disable read timeout, we would get a timeout after `IMAP_TIMEOUT`, which is a lot
+        // disable read timeout, we would get a timeout after `crate::net::TIMEOUT`, which is a lot
         // shorter than `IDLE_TIMEOUT`.
         handle.as_mut().set_read_timeout(None);
         let (idle_wait, interrupt) = handle.wait_with_timeout(IDLE_TIMEOUT);
@@ -89,9 +94,12 @@ impl Session {
             .await
             .with_context(|| format!("{folder}: IMAP IDLE protocol timed out"))?
             .with_context(|| format!("{folder}: IMAP IDLE failed"))?;
-        session.as_mut().set_read_timeout(Some(IMAP_TIMEOUT));
+        session.as_mut().set_read_timeout(Some(TIMEOUT));
         self.inner = session;
 
+        // Fetch mail once we exit IDLE.
+        self.new_mail = true;
+
         Ok(self)
     }
 }

src/imap/select_folder.rs

@@ -10,12 +10,6 @@ type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[error("IMAP Connection Lost or no connection established")]
-    ConnectionLost,
-
-    #[error("IMAP Folder name invalid: {0}")]
-    BadFolderName(String),
-
     #[error("Got a NO response when trying to select {0}, usually this means that it doesn't exist: {1}")]
     NoFolder(String, String),
 
@@ -33,7 +27,8 @@ impl ImapSession {
     /// Issues a CLOSE command if selected folder needs expunge,
     /// i.e. if Delta Chat marked a message there as deleted previously.
     ///
-    /// CLOSE is considerably faster than an EXPUNGE, see
+    /// CLOSE is considerably faster than an EXPUNGE
+    /// because no EXPUNGE responses are sent, see
     /// <https://tools.ietf.org/html/rfc3501#section-6.4.2>
     pub(super) async fn maybe_close_folder(&mut self, context: &Context) -> anyhow::Result<()> {
         if let Some(folder) = &self.selected_folder {
@@ -44,6 +39,7 @@ impl ImapSession {
                 info!(context, "close/expunge succeeded");
                 self.selected_folder = None;
                 self.selected_folder_needs_expunge = false;
+                self.new_mail = false;
             }
         }
         Ok(())
@@ -52,11 +48,7 @@ impl ImapSession {
     /// Selects a folder, possibly updating uid_validity and, if needed,
     /// expunging the folder to remove delete-marked messages.
     /// Returns whether a new folder was selected.
-    pub(crate) async fn select_folder(
-        &mut self,
-        context: &Context,
-        folder: &str,
-    ) -> Result<NewlySelected> {
+    async fn select_folder(&mut self, context: &Context, folder: &str) -> Result<NewlySelected> {
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
         // if there is _no_ new folder, we continue as we might want to expunge below.
         if let Some(selected_folder) = &self.selected_folder {
@@ -85,10 +77,6 @@ impl ImapSession {
                 self.selected_mailbox = Some(mailbox);
                 Ok(NewlySelected::Yes)
             }
-            Err(async_imap::error::Error::ConnectionLost) => Err(Error::ConnectionLost),
-            Err(async_imap::error::Error::Validate(_)) => {
-                Err(Error::BadFolderName(folder.to_string()))
-            }
             Err(async_imap::error::Error::No(response)) => {
                 Err(Error::NoFolder(folder.to_string(), response))
             }
@@ -128,13 +116,14 @@ impl ImapSession {
     /// When selecting a folder for the first time, sets the uid_next to the current
     /// mailbox.uid_next so that no old emails are fetched.
     ///
-    /// Returns Result<new_emails> (i.e. whether new emails arrived),
-    /// if in doubt, returns new_emails=true so emails are fetched.
+    /// Updates `self.new_mail` if folder was previously unselected
+    /// and new mails are detected after selecting,
+    /// i.e. UIDNEXT advanced while the folder was closed.
     pub(crate) async fn select_with_uidvalidity(
         &mut self,
         context: &Context,
         folder: &str,
-    ) -> Result<bool> {
+    ) -> Result<()> {
         let newly_selected = self
             .select_or_create_folder(context, folder)
             .await
@@ -191,28 +180,26 @@ impl ImapSession {
         mailbox.uid_next = new_uid_next;
 
         if new_uid_validity == old_uid_validity {
-            let new_emails = if newly_selected == NewlySelected::No {
-                // The folder was not newly selected i.e. no SELECT command was run. This means that mailbox.uid_next
-                // was not updated and may contain an incorrect value. So, just return true so that
-                // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
-                // new messages is only one command, just as a SELECT command)
-                true
-            } else if let Some(new_uid_next) = new_uid_next {
-                if new_uid_next < old_uid_next {
-                    warn!(
-                        context,
-                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
-                    );
-                    set_uid_next(context, folder, new_uid_next).await?;
-                    context.schedule_resync().await?;
-                }
-                new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
-            } else {
-                // We have no UIDNEXT and if in doubt, return true.
-                true
-            };
+            if newly_selected == NewlySelected::Yes {
+                if let Some(new_uid_next) = new_uid_next {
+                    if new_uid_next < old_uid_next {
+                        warn!(
+                            context,
+                            "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
+                        );
+                        set_uid_next(context, folder, new_uid_next).await?;
+                        context.schedule_resync().await?;
+                    }
 
-            return Ok(new_emails);
+                    // If UIDNEXT changed, there are new emails.
+                    self.new_mail |= new_uid_next != old_uid_next;
+                } else {
+                    warn!(context, "Folder {folder} was just selected but we failed to determine UIDNEXT, assume that it has new mail.");
+                    self.new_mail = true;
+                }
+            }
+
+            return Ok(());
         }
 
         // UIDVALIDITY is modified, reset highest seen MODSEQ.
@@ -223,6 +210,7 @@ impl ImapSession {
         let new_uid_next = new_uid_next.unwrap_or_default();
         set_uid_next(context, folder, new_uid_next).await?;
         set_uidvalidity(context, folder, new_uid_validity).await?;
+        self.new_mail = true;
 
         // Collect garbage entries in `imap` table.
         context
@@ -245,7 +233,7 @@ impl ImapSession {
             old_uid_next,
             old_uid_validity,
         );
-        Ok(false)
+        Ok(())
     }
 }
 

src/imap/session.rs

@@ -24,6 +24,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE
                               FROM \
                               IN-REPLY-TO REFERENCES \
                               CHAT-VERSION \
+                              AUTO-SUBMITTED \
                               AUTOCRYPT-SETUP-MESSAGE\
                               )])";
 
@@ -40,6 +41,11 @@ pub(crate) struct Session {
     pub selected_mailbox: Option<Mailbox>,
 
     pub selected_folder_needs_expunge: bool,
+
+    /// True if currently selected folder has new messages.
+    ///
+    /// Should be false if no folder is currently selected.
+    pub new_mail: bool,
 }
 
 impl Deref for Session {
@@ -67,6 +73,7 @@ impl Session {
             selected_folder: None,
             selected_mailbox: None,
             selected_folder_needs_expunge: false,
+            new_mail: false,
         }
     }
 

src/imex.rs

@@ -1,41 +1,35 @@
 //! # Import/export module.
 
-use std::any::Any;
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 
 use ::pgp::types::KeyTrait;
 use anyhow::{bail, ensure, format_err, Context as _, Result};
 use deltachat_contact_tools::EmailAddress;
-use futures::StreamExt;
+use futures::TryStreamExt;
 use futures_lite::FutureExt;
-use rand::{thread_rng, Rng};
+
 use tokio::fs::{self, File};
 use tokio_tar::Archive;
 
-use crate::blob::{BlobDirContents, BlobObject};
-use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
-use crate::config::Config;
-use crate::contact::ContactId;
+use crate::blob::BlobDirContents;
+use crate::chat::{self, delete_and_reset_all_device_msgs};
 use crate::context::Context;
 use crate::e2ee;
 use crate::events::EventType;
-use crate::key::{
-    self, load_self_secret_key, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey,
-};
+use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::log::LogExt;
-use crate::message::{Message, MsgId, Viewtype};
-use crate::mimeparser::SystemMessage;
-use crate::param::Param;
+use crate::message::{Message, Viewtype};
 use crate::pgp;
 use crate::sql;
-use crate::stock_str;
 use crate::tools::{
-    create_folder, delete_file, get_filesuffix_lc, open_file_std, read_file, time, write_file,
+    create_folder, delete_file, get_filesuffix_lc, read_file, time, write_file, TempPathGuard,
 };
 
+mod key_transfer;
 mod transfer;
 
+pub use key_transfer::{continue_key_transfer, initiate_key_transfer};
 pub use transfer::{get_backup, BackupProvider};
 
 // Name of the database file in the backup.
@@ -47,12 +41,13 @@ pub(crate) const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 #[repr(u32)]
 pub enum ImexMode {
     /// Export all private keys and all public keys of the user to the
-    /// directory given as `path`.  The default key is written to the files `public-key-default.asc`
-    /// and `private-key-default.asc`, if there are more keys, they are written to files as
-    /// `public-key-<id>.asc` and `private-key-<id>.asc`
+    /// directory given as `path`. The default key is written to the files
+    /// `{public,private}-key-<addr>-default-<fingerprint>.asc`, if there are more keys, they are
+    /// written to files as `{public,private}-key-<addr>-<id>-<fingerprint>.asc`.
     ExportSelfKeys = 1,
 
-    /// Import private keys found in the directory given as `path`.
+    /// Import private keys found in `path` if it is a directory, otherwise import a private key
+    /// from `path`.
     /// The last imported key is made the default keys unless its name contains the string `legacy`.
     /// Public keys are not imported.
     ImportSelfKeys = 2,
@@ -141,117 +136,6 @@ pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
     }
 }
 
-/// Initiates key transfer via Autocrypt Setup Message.
-///
-/// Returns setup code.
-pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
-    let setup_code = create_setup_code(context);
-    /* this may require a keypair to be created. this may take a second ... */
-    let setup_file_content = render_setup_file(context, &setup_code).await?;
-    /* encrypting may also take a while ... */
-    let setup_file_blob = BlobObject::create(
-        context,
-        "autocrypt-setup-message.html",
-        setup_file_content.as_bytes(),
-    )
-    .await?;
-
-    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
-    let mut msg = Message {
-        viewtype: Viewtype::File,
-        ..Default::default()
-    };
-    msg.param.set(Param::File, setup_file_blob.as_name());
-    msg.subject = stock_str::ac_setup_msg_subject(context).await;
-    msg.param
-        .set(Param::MimeType, "application/autocrypt-setup");
-    msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.force_plaintext();
-    msg.param.set_int(Param::SkipAutocrypt, 1);
-
-    chat::send_msg(context, chat_id, &mut msg).await?;
-    // no maybe_add_bcc_self_device_msg() here.
-    // the ui shows the dialog with the setup code on this device,
-    // it would be too much noise to have two things popping up at the same time.
-    // maybe_add_bcc_self_device_msg() is called on the other device
-    // once the transfer is completed.
-    Ok(setup_code)
-}
-
-/// Renders HTML body of a setup file message.
-///
-/// The `passphrase` must be at least 2 characters long.
-pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
-        passphrase_begin
-    } else {
-        bail!("Passphrase must be at least 2 chars long.");
-    };
-    let private_key = load_self_secret_key(context).await?;
-    let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await? {
-        false => None,
-        true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
-    };
-    let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(passphrase, private_key_asc.as_bytes())
-        .await?
-        .replace('\n', "\r\n");
-
-    let replacement = format!(
-        concat!(
-            "-----BEGIN PGP MESSAGE-----\r\n",
-            "Passphrase-Format: numeric9x4\r\n",
-            "Passphrase-Begin: {}"
-        ),
-        passphrase_begin
-    );
-    let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
-
-    let msg_subj = stock_str::ac_setup_msg_subject(context).await;
-    let msg_body = stock_str::ac_setup_msg_body(context).await;
-    let msg_body_html = msg_body.replace('\r', "").replace('\n', "<br>");
-    Ok(format!(
-        concat!(
-            "<!DOCTYPE html>\r\n",
-            "<html>\r\n",
-            "  <head>\r\n",
-            "    <title>{}</title>\r\n",
-            "  </head>\r\n",
-            "  <body>\r\n",
-            "    <h1>{}</h1>\r\n",
-            "    <p>{}</p>\r\n",
-            "    <pre>\r\n{}\r\n</pre>\r\n",
-            "  </body>\r\n",
-            "</html>\r\n"
-        ),
-        msg_subj, msg_subj, msg_body_html, pgp_msg
-    ))
-}
-
-/// Creates a new setup code for Autocrypt Setup Message.
-pub fn create_setup_code(_context: &Context) -> String {
-    let mut random_val: u16;
-    let mut rng = thread_rng();
-    let mut ret = String::new();
-
-    for i in 0..9 {
-        loop {
-            random_val = rng.gen();
-            if random_val as usize <= 60000 {
-                break;
-            }
-        }
-        random_val = (random_val as usize % 10000) as u16;
-        ret += &format!(
-            "{}{:04}",
-            if 0 != i { "-" } else { "" },
-            random_val as usize
-        );
-    }
-
-    ret
-}
-
 async fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
     if !context.sql.get_raw_config_bool("bcc_self").await? {
         let mut msg = Message::new(Viewtype::Text);
@@ -265,36 +149,6 @@ async fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
     Ok(())
 }
 
-/// Continue key transfer via Autocrypt Setup Message.
-///
-/// `msg_id` is the ID of the received Autocrypt Setup Message.
-/// `setup_code` is the code entered by the user.
-pub async fn continue_key_transfer(
-    context: &Context,
-    msg_id: MsgId,
-    setup_code: &str,
-) -> Result<()> {
-    ensure!(!msg_id.is_special(), "wrong id");
-
-    let msg = Message::load_from_db(context, msg_id).await?;
-    ensure!(
-        msg.is_setupmessage(),
-        "Message is no Autocrypt Setup Message."
-    );
-
-    if let Some(filename) = msg.get_file(context) {
-        let file = open_file_std(context, filename)?;
-        let sc = normalize_setup_code(setup_code);
-        let armored_key = decrypt_setup_file(&sc, file).await?;
-        set_self_key(context, &armored_key, true).await?;
-        maybe_add_bcc_self_device_msg(context).await?;
-
-        Ok(())
-    } else {
-        bail!("Message is no Autocrypt Setup Message.");
-    }
-}
-
 async fn set_self_key(context: &Context, armored: &str, set_default: bool) -> Result<()> {
     // try hard to only modify key-state
     let (private_key, header) = SignedSecretKey::from_asc(armored)?;
@@ -345,29 +199,6 @@ async fn set_self_key(context: &Context, armored: &str, set_default: bool) -> Re
     Ok(())
 }
 
-async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
-    passphrase: &str,
-    file: T,
-) -> Result<String> {
-    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
-    let plain_text = std::string::String::from_utf8(plain_bytes)?;
-
-    Ok(plain_text)
-}
-
-fn normalize_setup_code(s: &str) -> String {
-    let mut out = String::new();
-    for c in s.chars() {
-        if c.is_ascii_digit() {
-            out.push(c);
-            if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
-                out += "-"
-            }
-        }
-    }
-    out
-}
-
 async fn imex_inner(
     context: &Context,
     what: ImexMode,
@@ -438,51 +269,126 @@ async fn import_backup(
         context.get_dbfile().display()
     );
 
+    import_backup_stream(context, backup_file, file_size, passphrase).await?;
+    Ok(())
+}
+
+/// Imports backup by reading a tar file from a stream.
+///
+/// `file_size` is used to calculate the progress
+/// and emit progress events.
+/// Ideally it is the sum of the entry
+/// sizes without the header overhead,
+/// but can be estimated as tar file size
+/// in which case the progress is underestimated
+/// and may not reach 99.9% by the end of import.
+/// Underestimating is better than
+/// overestimating because the progress
+/// jumps to 100% instead of getting stuck at 99.9%
+/// for some time.
+pub(crate) async fn import_backup_stream<R: tokio::io::AsyncRead + Unpin>(
+    context: &Context,
+    backup_file: R,
+    file_size: u64,
+    passphrase: String,
+) -> Result<()> {
+    import_backup_stream_inner(context, backup_file, file_size, passphrase)
+        .await
+        .0
+}
+
+async fn import_backup_stream_inner<R: tokio::io::AsyncRead + Unpin>(
+    context: &Context,
+    backup_file: R,
+    file_size: u64,
+    passphrase: String,
+) -> (Result<()>,) {
     let mut archive = Archive::new(backup_file);
 
-    let mut entries = archive.entries()?;
-    let mut last_progress = 0;
-    while let Some(file) = entries.next().await {
-        let f = &mut file?;
-
-        let current_pos = f.raw_file_position();
-        let progress = 1000 * current_pos / file_size;
-        if progress != last_progress && progress > 10 && progress < 1000 {
-            // We already emitted ImexProgress(10) above
+    let mut entries = match archive.entries() {
+        Ok(entries) => entries,
+        Err(e) => return (Err(e).context("Failed to get archive entries"),),
+    };
+    let mut blobs = Vec::new();
+    // We already emitted ImexProgress(10) above
+    let mut last_progress = 10;
+    const PROGRESS_MIGRATIONS: u128 = 999;
+    let mut total_size: u64 = 0;
+    let mut res: Result<()> = loop {
+        let mut f = match entries.try_next().await {
+            Ok(Some(f)) => f,
+            Ok(None) => break Ok(()),
+            Err(e) => break Err(e).context("Failed to get next entry"),
+        };
+        total_size += match f.header().entry_size() {
+            Ok(size) => size,
+            Err(e) => break Err(e).context("Failed to get entry size"),
+        };
+        let max = PROGRESS_MIGRATIONS - 1;
+        let progress = std::cmp::min(
+            max * u128::from(total_size) / std::cmp::max(u128::from(file_size), 1),
+            max,
+        );
+        if progress > last_progress {
             context.emit_event(EventType::ImexProgress(progress as usize));
             last_progress = progress;
         }
 
-        if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
-            // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
-            f.unpack_in(context.get_blobdir()).await?;
-            let unpacked_database = context.get_blobdir().join(DBFILE_BACKUP_NAME);
-            context
-                .sql
-                .import(&unpacked_database, passphrase.clone())
-                .await
-                .context("cannot import unpacked database")?;
-            fs::remove_file(unpacked_database)
-                .await
-                .context("cannot remove unpacked database")?;
-        } else {
-            // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
-            f.unpack_in(context.get_blobdir()).await?;
-            let from_path = context.get_blobdir().join(f.path()?);
-            if from_path.is_file() {
-                if let Some(name) = from_path.file_name() {
-                    fs::rename(&from_path, context.get_blobdir().join(name)).await?;
-                } else {
-                    warn!(context, "No file name");
+        let path = match f.path() {
+            Ok(path) => path.to_path_buf(),
+            Err(e) => break Err(e).context("Failed to get entry path"),
+        };
+        if let Err(e) = f.unpack_in(context.get_blobdir()).await {
+            break Err(e).context("Failed to unpack file");
+        }
+        if path.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
+            continue;
+        }
+        // async_tar unpacked to $BLOBDIR/BLOBS_BACKUP_NAME/, so we move the file afterwards.
+        let from_path = context.get_blobdir().join(&path);
+        if from_path.is_file() {
+            if let Some(name) = from_path.file_name() {
+                let to_path = context.get_blobdir().join(name);
+                if let Err(e) = fs::rename(&from_path, &to_path).await {
+                    blobs.push(from_path);
+                    break Err(e).context("Failed to move file to blobdir");
                 }
+                blobs.push(to_path);
+            } else {
+                warn!(context, "No file name");
             }
         }
+    };
+    if res.is_err() {
+        for blob in blobs {
+            fs::remove_file(&blob).await.log_err(context).ok();
+        }
     }
 
-    context.sql.run_migrations(context).await?;
-    delete_and_reset_all_device_msgs(context).await?;
-
-    Ok(())
+    let unpacked_database = context.get_blobdir().join(DBFILE_BACKUP_NAME);
+    if res.is_ok() {
+        res = context
+            .sql
+            .import(&unpacked_database, passphrase.clone())
+            .await
+            .context("cannot import unpacked database");
+    }
+    fs::remove_file(unpacked_database)
+        .await
+        .context("cannot remove unpacked database")
+        .log_err(context)
+        .ok();
+    if res.is_ok() {
+        context.emit_event(EventType::ImexProgress(PROGRESS_MIGRATIONS as usize));
+        res = context.sql.run_migrations(context).await;
+    }
+    if res.is_ok() {
+        delete_and_reset_all_device_msgs(context)
+            .await
+            .log_err(context)
+            .ok();
+    }
+    (res,)
 }
 
 /*******************************************************************************
@@ -530,8 +436,8 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
     let now = time();
     let self_addr = context.get_primary_self_addr().await?;
     let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, &self_addr, now)?;
-    let _d1 = DeleteOnDrop(temp_db_path.clone());
-    let _d2 = DeleteOnDrop(temp_path.clone());
+    let temp_db_path = TempPathGuard::new(temp_db_path);
+    let temp_path = TempPathGuard::new(temp_path);
 
     export_database(context, &temp_db_path, passphrase, now)
         .await
@@ -544,52 +450,40 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
         dest_path.display(),
     );
 
-    let res = export_backup_inner(context, &temp_db_path, &temp_path).await;
-
-    match &res {
-        Ok(_) => {
-            fs::rename(temp_path, &dest_path).await?;
-            context.emit_event(EventType::ImexFileWritten(dest_path));
-        }
-        Err(e) => {
-            error!(context, "backup failed: {}", e);
-        }
-    }
-
-    res
-}
-struct DeleteOnDrop(PathBuf);
-impl Drop for DeleteOnDrop {
-    fn drop(&mut self) {
-        let file = self.0.clone();
-        // Not using `tools::delete_file` here because it would send a DeletedBlobFile event
-        // Hack to avoid panic in nested runtime calls of tokio
-        std::fs::remove_file(file).ok();
-    }
+    let file = File::create(&temp_path).await?;
+    let blobdir = BlobDirContents::new(context).await?;
+    export_backup_stream(context, &temp_db_path, blobdir, file)
+        .await
+        .context("Exporting backup to file failed")?;
+    fs::rename(temp_path, &dest_path).await?;
+    context.emit_event(EventType::ImexFileWritten(dest_path));
+    Ok(())
 }
 
-async fn export_backup_inner(
-    context: &Context,
+/// Exports the database and blobs into a stream.
+pub(crate) async fn export_backup_stream<'a, W>(
+    context: &'a Context,
     temp_db_path: &Path,
-    temp_path: &Path,
-) -> Result<()> {
-    let file = File::create(temp_path).await?;
-
-    let mut builder = tokio_tar::Builder::new(file);
+    blobdir: BlobDirContents<'a>,
+    writer: W,
+) -> Result<()>
+where
+    W: tokio::io::AsyncWrite + tokio::io::AsyncWriteExt + Unpin + Send + 'static,
+{
+    let mut builder = tokio_tar::Builder::new(writer);
 
     builder
         .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME)
         .await?;
 
-    let blobdir = BlobDirContents::new(context).await?;
-    let mut last_progress = 0;
+    let mut last_progress = 10;
 
     for (i, blob) in blobdir.iter().enumerate() {
         let mut file = File::open(blob.to_abs_path()).await?;
         let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(blob.as_name());
         builder.append_file(path_in_archive, &mut file).await?;
-        let progress = 1000 * i / blobdir.len();
-        if progress != last_progress && progress > 10 && progress < 1000 {
+        let progress = std::cmp::min(1000 * i / blobdir.len(), 999);
+        if progress > last_progress {
             context.emit_event(EventType::ImexProgress(progress));
             last_progress = progress;
         }
@@ -695,12 +589,12 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
             },
         )
         .await?;
-
+    let self_addr = context.get_primary_self_addr().await?;
     for (id, public_key, private_key, is_default) in keys {
         let id = Some(id).filter(|_| is_default == 0);
 
         if let Ok(key) = public_key {
-            if let Err(err) = export_key_to_asc_file(context, dir, id, &key).await {
+            if let Err(err) = export_key_to_asc_file(context, dir, &self_addr, id, &key).await {
                 error!(context, "Failed to export public key: {:#}.", err);
                 export_errors += 1;
             }
@@ -708,7 +602,7 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
             export_errors += 1;
         }
         if let Ok(key) = private_key {
-            if let Err(err) = export_key_to_asc_file(context, dir, id, &key).await {
+            if let Err(err) = export_key_to_asc_file(context, dir, &self_addr, id, &key).await {
                 error!(context, "Failed to export private key: {:#}.", err);
                 export_errors += 1;
             }
@@ -721,46 +615,43 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
     Ok(())
 }
 
-/*******************************************************************************
- * Classic key export
- ******************************************************************************/
+/// Returns the exported key file name inside `dir`.
 async fn export_key_to_asc_file<T>(
     context: &Context,
     dir: &Path,
+    addr: &str,
     id: Option<i64>,
     key: &T,
-) -> Result<()>
+) -> Result<String>
 where
-    T: DcKey + Any,
+    T: DcKey,
 {
     let file_name = {
-        let any_key = key as &dyn Any;
-        let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
-            "public"
-        } else if any_key.downcast_ref::<SignedSecretKey>().is_some() {
-            "private"
-        } else {
-            "unknown"
+        let kind = match T::is_private() {
+            false => "public",
+            true => "private",
         };
         let id = id.map_or("default".into(), |i| i.to_string());
-        dir.join(format!("{}-key-{}.asc", kind, &id))
+        let fp = DcKey::fingerprint(key).hex();
+        format!("{kind}-key-{addr}-{id}-{fp}.asc")
     };
+    let path = dir.join(&file_name);
     info!(
         context,
-        "Exporting key {:?} to {}",
+        "Exporting key {:?} to {}.",
         key.key_id(),
-        file_name.display()
+        path.display()
     );
 
     // Delete the file if it already exists.
-    delete_file(context, &file_name).await.ok();
+    delete_file(context, &path).await.ok();
 
     let content = key.to_asc(None).into_bytes();
-    write_file(context, &file_name, &content)
+    write_file(context, &path, &content)
         .await
-        .with_context(|| format!("cannot write key to {}", file_name.display()))?;
-    context.emit_event(EventType::ImexFileWritten(file_name));
-    Ok(())
+        .with_context(|| format!("cannot write key to {}", path.display()))?;
+    context.emit_event(EventType::ImexFileWritten(path));
+    Ok(file_name)
 }
 
 /// Exports the database to *dest*, encrypted using *passphrase*.
@@ -819,92 +710,57 @@ async fn export_database(
 mod tests {
     use std::time::Duration;
 
-    use ::pgp::armor::BlockType;
     use tokio::task;
 
     use super::*;
-    use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
-    use crate::receive_imf::receive_imf;
-    use crate::stock_str::StockMessage;
-    use crate::test_utils::{alice_keypair, TestContext, TestContextManager};
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_render_setup_file() {
-        let t = TestContext::new_alice().await;
-        let msg = render_setup_file(&t, "hello").await.unwrap();
-        println!("{}", &msg);
-        // Check some substrings, indicating things got substituted.
-        assert!(msg.contains("<title>Autocrypt Setup Message</title"));
-        assert!(msg.contains("<h1>Autocrypt Setup Message</h1>"));
-        assert!(msg.contains("<p>This is the Autocrypt Setup Message used to"));
-        assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
-        assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
-        assert!(msg.contains("Passphrase-Begin: he\r\n"));
-        assert!(msg.contains("-----END PGP MESSAGE-----\r\n"));
-
-        for line in msg.rsplit_terminator('\n') {
-            assert!(line.ends_with('\r'));
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_render_setup_file_newline_replace() {
-        let t = TestContext::new_alice().await;
-        t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
-            .await
-            .unwrap();
-        let msg = render_setup_file(&t, "pw").await.unwrap();
-        println!("{}", &msg);
-        assert!(msg.contains("<p>hello<br>there</p>"));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_create_setup_code() {
-        let t = TestContext::new().await;
-        let setupcode = create_setup_code(&t);
-        assert_eq!(setupcode.len(), 44);
-        assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(9).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(14).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(19).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(24).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(29).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(34).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
-    }
+    use crate::config::Config;
+    use crate::test_utils::{alice_keypair, TestContext};
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
         let blobdir = Path::new("$BLOBDIR");
-        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+        let filename = export_key_to_asc_file(&context.ctx, blobdir, "a@b", None, &key)
             .await
-            .is_ok());
+            .unwrap();
+        assert!(filename.starts_with("public-key-a@b-default-"));
+        assert!(filename.ends_with(".asc"));
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
-        let filename = format!("{blobdir}/public-key-default.asc");
+        let filename = format!("{blobdir}/{filename}");
         let bytes = tokio::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_export_private_key_to_asc_file() {
+    async fn test_import_private_key_exported_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().secret;
         let blobdir = Path::new("$BLOBDIR");
-        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
+        let filename = export_key_to_asc_file(&context.ctx, blobdir, "a@b", None, &key)
             .await
-            .is_ok());
+            .unwrap();
+        let fingerprint = filename
+            .strip_prefix("private-key-a@b-default-")
+            .unwrap()
+            .strip_suffix(".asc")
+            .unwrap();
+        assert_eq!(fingerprint, DcKey::fingerprint(&key).hex());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
-        let filename = format!("{blobdir}/private-key-default.asc");
+        let filename = format!("{blobdir}/{filename}");
         let bytes = tokio::fs::read(&filename).await.unwrap();
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
+
+        let alice = &TestContext::new_alice().await;
+        if let Err(err) = imex(alice, ImexMode::ImportSelfKeys, Path::new(&filename), None).await {
+            panic!("got error on import: {err:#}");
+        }
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_export_and_import_key() {
+    async fn test_export_and_import_key_from_dir() {
         let export_dir = tempfile::tempdir().unwrap();
 
         let context = TestContext::new_alice().await;
@@ -930,12 +786,6 @@ mod tests {
         {
             panic!("got error on import: {err:#}");
         }
-
-        let keyfile = export_dir.path().join("private-key-default.asc");
-        let context3 = TestContext::new_alice().await;
-        if let Err(err) = imex(&context3.ctx, ImexMode::ImportSelfKeys, &keyfile, None).await {
-            panic!("got error on import: {err:#}");
-        }
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1080,137 +930,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[test]
-    fn test_normalize_setup_code() {
-        let norm = normalize_setup_code("123422343234423452346234723482349234");
-        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
-
-        let norm =
-            normalize_setup_code("\t1 2 3422343234- foo bar-- 423-45 2 34 6234723482349234      ");
-        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
-    }
-
-    /* S_EM_SETUPFILE is a AES-256 symm. encrypted setup message created by Enigmail
-    with an "encrypted session key", see RFC 4880.  The code is in S_EM_SETUPCODE */
-    const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
-    const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
-
-    // Autocrypt Setup Message payload "encrypted" with plaintext algorithm.
-    const S_PLAINTEXT_SETUPFILE: &str =
-        include_str!("../test-data/message/plaintext-autocrypt-setup.txt");
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_split_and_decrypt() {
-        let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
-        let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
-        assert_eq!(typ, BlockType::Message);
-        assert!(S_EM_SETUPCODE.starts_with(headers.get(HEADER_SETUPCODE).unwrap()));
-        assert!(!headers.contains_key(HEADER_AUTOCRYPT));
-
-        assert!(!base64.is_empty());
-
-        let setup_file = S_EM_SETUPFILE.to_string();
-        let decrypted =
-            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
-                .await
-                .unwrap();
-
-        let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
-
-        assert_eq!(typ, BlockType::PrivateKey);
-        assert_eq!(headers.get(HEADER_AUTOCRYPT), Some(&"mutual".to_string()));
-        assert!(!headers.contains_key(HEADER_SETUPCODE));
-    }
-
-    /// Tests that Autocrypt Setup Message encrypted with "plaintext" algorithm cannot be
-    /// decrypted.
-    ///
-    /// According to <https://datatracker.ietf.org/doc/html/rfc4880#section-13.4>
-    /// "Implementations MUST NOT use plaintext in Symmetrically Encrypted Data packets".
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_decrypt_plaintext_autocrypt_setup_message() {
-        let setup_file = S_PLAINTEXT_SETUPFILE.to_string();
-        let incorrect_setupcode = "0000-0000-0000-0000-0000-0000-0000-0000-0000";
-        assert!(decrypt_setup_file(
-            incorrect_setupcode,
-            std::io::Cursor::new(setup_file.as_bytes()),
-        )
-        .await
-        .is_err());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_key_transfer() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        let setup_code = initiate_key_transfer(&alice).await?;
-
-        // Get Autocrypt Setup Message.
-        let sent = alice.pop_sent_msg().await;
-
-        // Alice sets up a second device.
-        let alice2 = TestContext::new().await;
-        alice2.set_name("alice2");
-        alice2.configure_addr("alice@example.org").await;
-        alice2.recv_msg(&sent).await;
-        let msg = alice2.get_last_msg().await;
-        assert!(msg.is_setupmessage());
-
-        // Send a message that cannot be decrypted because the keys are
-        // not synchronized yet.
-        let sent = alice2.send_text(msg.chat_id, "Test").await;
-        let trashed_message = alice.recv_msg_opt(&sent).await;
-        assert!(trashed_message.is_none());
-        assert_ne!(alice.get_last_msg().await.get_text(), "Test");
-
-        // Transfer the key.
-        continue_key_transfer(&alice2, msg.id, &setup_code).await?;
-
-        // Alice sends a message to self from the new device.
-        let sent = alice2.send_text(msg.chat_id, "Test").await;
-        alice.recv_msg(&sent).await;
-        assert_eq!(alice.get_last_msg().await.get_text(), "Test");
-
-        Ok(())
-    }
-
-    /// Tests that Autocrypt Setup Messages is only clickable if it is self-sent.
-    /// This prevents Bob from tricking Alice into changing the key
-    /// by sending her an Autocrypt Setup Message as long as Alice's server
-    /// does not allow to forge the `From:` header.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_key_transfer_non_self_sent() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
-
-        let _setup_code = initiate_key_transfer(&alice).await?;
-
-        // Get Autocrypt Setup Message.
-        let sent = alice.pop_sent_msg().await;
-
-        let rcvd = bob.recv_msg(&sent).await;
-        assert!(!rcvd.is_setupmessage());
-
-        Ok(())
-    }
-
-    /// Tests reception of Autocrypt Setup Message from K-9 6.802.
-    ///
-    /// Unlike Autocrypt Setup Message sent by Delta Chat,
-    /// this message does not contain `Autocrypt-Prefer-Encrypt` header.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_key_transfer_k_9() -> Result<()> {
-        let t = &TestContext::new().await;
-        t.configure_addr("autocrypt@nine.testrun.org").await;
-
-        let raw = include_bytes!("../test-data/message/k-9-autocrypt-setup-message.eml");
-        let received = receive_imf(t, raw, false).await?.unwrap();
-
-        let setup_code = "0655-9868-8252-5455-4232-5158-1237-5333-2638";
-        continue_key_transfer(t, *received.msg_ids.last().unwrap(), setup_code).await?;
-
-        Ok(())
-    }
 }

src/imex/key_transfer.rs

@@ -0,0 +1,372 @@
+//! # Key transfer via Autocrypt Setup Message.
+use rand::{thread_rng, Rng};
+
+use anyhow::{bail, ensure, Result};
+
+use crate::blob::BlobObject;
+use crate::chat::{self, ChatId};
+use crate::config::Config;
+use crate::contact::ContactId;
+use crate::context::Context;
+use crate::imex::maybe_add_bcc_self_device_msg;
+use crate::imex::set_self_key;
+use crate::key::{load_self_secret_key, DcKey};
+use crate::message::{Message, MsgId, Viewtype};
+use crate::mimeparser::SystemMessage;
+use crate::param::Param;
+use crate::pgp;
+use crate::stock_str;
+use crate::tools::open_file_std;
+
+/// Initiates key transfer via Autocrypt Setup Message.
+///
+/// Returns setup code.
+pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
+    let setup_code = create_setup_code(context);
+    /* this may require a keypair to be created. this may take a second ... */
+    let setup_file_content = render_setup_file(context, &setup_code).await?;
+    /* encrypting may also take a while ... */
+    let setup_file_blob = BlobObject::create(
+        context,
+        "autocrypt-setup-message.html",
+        setup_file_content.as_bytes(),
+    )
+    .await?;
+
+    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
+    let mut msg = Message {
+        viewtype: Viewtype::File,
+        ..Default::default()
+    };
+    msg.param.set(Param::File, setup_file_blob.as_name());
+    msg.subject = stock_str::ac_setup_msg_subject(context).await;
+    msg.param
+        .set(Param::MimeType, "application/autocrypt-setup");
+    msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
+    msg.force_plaintext();
+    msg.param.set_int(Param::SkipAutocrypt, 1);
+
+    chat::send_msg(context, chat_id, &mut msg).await?;
+    // no maybe_add_bcc_self_device_msg() here.
+    // the ui shows the dialog with the setup code on this device,
+    // it would be too much noise to have two things popping up at the same time.
+    // maybe_add_bcc_self_device_msg() is called on the other device
+    // once the transfer is completed.
+    Ok(setup_code)
+}
+
+/// Continue key transfer via Autocrypt Setup Message.
+///
+/// `msg_id` is the ID of the received Autocrypt Setup Message.
+/// `setup_code` is the code entered by the user.
+pub async fn continue_key_transfer(
+    context: &Context,
+    msg_id: MsgId,
+    setup_code: &str,
+) -> Result<()> {
+    ensure!(!msg_id.is_special(), "wrong id");
+
+    let msg = Message::load_from_db(context, msg_id).await?;
+    ensure!(
+        msg.is_setupmessage(),
+        "Message is no Autocrypt Setup Message."
+    );
+
+    if let Some(filename) = msg.get_file(context) {
+        let file = open_file_std(context, filename)?;
+        let sc = normalize_setup_code(setup_code);
+        let armored_key = decrypt_setup_file(&sc, file).await?;
+        set_self_key(context, &armored_key, true).await?;
+        maybe_add_bcc_self_device_msg(context).await?;
+
+        Ok(())
+    } else {
+        bail!("Message is no Autocrypt Setup Message.");
+    }
+}
+
+/// Renders HTML body of a setup file message.
+///
+/// The `passphrase` must be at least 2 characters long.
+pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
+    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
+        passphrase_begin
+    } else {
+        bail!("Passphrase must be at least 2 chars long.");
+    };
+    let private_key = load_self_secret_key(context).await?;
+    let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await? {
+        false => None,
+        true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
+    };
+    let private_key_asc = private_key.to_asc(ac_headers);
+    let encr = pgp::symm_encrypt(passphrase, private_key_asc.as_bytes())
+        .await?
+        .replace('\n', "\r\n");
+
+    let replacement = format!(
+        concat!(
+            "-----BEGIN PGP MESSAGE-----\r\n",
+            "Passphrase-Format: numeric9x4\r\n",
+            "Passphrase-Begin: {}"
+        ),
+        passphrase_begin
+    );
+    let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
+
+    let msg_subj = stock_str::ac_setup_msg_subject(context).await;
+    let msg_body = stock_str::ac_setup_msg_body(context).await;
+    let msg_body_html = msg_body.replace('\r', "").replace('\n', "<br>");
+    Ok(format!(
+        concat!(
+            "<!DOCTYPE html>\r\n",
+            "<html>\r\n",
+            "  <head>\r\n",
+            "    <title>{}</title>\r\n",
+            "  </head>\r\n",
+            "  <body>\r\n",
+            "    <h1>{}</h1>\r\n",
+            "    <p>{}</p>\r\n",
+            "    <pre>\r\n{}\r\n</pre>\r\n",
+            "  </body>\r\n",
+            "</html>\r\n"
+        ),
+        msg_subj, msg_subj, msg_body_html, pgp_msg
+    ))
+}
+
+/// Creates a new setup code for Autocrypt Setup Message.
+fn create_setup_code(_context: &Context) -> String {
+    let mut random_val: u16;
+    let mut rng = thread_rng();
+    let mut ret = String::new();
+
+    for i in 0..9 {
+        loop {
+            random_val = rng.gen();
+            if random_val as usize <= 60000 {
+                break;
+            }
+        }
+        random_val = (random_val as usize % 10000) as u16;
+        ret += &format!(
+            "{}{:04}",
+            if 0 != i { "-" } else { "" },
+            random_val as usize
+        );
+    }
+
+    ret
+}
+
+async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
+    passphrase: &str,
+    file: T,
+) -> Result<String> {
+    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
+    let plain_text = std::string::String::from_utf8(plain_bytes)?;
+
+    Ok(plain_text)
+}
+
+fn normalize_setup_code(s: &str) -> String {
+    let mut out = String::new();
+    for c in s.chars() {
+        if c.is_ascii_digit() {
+            out.push(c);
+            if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
+                out += "-"
+            }
+        }
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
+    use crate::receive_imf::receive_imf;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::{TestContext, TestContextManager};
+    use ::pgp::armor::BlockType;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_render_setup_file() {
+        let t = TestContext::new_alice().await;
+        let msg = render_setup_file(&t, "hello").await.unwrap();
+        println!("{}", &msg);
+        // Check some substrings, indicating things got substituted.
+        assert!(msg.contains("<title>Autocrypt Setup Message</title"));
+        assert!(msg.contains("<h1>Autocrypt Setup Message</h1>"));
+        assert!(msg.contains("<p>This is the Autocrypt Setup Message used to"));
+        assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
+        assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
+        assert!(msg.contains("Passphrase-Begin: he\r\n"));
+        assert!(msg.contains("-----END PGP MESSAGE-----\r\n"));
+
+        for line in msg.rsplit_terminator('\n') {
+            assert!(line.ends_with('\r'));
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_render_setup_file_newline_replace() {
+        let t = TestContext::new_alice().await;
+        t.set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+            .await
+            .unwrap();
+        let msg = render_setup_file(&t, "pw").await.unwrap();
+        println!("{}", &msg);
+        assert!(msg.contains("<p>hello<br>there</p>"));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_create_setup_code() {
+        let t = TestContext::new().await;
+        let setupcode = create_setup_code(&t);
+        assert_eq!(setupcode.len(), 44);
+        assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(9).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(14).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(19).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(24).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(29).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(34).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
+    }
+
+    #[test]
+    fn test_normalize_setup_code() {
+        let norm = normalize_setup_code("123422343234423452346234723482349234");
+        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
+
+        let norm =
+            normalize_setup_code("\t1 2 3422343234- foo bar-- 423-45 2 34 6234723482349234      ");
+        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
+    }
+
+    /* S_EM_SETUPFILE is a AES-256 symm. encrypted setup message created by Enigmail
+    with an "encrypted session key", see RFC 4880.  The code is in S_EM_SETUPCODE */
+    const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
+    const S_EM_SETUPFILE: &str = include_str!("../../test-data/message/stress.txt");
+
+    // Autocrypt Setup Message payload "encrypted" with plaintext algorithm.
+    const S_PLAINTEXT_SETUPFILE: &str =
+        include_str!("../../test-data/message/plaintext-autocrypt-setup.txt");
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_split_and_decrypt() {
+        let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
+        let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
+        assert_eq!(typ, BlockType::Message);
+        assert!(S_EM_SETUPCODE.starts_with(headers.get(HEADER_SETUPCODE).unwrap()));
+        assert!(!headers.contains_key(HEADER_AUTOCRYPT));
+
+        assert!(!base64.is_empty());
+
+        let setup_file = S_EM_SETUPFILE.to_string();
+        let decrypted =
+            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
+                .await
+                .unwrap();
+
+        let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
+
+        assert_eq!(typ, BlockType::PrivateKey);
+        assert_eq!(headers.get(HEADER_AUTOCRYPT), Some(&"mutual".to_string()));
+        assert!(!headers.contains_key(HEADER_SETUPCODE));
+    }
+
+    /// Tests that Autocrypt Setup Message encrypted with "plaintext" algorithm cannot be
+    /// decrypted.
+    ///
+    /// According to <https://datatracker.ietf.org/doc/html/rfc4880#section-13.4>
+    /// "Implementations MUST NOT use plaintext in Symmetrically Encrypted Data packets".
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decrypt_plaintext_autocrypt_setup_message() {
+        let setup_file = S_PLAINTEXT_SETUPFILE.to_string();
+        let incorrect_setupcode = "0000-0000-0000-0000-0000-0000-0000-0000-0000";
+        assert!(decrypt_setup_file(
+            incorrect_setupcode,
+            std::io::Cursor::new(setup_file.as_bytes()),
+        )
+        .await
+        .is_err());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_key_transfer() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let setup_code = initiate_key_transfer(&alice).await?;
+
+        // Get Autocrypt Setup Message.
+        let sent = alice.pop_sent_msg().await;
+
+        // Alice sets up a second device.
+        let alice2 = TestContext::new().await;
+        alice2.set_name("alice2");
+        alice2.configure_addr("alice@example.org").await;
+        alice2.recv_msg(&sent).await;
+        let msg = alice2.get_last_msg().await;
+        assert!(msg.is_setupmessage());
+
+        // Send a message that cannot be decrypted because the keys are
+        // not synchronized yet.
+        let sent = alice2.send_text(msg.chat_id, "Test").await;
+        let trashed_message = alice.recv_msg_opt(&sent).await;
+        assert!(trashed_message.is_none());
+        assert_ne!(alice.get_last_msg().await.get_text(), "Test");
+
+        // Transfer the key.
+        continue_key_transfer(&alice2, msg.id, &setup_code).await?;
+
+        // Alice sends a message to self from the new device.
+        let sent = alice2.send_text(msg.chat_id, "Test").await;
+        alice.recv_msg(&sent).await;
+        assert_eq!(alice.get_last_msg().await.get_text(), "Test");
+
+        Ok(())
+    }
+
+    /// Tests that Autocrypt Setup Messages is only clickable if it is self-sent.
+    /// This prevents Bob from tricking Alice into changing the key
+    /// by sending her an Autocrypt Setup Message as long as Alice's server
+    /// does not allow to forge the `From:` header.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_key_transfer_non_self_sent() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
+        let _setup_code = initiate_key_transfer(&alice).await?;
+
+        // Get Autocrypt Setup Message.
+        let sent = alice.pop_sent_msg().await;
+
+        let rcvd = bob.recv_msg(&sent).await;
+        assert!(!rcvd.is_setupmessage());
+
+        Ok(())
+    }
+
+    /// Tests reception of Autocrypt Setup Message from K-9 6.802.
+    ///
+    /// Unlike Autocrypt Setup Message sent by Delta Chat,
+    /// this message does not contain `Autocrypt-Prefer-Encrypt` header.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_key_transfer_k_9() -> Result<()> {
+        let t = &TestContext::new().await;
+        t.configure_addr("autocrypt@nine.testrun.org").await;
+
+        let raw = include_bytes!("../../test-data/message/k-9-autocrypt-setup-message.eml");
+        let received = receive_imf(t, raw, false).await?.unwrap();
+
+        let setup_code = "0655-9868-8252-5455-4232-5158-1237-5333-2638";
+        continue_key_transfer(t, *received.msg_ids.last().unwrap(), setup_code).await?;
+
+        Ok(())
+    }
+}

src/imex/transfer.rs

@@ -1,17 +1,12 @@
 //! Transfer a backup to an other device.
 //!
-//! This module provides support for using n0's iroh tool to initiate transfer of a backup
-//! to another device using a QR code.
-//!
-//! Using the iroh terminology there are two parties to this:
+//! This module provides support for using [iroh](https://iroh.computer/)
+//! to initiate transfer of a backup to another device using a QR code.
 //!
+//! There are two parties to this:
 //! - The *Provider*, which starts a server and listens for connections.
 //! - The *Getter*, which connects to the server and retrieves the data.
 //!
-//! Iroh is designed around the idea of verifying hashes, the downloads are verified as
-//! they are retrieved.  The entire transfer is initiated by requesting the data of a single
-//! root hash.
-//!
 //! Both the provider and the getter are authenticated:
 //!
 //! - The provider is known by its *peer ID*.
@@ -21,24 +16,30 @@
 //! Both these are transferred in the QR code offered to the getter.  This ensures that the
 //! getter can not connect to an impersonated provider and the provider does not offer the
 //! download to an impersonated getter.
+//!
+//! Protocol starts by getter opening a bidirectional QUIC stream
+//! to the provider and sending authentication token.
+//! Provider verifies received authentication token,
+//! sends the size of all files in a backup (database and all blobs)
+//! as an unsigned 64-bit big endian integer and streams the backup in tar format.
+//! Getter receives the backup and acknowledges successful reception
+//! by sending a single byte.
+//! Provider closes the endpoint after receiving an acknowledgment.
 
 use std::future::Future;
-use std::net::Ipv4Addr;
-use std::ops::Deref;
-use std::path::{Path, PathBuf};
 use std::pin::Pin;
+use std::sync::Arc;
 use std::task::Poll;
 
 use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result};
-use async_channel::Receiver;
 use futures_lite::StreamExt;
-use iroh::blobs::Collection;
-use iroh::get::DataStream;
-use iroh::progress::ProgressEmitter;
-use iroh::protocol::AuthToken;
-use iroh::provider::{DataSource, Event, Provider, Ticket};
-use iroh::Hash;
-use iroh_old as iroh;
+use iroh_net::relay::RelayMode;
+use iroh_net::Endpoint;
+use iroh_old;
+use iroh_old::blobs::Collection;
+use iroh_old::get::DataStream;
+use iroh_old::progress::ProgressEmitter;
+use iroh_old::provider::Ticket;
 use tokio::fs::{self, File};
 use tokio::io::{self, AsyncWriteExt, BufWriter};
 use tokio::sync::broadcast::error::RecvError;
@@ -47,19 +48,22 @@ use tokio::task::{JoinHandle, JoinSet};
 use tokio_stream::wrappers::ReadDirStream;
 use tokio_util::sync::CancellationToken;
 
-use crate::blob::BlobDirContents;
 use crate::chat::{add_device_msg, delete_and_reset_all_device_msgs};
 use crate::context::Context;
+use crate::imex::BlobDirContents;
 use crate::message::{Message, Viewtype};
 use crate::qr::{self, Qr};
 use crate::stock_str::backup_transfer_msg_body;
-use crate::tools::time;
-use crate::{e2ee, EventType};
+use crate::tools::{create_id, time, TempPathGuard};
+use crate::EventType;
 
-use super::{export_database, DBFILE_BACKUP_NAME};
+use super::{export_backup_stream, export_database, import_backup_stream, DBFILE_BACKUP_NAME};
 
 const MAX_CONCURRENT_DIALS: u8 = 16;
 
+/// ALPN protocol identifier for the backup transfer protocol.
+const BACKUP_ALPN: &[u8] = b"/deltachat/backup";
+
 /// Provide or send a backup of this device.
 ///
 /// This creates a backup of the current device and starts a service which offers another
@@ -70,15 +74,21 @@ const MAX_CONCURRENT_DIALS: u8 = 16;
 ///
 /// This starts a task which acquires the global "ongoing" mutex.  If you need to stop the
 /// task use the [`Context::stop_ongoing`] mechanism.
-///
-/// The task implements [`Future`] and awaiting it will complete once a transfer has been
-/// either completed or aborted.
 #[derive(Debug)]
 pub struct BackupProvider {
-    /// The supervisor task, run by [`BackupProvider::watch_provider`].
+    /// iroh-net endpoint.
+    _endpoint: Endpoint,
+
+    /// iroh-net address.
+    node_addr: iroh_net::NodeAddr,
+
+    /// Authentication token that should be submitted
+    /// to retrieve the backup.
+    auth_token: String,
+
+    /// Handle for the task accepting backup transfer requests.
     handle: JoinHandle<Result<()>>,
-    /// The ticket to retrieve the backup collection.
-    ticket: Ticket,
+
     /// Guard to cancel the provider on drop.
     _drop_guard: tokio_util::sync::DropGuard,
 }
@@ -95,9 +105,13 @@ impl BackupProvider {
     ///
     /// [`Accounts::stop_io`]: crate::accounts::Accounts::stop_io
     pub async fn prepare(context: &Context) -> Result<Self> {
-        e2ee::ensure_secret_key_exists(context)
-            .await
-            .context("Private key not available, aborting backup export")?;
+        let relay_mode = RelayMode::Disabled;
+        let endpoint = Endpoint::builder()
+            .alpns(vec![BACKUP_ALPN.to_vec()])
+            .relay_mode(relay_mode)
+            .bind(0)
+            .await?;
+        let node_addr = endpoint.node_addr().await?;
 
         // Acquire global "ongoing" mutex.
         let cancel_token = context.alloc_ongoing().await?;
@@ -105,195 +119,153 @@ impl BackupProvider {
         let context_dir = context
             .get_blobdir()
             .parent()
-            .ok_or_else(|| anyhow!("Context dir not found"))?;
+            .context("Context dir not found")?;
         let dbfile = context_dir.join(DBFILE_BACKUP_NAME);
         if fs::metadata(&dbfile).await.is_ok() {
             fs::remove_file(&dbfile).await?;
             warn!(context, "Previous database export deleted");
         }
         let dbfile = TempPathGuard::new(dbfile);
-        let res = tokio::select! {
-            biased;
-            res = Self::prepare_inner(context, &dbfile) => {
-                match res {
-                    Ok(slf) => Ok(slf),
-                    Err(err) => {
-                        error!(context, "Failed to set up second device setup: {:#}", err);
-                        Err(err)
-                    },
-                }
-            },
-            _ = cancel_token.recv() => Err(format_err!("cancelled")),
-        };
-        let (provider, ticket) = match res {
-            Ok((provider, ticket)) => (provider, ticket),
-            Err(err) => {
-                context.free_ongoing().await;
-                return Err(err);
-            }
-        };
+
+        // Authentication token that receiver should send us to receive a backup.
+        let auth_token = create_id();
+
+        let passphrase = String::new();
+
+        export_database(context, &dbfile, passphrase, time())
+            .await
+            .context("Database export failed")?;
+        context.emit_event(EventType::ImexProgress(300));
+
         let drop_token = CancellationToken::new();
         let handle = {
             let context = context.clone();
             let drop_token = drop_token.clone();
+            let endpoint = endpoint.clone();
+            let auth_token = auth_token.clone();
             tokio::spawn(async move {
-                let res = Self::watch_provider(&context, provider, cancel_token, drop_token).await;
+                Self::accept_loop(
+                    context.clone(),
+                    endpoint,
+                    auth_token,
+                    cancel_token,
+                    drop_token,
+                    dbfile,
+                )
+                .await;
+                info!(context, "Finished accept loop.");
+
                 context.free_ongoing().await;
 
                 // Explicit drop to move the guards into this future
                 drop(paused_guard);
-                drop(dbfile);
-                res
+                Ok(())
             })
         };
         Ok(Self {
+            _endpoint: endpoint,
+            node_addr,
+            auth_token,
             handle,
-            ticket,
             _drop_guard: drop_token.drop_guard(),
         })
     }
 
-    /// Creates the provider task.
-    ///
-    /// Having this as a function makes it easier to cancel it when needed.
-    async fn prepare_inner(context: &Context, dbfile: &Path) -> Result<(Provider, Ticket)> {
-        // Generate the token up front: we also use it to encrypt the database.
-        let token = AuthToken::generate();
-        context.emit_event(SendProgress::Started.into());
-        export_database(context, dbfile, token.to_string(), time())
-            .await
-            .context("Database export failed")?;
-        context.emit_event(SendProgress::DatabaseExported.into());
+    async fn handle_connection(
+        context: Context,
+        conn: iroh_net::endpoint::Connecting,
+        auth_token: String,
+        dbfile: Arc<TempPathGuard>,
+    ) -> Result<()> {
+        let conn = conn.await?;
+        let (mut send_stream, mut recv_stream) = conn.accept_bi().await?;
 
-        // Now we can be sure IO is not running.
-        let mut files = vec![DataSource::with_name(
-            dbfile.to_owned(),
-            format!("db/{DBFILE_BACKUP_NAME}"),
-        )];
-        let blobdir = BlobDirContents::new(context).await?;
-        for blob in blobdir.iter() {
-            let path = blob.to_abs_path();
-            let name = format!("blob/{}", blob.as_file_name());
-            files.push(DataSource::with_name(path, name));
+        // Read authentication token from the stream.
+        let mut received_auth_token = vec![0u8; auth_token.len()];
+        recv_stream.read_exact(&mut received_auth_token).await?;
+        if received_auth_token.as_slice() != auth_token.as_bytes() {
+            warn!(context, "Received wrong backup authentication token.");
+            return Ok(());
         }
 
-        // Start listening.
-        let (db, hash) = iroh::provider::create_collection(files).await?;
-        context.emit_event(SendProgress::CollectionCreated.into());
-        let provider = Provider::builder(db)
-            .bind_addr((Ipv4Addr::UNSPECIFIED, 0).into())
-            .auth_token(token)
-            .spawn()?;
-        context.emit_event(SendProgress::ProviderListening.into());
-        info!(context, "Waiting for remote to connect");
-        let ticket = provider.ticket(hash)?;
-        Ok((provider, ticket))
+        info!(context, "Received valid backup authentication token.");
+
+        let blobdir = BlobDirContents::new(&context).await?;
+
+        let mut file_size = 0;
+        file_size += dbfile.metadata()?.len();
+        for blob in blobdir.iter() {
+            file_size += blob.to_abs_path().metadata()?.len()
+        }
+
+        send_stream.write_all(&file_size.to_be_bytes()).await?;
+
+        export_backup_stream(&context, &dbfile, blobdir, send_stream)
+            .await
+            .context("Failed to write backup into QUIC stream")?;
+        info!(context, "Finished writing backup into QUIC stream.");
+        let mut buf = [0u8; 1];
+        info!(context, "Waiting for acknowledgment.");
+        recv_stream.read_exact(&mut buf).await?;
+        info!(context, "Received backup reception acknowledgement.");
+        context.emit_event(EventType::ImexProgress(1000));
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = backup_transfer_msg_body(&context).await;
+        add_device_msg(&context, None, Some(&mut msg)).await?;
+
+        Ok(())
     }
 
-    /// Supervises the iroh [`Provider`], terminating it when needed.
-    ///
-    /// This will watch the provider and terminate it when:
-    ///
-    /// - A transfer is completed, successful or unsuccessful.
-    /// - An event could not be observed to protect against not knowing of a completed event.
-    /// - The ongoing process is cancelled.
-    ///
-    /// The *cancel_token* is the handle for the ongoing process mutex, when this completes
-    /// we must cancel this operation.
-    async fn watch_provider(
-        context: &Context,
-        mut provider: Provider,
-        cancel_token: Receiver<()>,
+    async fn accept_loop(
+        context: Context,
+        endpoint: Endpoint,
+        auth_token: String,
+        cancel_token: async_channel::Receiver<()>,
         drop_token: CancellationToken,
-    ) -> Result<()> {
-        let mut events = provider.subscribe();
-        let mut total_size = 0;
-        let mut current_size = 0;
-        let res = loop {
+        dbfile: TempPathGuard,
+    ) {
+        let dbfile = Arc::new(dbfile);
+        loop {
             tokio::select! {
                 biased;
-                res = &mut provider => {
-                    break res.context("BackupProvider failed");
-                },
-                maybe_event = events.recv() => {
-                    match maybe_event {
-                        Ok(event) => {
-                            match event {
-                                Event::ClientConnected { ..} => {
-                                    context.emit_event(SendProgress::ClientConnected.into());
-                                }
-                                Event::RequestReceived { .. } => {
-                                }
-                                Event::TransferCollectionStarted { total_blobs_size, .. } => {
-                                    total_size = total_blobs_size;
-                                    context.emit_event(SendProgress::TransferInProgress {
-                                        current_size,
-                                        total_size,
-                                    }.into());
-                                }
-                                Event::TransferBlobCompleted { size, .. } => {
-                                    current_size += size;
-                                    context.emit_event(SendProgress::TransferInProgress {
-                                        current_size,
-                                        total_size,
-                                    }.into());
-                                }
-                                Event::TransferCollectionCompleted { .. } => {
-                                    context.emit_event(SendProgress::TransferInProgress {
-                                        current_size: total_size,
-                                        total_size
-                                    }.into());
-                                    provider.shutdown();
-                                }
-                                Event::TransferAborted { .. } => {
-                                    provider.shutdown();
-                                    break Err(anyhow!("BackupProvider transfer aborted"));
-                                }
-                            }
-                        }
-                        Err(broadcast::error::RecvError::Closed) => {
-                            // We should never see this, provider.join() should complete
-                            // first.
-                        }
-                        Err(broadcast::error::RecvError::Lagged(_)) => {
-                            // We really shouldn't be lagging, if we did we may have missed
-                            // a completion event.
-                            provider.shutdown();
-                            break Err(anyhow!("Missed events from BackupProvider"));
+
+                conn = endpoint.accept() => {
+                    if let Some(conn) = conn {
+                        // Got a new in-progress connection.
+                        let context = context.clone();
+                        let auth_token = auth_token.clone();
+                        let dbfile = dbfile.clone();
+                        if let Err(err) = Self::handle_connection(context.clone(), conn, auth_token, dbfile).await {
+                            warn!(context, "Error while handling backup connection: {err:#}.");
+                        } else {
+                            info!(context, "Backup transfer finished successfully.");
+                            break;
                         }
+                    } else {
+                        break;
                     }
                 },
                 _ = cancel_token.recv() => {
-                    provider.shutdown();
-                    break Err(anyhow!("BackupProvider cancelled"));
-                },
+                    context.emit_event(EventType::ImexProgress(0));
+                    break;
+                }
                 _ = drop_token.cancelled() => {
-                    provider.shutdown();
-                    break Err(anyhow!("BackupProvider dropped"));
+                    context.emit_event(EventType::ImexProgress(0));
+                    break;
                 }
             }
-        };
-        match &res {
-            Ok(_) => {
-                context.emit_event(SendProgress::Completed.into());
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = backup_transfer_msg_body(context).await;
-                add_device_msg(context, None, Some(&mut msg)).await?;
-            }
-            Err(err) => {
-                error!(context, "Backup transfer failure: {err:#}");
-                context.emit_event(SendProgress::Failed.into())
-            }
         }
-        res
     }
 
     /// Returns a QR code that allows fetching this backup.
     ///
     /// This QR code can be passed to [`get_backup`] on a (different) device.
     pub fn qr(&self) -> Qr {
-        Qr::Backup {
-            ticket: self.ticket.clone(),
+        Qr::Backup2 {
+            node_addr: self.node_addr.clone(),
+
+            auth_token: self.auth_token.clone(),
         }
     }
 }
@@ -301,92 +273,14 @@ impl BackupProvider {
 impl Future for BackupProvider {
     type Output = Result<()>;
 
+    /// Waits for the backup transfer to complete.
     fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
         Pin::new(&mut self.handle).poll(cx)?
     }
 }
 
-/// A guard which will remove the path when dropped.
-///
-/// It implements [`Deref`] it it can be used as a `&Path`.
-#[derive(Debug)]
-struct TempPathGuard {
-    path: PathBuf,
-}
-
-impl TempPathGuard {
-    fn new(path: PathBuf) -> Self {
-        Self { path }
-    }
-}
-
-impl Drop for TempPathGuard {
-    fn drop(&mut self) {
-        let path = self.path.clone();
-        tokio::spawn(async move {
-            fs::remove_file(&path).await.ok();
-        });
-    }
-}
-
-impl Deref for TempPathGuard {
-    type Target = Path;
-
-    fn deref(&self) -> &Self::Target {
-        &self.path
-    }
-}
-
-/// Create [`EventType::ImexProgress`] events using readable names.
-///
-/// Plus you get warnings if you don't use all variants.
-#[derive(Debug)]
-enum SendProgress {
-    Failed,
-    Started,
-    DatabaseExported,
-    CollectionCreated,
-    ProviderListening,
-    ClientConnected,
-    TransferInProgress { current_size: u64, total_size: u64 },
-    Completed,
-}
-
-impl From<SendProgress> for EventType {
-    fn from(source: SendProgress) -> Self {
-        use SendProgress::*;
-        let num: u16 = match source {
-            Failed => 0,
-            Started => 100,
-            DatabaseExported => 300,
-            CollectionCreated => 350,
-            ProviderListening => 400,
-            ClientConnected => 450,
-            TransferInProgress {
-                current_size,
-                total_size,
-            } => {
-                // the range is 450..=950
-                450 + ((current_size as f64 / total_size as f64) * 500.).floor() as u16
-            }
-            Completed => 1000,
-        };
-        Self::ImexProgress(num.into())
-    }
-}
-
-/// Contacts a backup provider and receives the backup from it.
-///
-/// This uses a QR code to contact another instance of deltachat which is providing a backup
-/// using the [`BackupProvider`].  Once connected it will authenticate using the secrets in
-/// the QR code and retrieve the backup.
-///
-/// This is a long running operation which will only when completed.
-///
-/// Using [`Qr`] as argument is a bit odd as it only accepts one specific variant of it.  It
-/// does avoid having [`iroh::provider::Ticket`] in the primary API however, without
-/// having to revert to untyped bytes.
-pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> {
+/// Retrieves backup from a legacy backup provider using iroh 0.4.
+pub async fn get_legacy_backup(context: &Context, qr: Qr) -> Result<()> {
     ensure!(
         matches!(qr, Qr::Backup { .. }),
         "QR code for backup must be of type DCBACKUP"
@@ -412,6 +306,64 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> {
     res
 }
 
+pub async fn get_backup2(
+    context: &Context,
+    node_addr: iroh_net::NodeAddr,
+    auth_token: String,
+) -> Result<()> {
+    let relay_mode = RelayMode::Disabled;
+
+    let endpoint = Endpoint::builder().relay_mode(relay_mode).bind(0).await?;
+
+    let conn = endpoint.connect(node_addr, BACKUP_ALPN).await?;
+    let (mut send_stream, mut recv_stream) = conn.open_bi().await?;
+    info!(context, "Sending backup authentication token.");
+    send_stream.write_all(auth_token.as_bytes()).await?;
+
+    let passphrase = String::new();
+    info!(context, "Starting to read backup from the stream.");
+
+    let mut file_size_buf = [0u8; 8];
+    recv_stream.read_exact(&mut file_size_buf).await?;
+    let file_size = u64::from_be_bytes(file_size_buf);
+    import_backup_stream(context, recv_stream, file_size, passphrase)
+        .await
+        .context("Failed to import backup from QUIC stream")?;
+    info!(context, "Finished importing backup from the stream.");
+    context.emit_event(EventType::ImexProgress(1000));
+
+    // Send an acknowledgement, but ignore the errors.
+    // We have imported backup successfully already.
+    send_stream.write_all(b".").await.ok();
+    send_stream.finish().await.ok();
+    info!(context, "Sent backup reception acknowledgment.");
+
+    Ok(())
+}
+
+/// Contacts a backup provider and receives the backup from it.
+///
+/// This uses a QR code to contact another instance of deltachat which is providing a backup
+/// using the [`BackupProvider`].  Once connected it will authenticate using the secrets in
+/// the QR code and retrieve the backup.
+///
+/// This is a long running operation which will return only when completed.
+///
+/// Using [`Qr`] as argument is a bit odd as it only accepts specific variants of it.  It
+/// does avoid having [`iroh_old::provider::Ticket`] in the primary API however, without
+/// having to revert to untyped bytes.
+pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> {
+    match qr {
+        Qr::Backup { .. } => get_legacy_backup(context, qr).await?,
+        Qr::Backup2 {
+            node_addr,
+            auth_token,
+        } => get_backup2(context, node_addr, auth_token).await?,
+        _ => bail!("QR code for backup must be of type DCBACKUP or DCBACKUP2"),
+    }
+    Ok(())
+}
+
 async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> {
     let ticket = match qr {
         Qr::Backup { ticket } => ticket,
@@ -458,7 +410,7 @@ async fn transfer_from_provider(context: &Context, ticket: &Ticket) -> Result<()
 
     // Perform the transfer.
     let keylog = false; // Do not enable rustls SSLKEYLOGFILE env var functionality
-    let stats = iroh::get::run_ticket(
+    let stats = iroh_old::get::run_ticket(
         ticket,
         keylog,
         MAX_CONCURRENT_DIALS,
@@ -490,7 +442,7 @@ async fn on_blob(
     progress: &ProgressEmitter,
     jobs: &Mutex<JoinSet<()>>,
     ticket: &Ticket,
-    _hash: Hash,
+    _hash: iroh_old::Hash,
     mut reader: DataStream,
     name: String,
 ) -> Result<DataStream> {
@@ -672,24 +624,6 @@ mod tests {
             .await;
     }
 
-    #[test]
-    fn test_send_progress() {
-        let cases = [
-            ((0, 100), 450),
-            ((10, 100), 500),
-            ((50, 100), 700),
-            ((100, 100), 950),
-        ];
-
-        for ((current_size, total_size), progress) in cases {
-            let out = EventType::from(SendProgress::TransferInProgress {
-                current_size,
-                total_size,
-            });
-            assert_eq!(out, EventType::ImexProgress(progress));
-        }
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_drop_provider() {
         let mut tcm = TestContextManager::new();

src/key.rs

@@ -4,7 +4,7 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
 
-use anyhow::{ensure, Context as _, Result};
+use anyhow::{bail, ensure, Context as _, Result};
 use base64::Engine as _;
 use deltachat_contact_tools::EmailAddress;
 use num_traits::FromPrimitive;
@@ -46,7 +46,26 @@ pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// the ASCII-armored representation.
     fn from_asc(data: &str) -> Result<(Self, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::from_armor_single(Cursor::new(bytes)).context("rPGP error")
+        let res = Self::from_armor_single(Cursor::new(bytes));
+        let (key, headers) = match res {
+            Err(pgp::errors::Error::NoMatchingPacket) => match Self::is_private() {
+                true => bail!("No private key packet found"),
+                false => bail!("No public key packet found"),
+            },
+            _ => res.context("rPGP error")?,
+        };
+        let headers = headers
+            .into_iter()
+            .map(|(key, values)| {
+                (
+                    key.trim().to_lowercase(),
+                    values
+                        .last()
+                        .map_or_else(String::new, |s| s.trim().to_string()),
+                )
+            })
+            .collect();
+        Ok((key, headers))
     }
 
     /// Serialise the key as bytes.
@@ -77,6 +96,8 @@ pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     fn fingerprint(&self) -> Fingerprint {
         Fingerprint::new(KeyTrait::fingerprint(self))
     }
+
+    fn is_private() -> bool;
 }
 
 pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPublicKey> {
@@ -168,16 +189,17 @@ impl DcKey for SignedPublicKey {
         // safe to ignore this error.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.
-        let headers = header.map(|(key, value)| {
-            let mut m = BTreeMap::new();
-            m.insert(key.to_string(), value.to_string());
-            m
-        });
+        let headers =
+            header.map(|(key, value)| BTreeMap::from([(key.to_string(), vec![value.to_string()])]));
         let mut buf = Vec::new();
-        self.to_armored_writer(&mut buf, headers.as_ref())
+        self.to_armored_writer(&mut buf, headers.as_ref().into())
             .unwrap_or_default();
         std::string::String::from_utf8(buf).unwrap_or_default()
     }
+
+    fn is_private() -> bool {
+        false
+    }
 }
 
 impl DcKey for SignedSecretKey {
@@ -186,16 +208,17 @@ impl DcKey for SignedSecretKey {
         // safe to do these unwraps.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.  The string is always ASCII.
-        let headers = header.map(|(key, value)| {
-            let mut m = BTreeMap::new();
-            m.insert(key.to_string(), value.to_string());
-            m
-        });
+        let headers =
+            header.map(|(key, value)| BTreeMap::from([(key.to_string(), vec![value.to_string()])]));
         let mut buf = Vec::new();
-        self.to_armored_writer(&mut buf, headers.as_ref())
+        self.to_armored_writer(&mut buf, headers.as_ref().into())
             .unwrap_or_default();
         std::string::String::from_utf8(buf).unwrap_or_default()
     }
+
+    fn is_private() -> bool {
+        true
+    }
 }
 
 /// Deltachat extension trait for secret keys.

src/location.rs

@@ -109,7 +109,7 @@ impl Kml {
         ensure!(to_parse.len() <= 1024 * 1024, "kml-file is too large");
 
         let mut reader = quick_xml::Reader::from_reader(to_parse);
-        reader.trim_text(true);
+        reader.config_mut().trim_text(true);
 
         let mut kml = Kml::new();
         kml.locations = Vec::with_capacity(100);
@@ -226,7 +226,7 @@ impl Kml {
                     == "addr"
             }) {
                 self.addr = addr
-                    .decode_and_unescape_value(reader)
+                    .decode_and_unescape_value(reader.decoder())
                     .ok()
                     .map(|a| a.into_owned());
             }
@@ -256,7 +256,7 @@ impl Kml {
             }) {
                 let v = acc
                     .unwrap()
-                    .decode_and_unescape_value(reader)
+                    .decode_and_unescape_value(reader.decoder())
                     .unwrap_or_default();
 
                 self.curr.accuracy = v.trim().parse().unwrap_or_default();

src/login_param.rs

@@ -10,7 +10,7 @@ use crate::provider::Socket;
 use crate::provider::{get_provider_by_id, Provider};
 use crate::socks::Socks5Config;
 
-#[derive(Copy, Clone, Debug, Display, FromPrimitive, ToPrimitive, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, Default, Display, FromPrimitive, ToPrimitive, PartialEq, Eq)]
 #[repr(u32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
@@ -30,6 +30,7 @@ pub enum CertificateChecks {
     /// means that provider database setting should be taken.
     /// If there is no provider database setting for certificate checks,
     /// `Automatic` is the same as `Strict`.
+    #[default]
     Automatic = 0,
 
     Strict = 1,
@@ -41,12 +42,6 @@ pub enum CertificateChecks {
     AcceptInvalidCertificates = 3,
 }
 
-impl Default for CertificateChecks {
-    fn default() -> Self {
-        Self::Automatic
-    }
-}
-
 /// Login parameters for a single server, either IMAP or SMTP
 #[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub struct ServerLoginParam {
@@ -56,10 +51,6 @@ pub struct ServerLoginParam {
     pub port: u16,
     pub security: Socket,
     pub oauth2: bool,
-
-    /// TLS options: whether to allow invalid certificates and/or
-    /// invalid hostnames
-    pub certificate_checks: CertificateChecks,
 }
 
 #[derive(Default, Debug, Clone, PartialEq, Eq)]
@@ -69,6 +60,10 @@ pub struct LoginParam {
     pub smtp: ServerLoginParam,
     pub provider: Option<&'static Provider>,
     pub socks5_config: Option<Socks5Config>,
+
+    /// TLS options: whether to allow invalid certificates and/or
+    /// invalid hostnames
+    pub certificate_checks: CertificateChecks,
 }
 
 impl LoginParam {
@@ -130,8 +125,12 @@ impl LoginParam {
             .and_then(num_traits::FromPrimitive::from_i32)
             .unwrap_or_default();
 
+        // The setting is named `imap_certificate_checks`
+        // for backwards compatibility,
+        // but now it is a global setting applied to all protocols,
+        // while `smtp_certificate_checks` is ignored.
         let key = &format!("{prefix}imap_certificate_checks");
-        let imap_certificate_checks =
+        let certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(key).await? {
                 num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
             } else {
@@ -157,14 +156,6 @@ impl LoginParam {
             .and_then(num_traits::FromPrimitive::from_i32)
             .unwrap_or_default();
 
-        let key = &format!("{prefix}smtp_certificate_checks");
-        let smtp_certificate_checks =
-            if let Some(certificate_checks) = sql.get_raw_config_int(key).await? {
-                num_traits::FromPrimitive::from_i32(certificate_checks).unwrap_or_default()
-            } else {
-                Default::default()
-            };
-
         let key = &format!("{prefix}server_flags");
         let server_flags = sql.get_raw_config_int(key).await?.unwrap_or_default();
         let oauth2 = matches!(server_flags & DC_LP_AUTH_FLAGS, DC_LP_AUTH_OAUTH2);
@@ -186,7 +177,6 @@ impl LoginParam {
                 port: mail_port as u16,
                 security: mail_security,
                 oauth2,
-                certificate_checks: imap_certificate_checks,
             },
             smtp: ServerLoginParam {
                 server: send_server,
@@ -195,8 +185,8 @@ impl LoginParam {
                 port: send_port as u16,
                 security: send_security,
                 oauth2,
-                certificate_checks: smtp_certificate_checks,
             },
+            certificate_checks,
             provider,
             socks5_config,
         })
@@ -227,7 +217,7 @@ impl LoginParam {
             .await?;
 
         let key = &format!("{prefix}imap_certificate_checks");
-        sql.set_raw_config_int(key, self.imap.certificate_checks as i32)
+        sql.set_raw_config_int(key, self.certificate_checks as i32)
             .await?;
 
         let key = &format!("{prefix}send_server");
@@ -247,8 +237,9 @@ impl LoginParam {
         sql.set_raw_config_int(key, self.smtp.security as i32)
             .await?;
 
+        // This is only saved for compatibility reasons, but never loaded.
         let key = &format!("{prefix}smtp_certificate_checks");
-        sql.set_raw_config_int(key, self.smtp.certificate_checks as i32)
+        sql.set_raw_config_int(key, self.certificate_checks as i32)
             .await?;
 
         // The OAuth2 flag is either set for both IMAP and SMTP or not at all.
@@ -259,13 +250,23 @@ impl LoginParam {
         };
         sql.set_raw_config_int(key, server_flags).await?;
 
-        if let Some(provider) = self.provider {
-            let key = &format!("{prefix}provider");
-            sql.set_raw_config(key, Some(provider.id)).await?;
-        }
+        let key = &format!("{prefix}provider");
+        sql.set_raw_config(key, self.provider.map(|provider| provider.id))
+            .await?;
 
         Ok(())
     }
+
+    pub fn strict_tls(&self) -> bool {
+        let user_strict_tls = match self.certificate_checks {
+            CertificateChecks::Automatic => None,
+            CertificateChecks::Strict => Some(true),
+            CertificateChecks::AcceptInvalidCertificates
+            | CertificateChecks::AcceptInvalidCertificates2 => Some(false),
+        };
+        let provider_strict_tls = self.provider.map(|provider| provider.opt.strict_tls);
+        user_strict_tls.or(provider_strict_tls).unwrap_or(true)
+    }
 }
 
 impl fmt::Display for LoginParam {
@@ -275,7 +276,7 @@ impl fmt::Display for LoginParam {
 
         write!(
             f,
-            "{} imap:{}:{}:{}:{}:{}:cert_{}:{} smtp:{}:{}:{}:{}:{}:cert_{}:{}",
+            "{} imap:{}:{}:{}:{}:{}:{} smtp:{}:{}:{}:{}:{}:{} cert_{}",
             unset_empty(&self.addr),
             unset_empty(&self.imap.user),
             if !self.imap.password.is_empty() {
@@ -286,7 +287,6 @@ impl fmt::Display for LoginParam {
             unset_empty(&self.imap.server),
             self.imap.port,
             self.imap.security,
-            self.imap.certificate_checks,
             if self.imap.oauth2 {
                 "OAUTH2"
             } else {
@@ -301,12 +301,12 @@ impl fmt::Display for LoginParam {
             unset_empty(&self.smtp.server),
             self.smtp.port,
             self.smtp.security,
-            self.smtp.certificate_checks,
             if self.smtp.oauth2 {
                 "OAUTH2"
             } else {
                 "AUTH_NORMAL"
             },
+            self.certificate_checks
         )
     }
 }
@@ -347,7 +347,6 @@ mod tests {
                 port: 123,
                 security: Socket::Starttls,
                 oauth2: false,
-                certificate_checks: CertificateChecks::Strict,
             },
             smtp: ServerLoginParam {
                 server: "smtp.example.com".to_string(),
@@ -356,16 +355,24 @@ mod tests {
                 port: 456,
                 security: Socket::Ssl,
                 oauth2: false,
-                certificate_checks: CertificateChecks::AcceptInvalidCertificates,
             },
             provider: get_provider_by_id("example.com"),
             // socks5_config is not saved by `save_to_database`, using default value
             socks5_config: None,
+            certificate_checks: CertificateChecks::Strict,
         };
 
         param.save_as_configured_params(&t).await?;
         let loaded = LoginParam::load_configured_params(&t).await?;
+        assert_eq!(param, loaded);
 
+        // Remove provider.
+        let param = LoginParam {
+            provider: None,
+            ..param
+        };
+        param.save_as_configured_params(&t).await?;
+        let loaded = LoginParam::load_configured_params(&t).await?;
         assert_eq!(param, loaded);
         Ok(())
     }

src/message.rs

@@ -2,6 +2,7 @@
 
 use std::collections::BTreeSet;
 use std::path::{Path, PathBuf};
+use std::str;
 
 use anyhow::{ensure, format_err, Context as _, Result};
 use deltachat_contact_tools::{parse_vcard, VcardContact};
@@ -10,13 +11,13 @@ use serde::{Deserialize, Serialize};
 use tokio::{fs, io};
 
 use crate::blob::BlobObject;
-use crate::chat::{Chat, ChatId, ChatIdBlocked};
+use crate::chat::{Chat, ChatId, ChatIdBlocked, ChatVisibility};
 use crate::chatlist_events;
 use crate::config::Config;
 use crate::constants::{
     Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
 };
-use crate::contact::{Contact, ContactId};
+use crate::contact::{self, Contact, ContactId};
 use crate::context::Context;
 use crate::debug_logging::set_debug_logging_xdc;
 use crate::download::DownloadState;
@@ -80,7 +81,20 @@ impl MsgId {
     pub async fn get_state(self, context: &Context) -> Result<MessageState> {
         let result = context
             .sql
-            .query_get_value("SELECT state FROM msgs WHERE id=?", (self,))
+            .query_row_optional(
+                concat!(
+                    "SELECT m.state, mdns.msg_id",
+                    " FROM msgs m LEFT JOIN msgs_mdns mdns ON mdns.msg_id=m.id",
+                    " WHERE id=?",
+                    " LIMIT 1",
+                ),
+                (self,),
+                |row| {
+                    let state: MessageState = row.get(0)?;
+                    let mdn_msg_id: Option<MsgId> = row.get(1)?;
+                    Ok(state.with_mdns(mdn_msg_id.is_some()))
+                },
+            )
             .await?
             .unwrap_or_default();
         Ok(result)
@@ -102,23 +116,31 @@ impl MsgId {
     /// We keep some infos to
     /// 1. not download the same message again
     /// 2. be able to delete the message on the server if we want to
-    pub async fn trash(self, context: &Context) -> Result<()> {
+    ///
+    /// * `on_server`: Delete the message on the server also if it is seen on IMAP later, but only
+    ///   if all parts of the message are trashed with this flag. `true` if the user explicitly
+    ///   deletes the message. As for trashing a partially downloaded message when replacing it with
+    ///   a fully downloaded one, see `receive_imf::add_parts()`.
+    pub async fn trash(self, context: &Context, on_server: bool) -> Result<()> {
         let chat_id = DC_CHAT_ID_TRASH;
+        let deleted_subst = match on_server {
+            true => ", deleted=1",
+            false => "",
+        };
         context
             .sql
             .execute(
                 // If you change which information is removed here, also change delete_expired_messages() and
                 // which information receive_imf::add_parts() still adds to the db if the chat_id is TRASH
-                r#"
-UPDATE msgs 
-SET 
-  chat_id=?, txt='', 
-  subject='', txt_raw='', 
-  mime_headers='', 
-  from_id=0, to_id=0, 
-  param='' 
-WHERE id=?;
-"#,
+                &format!(
+                    "UPDATE msgs SET \
+                     chat_id=?, txt='', txt_normalized=NULL, \
+                     subject='', txt_raw='', \
+                     mime_headers='', \
+                     from_id=0, to_id=0, \
+                     param=''{deleted_subst} \
+                     WHERE id=?"
+                ),
                 (chat_id, self),
             )
             .await?;
@@ -510,6 +532,7 @@ impl Message {
                     "    m.ephemeral_timestamp AS ephemeral_timestamp,",
                     "    m.type AS type,",
                     "    m.state AS state,",
+                    "    mdns.msg_id AS mdn_msg_id,",
                     "    m.download_state AS download_state,",
                     "    m.error AS error,",
                     "    m.msgrmsg AS msgrmsg,",
@@ -520,11 +543,16 @@ impl Message {
                     "    m.hidden AS hidden,",
                     "    m.location_id AS location,",
                     "    c.blocked AS blocked",
-                    " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
-                    " WHERE m.id=? AND chat_id!=3;"
+                    " FROM msgs m",
+                    " LEFT JOIN chats c ON c.id=m.chat_id",
+                    " LEFT JOIN msgs_mdns mdns ON mdns.msg_id=m.id",
+                    " WHERE m.id=? AND chat_id!=3",
+                    " LIMIT 1",
                 ),
                 (id,),
                 |row| {
+                    let state: MessageState = row.get("state")?;
+                    let mdn_msg_id: Option<MsgId> = row.get("mdn_msg_id")?;
                     let text = match row.get_ref("txt")? {
                         rusqlite::types::ValueRef::Text(buf) => {
                             match String::from_utf8(buf.to_vec()) {
@@ -559,7 +587,7 @@ impl Message {
                         ephemeral_timer: row.get("ephemeral_timer")?,
                         ephemeral_timestamp: row.get("ephemeral_timestamp")?,
                         viewtype: row.get("type")?,
-                        state: row.get("state")?,
+                        state: state.with_mdns(mdn_msg_id.is_some()),
                         download_state: row.get("download_state")?,
                         error: Some(row.get::<_, String>("error")?)
                             .filter(|error| !error.is_empty()),
@@ -1081,6 +1109,30 @@ impl Message {
         Ok(())
     }
 
+    /// Makes message a vCard-containing message using the specified contacts.
+    pub async fn make_vcard(&mut self, context: &Context, contacts: &[ContactId]) -> Result<()> {
+        ensure!(
+            matches!(self.viewtype, Viewtype::File | Viewtype::Vcard),
+            "Wrong viewtype for vCard: {}",
+            self.viewtype,
+        );
+        let vcard = contact::make_vcard(context, contacts).await?;
+        self.set_file_from_bytes(context, "vcard.vcf", vcard.as_bytes(), None)
+            .await
+    }
+
+    /// Updates message state from the vCard attachment.
+    pub(crate) async fn try_set_vcard(&mut self, context: &Context, path: &Path) -> Result<()> {
+        let vcard = fs::read(path).await.context("Could not read {path}")?;
+        if let Some(summary) = get_vcard_summary(&vcard) {
+            self.param.set(Param::Summary1, summary);
+        } else {
+            warn!(context, "try_set_vcard: Not a valid DeltaChat vCard.");
+            self.viewtype = Viewtype::File;
+        }
+        Ok(())
+    }
+
     /// Set different sender name for a message.
     /// This overrides the name set by the `set_config()`-option `displayname`.
     pub fn set_override_sender_name(&mut self, name: Option<String>) {
@@ -1124,6 +1176,27 @@ impl Message {
         Ok(())
     }
 
+    /// Sets message quote text.
+    ///
+    /// If `text` is `Some((text_str, protect))`, `protect` specifies whether `text_str` should only
+    /// be sent encrypted. If it should, but the message is unencrypted, `text_str` is replaced with
+    /// "...".
+    pub fn set_quote_text(&mut self, text: Option<(String, bool)>) {
+        let Some((text, protect)) = text else {
+            self.param.remove(Param::Quote);
+            self.param.remove(Param::ProtectQuote);
+            return;
+        };
+        self.param.set(Param::Quote, text);
+        self.param.set_optional(
+            Param::ProtectQuote,
+            match protect {
+                true => Some("1"),
+                false => None,
+            },
+        );
+    }
+
     /// Sets message quote.
     ///
     /// Message-Id is used to set Reply-To field, message text is used for quote.
@@ -1140,31 +1213,27 @@ impl Message {
             );
             self.in_reply_to = Some(quote.rfc724_mid.clone());
 
-            if quote
-                .param
-                .get_bool(Param::GuaranteeE2ee)
-                .unwrap_or_default()
-            {
-                self.param.set(Param::ProtectQuote, "1");
-            }
-
             let text = quote.get_text();
-            self.param.set(
-                Param::Quote,
-                if text.is_empty() {
-                    // Use summary, similar to "Image" to avoid sending empty quote.
-                    quote
-                        .get_summary(context, None)
-                        .await?
-                        .truncated_text(500)
-                        .to_string()
-                } else {
-                    text
-                },
-            );
+            let text = if text.is_empty() {
+                // Use summary, similar to "Image" to avoid sending empty quote.
+                quote
+                    .get_summary(context, None)
+                    .await?
+                    .truncated_text(500)
+                    .to_string()
+            } else {
+                text
+            };
+            self.set_quote_text(Some((
+                text,
+                quote
+                    .param
+                    .get_bool(Param::GuaranteeE2ee)
+                    .unwrap_or_default(),
+            )));
         } else {
             self.in_reply_to = None;
-            self.param.remove(Param::Quote);
+            self.set_quote_text(None);
         }
 
         Ok(())
@@ -1303,7 +1372,7 @@ pub enum MessageState {
     OutDelivered = 26,
 
     /// Outgoing message read by the recipient (two checkmarks; this
-    /// requires goodwill on the receiver's side)
+    /// requires goodwill on the receiver's side). Not used in the db for new messages.
     OutMdnRcvd = 28,
 }
 
@@ -1346,6 +1415,14 @@ impl MessageState {
             OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
         )
     }
+
+    /// Returns adjusted message state if the message has MDNs.
+    pub(crate) fn with_mdns(self, has_mdns: bool) -> Self {
+        if self == MessageState::OutDelivered && has_mdns {
+            return MessageState::OutMdnRcvd;
+        }
+        self
+    }
 }
 
 /// Returns contacts that sent read receipts and the time of reading.
@@ -1524,8 +1601,9 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
         if msg.location_id > 0 {
             delete_poi_location(context, msg.location_id).await?;
         }
+        let on_server = true;
         msg_id
-            .trash(context)
+            .trash(context, on_server)
             .await
             .with_context(|| format!("Unable to trash message {msg_id}"))?;
 
@@ -1613,6 +1691,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
                     m.param AS param,
                     m.from_id AS from_id,
                     m.rfc724_mid AS rfc724_mid,
+                    c.archived AS archived,
                     c.blocked AS blocked
                  FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id
                  WHERE m.id IN ({}) AND m.chat_id>9",
@@ -1626,16 +1705,20 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
                 let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default();
                 let from_id: ContactId = row.get("from_id")?;
                 let rfc724_mid: String = row.get("rfc724_mid")?;
+                let visibility: ChatVisibility = row.get("archived")?;
                 let blocked: Option<Blocked> = row.get("blocked")?;
                 let ephemeral_timer: EphemeralTimer = row.get("ephemeral_timer")?;
                 Ok((
-                    id,
-                    chat_id,
-                    state,
-                    param,
-                    from_id,
-                    rfc724_mid,
-                    blocked.unwrap_or_default(),
+                    (
+                        id,
+                        chat_id,
+                        state,
+                        param,
+                        from_id,
+                        rfc724_mid,
+                        visibility,
+                        blocked.unwrap_or_default(),
+                    ),
                     ephemeral_timer,
                 ))
             },
@@ -1643,25 +1726,28 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
         )
         .await?;
 
-    if msgs.iter().any(
-        |(_id, _chat_id, _state, _param, _from_id, _rfc724_mid, _blocked, ephemeral_timer)| {
-            *ephemeral_timer != EphemeralTimer::Disabled
-        },
-    ) {
+    if msgs
+        .iter()
+        .any(|(_, ephemeral_timer)| *ephemeral_timer != EphemeralTimer::Disabled)
+    {
         start_ephemeral_timers_msgids(context, &msg_ids)
             .await
             .context("failed to start ephemeral timers")?;
     }
 
     let mut updated_chat_ids = BTreeSet::new();
+    let mut archived_chats_maybe_noticed = false;
     for (
-        id,
-        curr_chat_id,
-        curr_state,
-        curr_param,
-        curr_from_id,
-        curr_rfc724_mid,
-        curr_blocked,
+        (
+            id,
+            curr_chat_id,
+            curr_state,
+            curr_param,
+            curr_from_id,
+            curr_rfc724_mid,
+            curr_visibility,
+            curr_blocked,
+        ),
         _curr_ephemeral_timer,
     ) in msgs
     {
@@ -1683,28 +1769,31 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
             if curr_blocked == Blocked::Not
                 && curr_param.get_bool(Param::WantsMdn).unwrap_or_default()
                 && curr_param.get_cmd() == SystemMessage::Unknown
+                && context.should_send_mdns().await?
             {
-                let mdns_enabled = context.get_config_bool(Config::MdnsEnabled).await?;
-                if mdns_enabled {
-                    context
-                        .sql
-                        .execute(
-                            "INSERT INTO smtp_mdns (msg_id, from_id, rfc724_mid) VALUES(?, ?, ?)",
-                            (id, curr_from_id, curr_rfc724_mid),
-                        )
-                        .await
-                        .context("failed to insert into smtp_mdns")?;
-                    context.scheduler.interrupt_smtp().await;
-                }
+                context
+                    .sql
+                    .execute(
+                        "INSERT INTO smtp_mdns (msg_id, from_id, rfc724_mid) VALUES(?, ?, ?)",
+                        (id, curr_from_id, curr_rfc724_mid),
+                    )
+                    .await
+                    .context("failed to insert into smtp_mdns")?;
+                context.scheduler.interrupt_smtp().await;
             }
             updated_chat_ids.insert(curr_chat_id);
         }
+        archived_chats_maybe_noticed |=
+            curr_state == MessageState::InFresh && curr_visibility == ChatVisibility::Archived;
     }
 
     for updated_chat_id in updated_chat_ids {
         context.emit_event(EventType::MsgsNoticed(updated_chat_id));
         chatlist_events::emit_chatlist_item_changed(context, updated_chat_id);
     }
+    if archived_chats_maybe_noticed {
+        context.on_archived_chats_maybe_noticed();
+    }
 
     Ok(())
 }
@@ -1714,6 +1803,10 @@ pub(crate) async fn update_msg_state(
     msg_id: MsgId,
     state: MessageState,
 ) -> Result<()> {
+    ensure!(
+        state != MessageState::OutMdnRcvd,
+        "Update msgs_mdns table instead!"
+    );
     ensure!(state != MessageState::OutFailed, "use set_msg_failed()!");
     let error_subst = match state >= MessageState::OutPending {
         true => ", error=''",
@@ -1869,23 +1962,26 @@ pub async fn estimate_deletion_cnt(
     Ok(cnt)
 }
 
-/// See [`rfc724_mid_exists_and()`].
+/// See [`rfc724_mid_exists_ex()`].
 pub(crate) async fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
 ) -> Result<Option<(MsgId, i64)>> {
-    rfc724_mid_exists_and(context, rfc724_mid, "1").await
+    Ok(rfc724_mid_exists_ex(context, rfc724_mid, "1")
+        .await?
+        .map(|(id, ts_sent, _)| (id, ts_sent)))
 }
 
-/// Returns [MsgId] and "sent" timestamp of the message with given `rfc724_mid` (Message-ID header)
-/// if it exists in the db.
+/// Returns [MsgId] and "sent" timestamp of the most recent message with given `rfc724_mid`
+/// (Message-ID header) and bool `expr` result if such messages exists in the db.
 ///
-/// @param cond SQL subexpression for filtering messages.
-pub(crate) async fn rfc724_mid_exists_and(
+/// * `expr`: SQL expression additionally passed into `SELECT`. Evaluated to `true` iff it is true
+///   for all messages with the given `rfc724_mid`.
+pub(crate) async fn rfc724_mid_exists_ex(
     context: &Context,
     rfc724_mid: &str,
-    cond: &str,
-) -> Result<Option<(MsgId, i64)>> {
+    expr: &str,
+) -> Result<Option<(MsgId, i64, bool)>> {
     let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>');
     if rfc724_mid.is_empty() {
         warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
@@ -1895,13 +1991,15 @@ pub(crate) async fn rfc724_mid_exists_and(
     let res = context
         .sql
         .query_row_optional(
-            &("SELECT id, timestamp_sent FROM msgs WHERE rfc724_mid=? AND ".to_string() + cond),
+            &("SELECT id, timestamp_sent, MIN(".to_string()
+                + expr
+                + ") FROM msgs WHERE rfc724_mid=? ORDER BY timestamp_sent DESC"),
             (rfc724_mid,),
             |row| {
                 let msg_id: MsgId = row.get(0)?;
                 let timestamp_sent: i64 = row.get(1)?;
-
-                Ok((msg_id, timestamp_sent))
+                let expr_res: bool = row.get(2)?;
+                Ok((msg_id, timestamp_sent, expr_res))
             },
         )
         .await?;
@@ -1909,21 +2007,43 @@ pub(crate) async fn rfc724_mid_exists_and(
     Ok(res)
 }
 
-/// Given a list of Message-IDs, returns the latest message found in the database.
+/// Given a list of Message-IDs, returns the most relevant message found in the database.
 ///
+/// Relevance here is `(download_state == Done, index)`, where `index` is an index of Message-ID in
+/// `mids`. This means Message-IDs should be ordered from the least late to the latest one (like in
+/// the References header).
 /// Only messages that are not in the trash chat are considered.
-pub(crate) async fn get_latest_by_rfc724_mids(
+pub(crate) async fn get_by_rfc724_mids(
     context: &Context,
     mids: &[String],
 ) -> Result<Option<Message>> {
+    let mut latest = None;
     for id in mids.iter().rev() {
-        if let Some((msg_id, _)) = rfc724_mid_exists(context, id).await? {
-            if let Some(msg) = Message::load_from_db_optional(context, msg_id).await? {
-                return Ok(Some(msg));
-            }
+        let Some((msg_id, _)) = rfc724_mid_exists(context, id).await? else {
+            continue;
+        };
+        let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else {
+            continue;
+        };
+        if msg.download_state == DownloadState::Done {
+            return Ok(Some(msg));
         }
+        latest.get_or_insert(msg);
     }
-    Ok(None)
+    Ok(latest)
+}
+
+/// Returns the 1st part of summary text (i.e. before the dash if any) for a valid DeltaChat vCard.
+pub(crate) fn get_vcard_summary(vcard: &[u8]) -> Option<String> {
+    let vcard = str::from_utf8(vcard).ok()?;
+    let contacts = deltachat_contact_tools::parse_vcard(vcard);
+    let [c] = &contacts[..] else {
+        return None;
+    };
+    if !deltachat_contact_tools::may_be_valid_addr(&c.addr) {
+        return None;
+    }
+    Some(c.display_name().to_string())
 }
 
 /// How a message is primarily displayed.
@@ -2025,6 +2145,15 @@ impl Viewtype {
     }
 }
 
+/// Returns text for storing in the `msgs.txt_normalized` column (to make case-insensitive search
+/// possible for non-ASCII messages).
+pub(crate) fn normalize_text(text: &str) -> Option<String> {
+    if text.is_ascii() {
+        return None;
+    };
+    Some(text.to_lowercase()).filter(|t| t != text)
+}
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;
@@ -2255,12 +2384,23 @@ mod tests {
         // Alice quotes encrypted message in unencrypted chat.
         let mut msg = Message::new(Viewtype::Text);
         msg.set_quote(alice, Some(&alice_received_message)).await?;
+        msg.set_text("unencrypted".to_string());
         chat::send_msg(alice, alice_group, &mut msg).await?;
 
         let bob_received_message = bob.recv_msg(&alice.pop_sent_msg().await).await;
         assert_eq!(bob_received_message.quoted_text().unwrap(), "...");
         assert_eq!(bob_received_message.get_showpadlock(), false);
 
+        // Alice replaces a quote of encrypted message with a quote of unencrypted one.
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.set_quote(alice, Some(&alice_received_message)).await?;
+        msg1.set_quote(alice, Some(&msg)).await?;
+        chat::send_msg(alice, alice_group, &mut msg1).await?;
+
+        let bob_received_message = bob.recv_msg(&alice.pop_sent_msg().await).await;
+        assert_eq!(bob_received_message.quoted_text().unwrap(), "unencrypted");
+        assert_eq!(bob_received_message.get_showpadlock(), false);
+
         Ok(())
     }
 
@@ -2454,9 +2594,6 @@ mod tests {
         let payload = alice.pop_sent_msg().await;
         assert_state(&alice, alice_msg.id, MessageState::OutDelivered).await;
 
-        update_msg_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await?;
-        assert_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await;
-
         set_msg_failed(&alice, &mut alice_msg, "badly failed").await?;
         assert_state(&alice, alice_msg.id, MessageState::OutFailed).await;
 

src/mimeparser.rs

@@ -6,11 +6,12 @@ use std::path::Path;
 use std::str;
 
 use anyhow::{bail, Context as _, Result};
-use deltachat_contact_tools::{addr_cmp, addr_normalize, strip_rtlo_characters};
+use deltachat_contact_tools::{addr_cmp, addr_normalize, sanitize_bidi_characters};
 use deltachat_derive::{FromSql, ToSql};
 use format_flowed::unformat_flowed;
 use lettre_email::mime::Mime;
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
+use rand::distributions::{Alphanumeric, DistString};
 
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::blob::BlobObject;
@@ -27,9 +28,7 @@ use crate::dehtml::dehtml;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{self, load_self_secret_keyring, DcKey, Fingerprint, SignedPublicKey};
-use crate::message::{
-    self, set_msg_failed, update_msg_state, Message, MessageState, MsgId, Viewtype,
-};
+use crate::message::{self, get_vcard_summary, set_msg_failed, Message, MsgId, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::{simplify, SimplifiedText};
@@ -219,13 +218,8 @@ impl MimeMessage {
         let mail = mailparse::parse_mail(body)?;
 
         let timestamp_rcvd = smeared_time(context);
-        let timestamp_sent = mail
-            .headers
-            .get_header_value(HeaderDef::Date)
-            .and_then(|v| mailparse::dateparse(&v).ok())
-            .map_or(timestamp_rcvd, |value| {
-                min(value, timestamp_rcvd + constants::TIMESTAMP_SENT_TOLERANCE)
-            });
+        let mut timestamp_sent =
+            Self::get_timestamp_sent(&mail.headers, timestamp_rcvd, timestamp_rcvd);
         let mut hop_info = parse_receive_headers(&mail.get_headers());
 
         let mut headers = Default::default();
@@ -253,6 +247,8 @@ impl MimeMessage {
                     // We don't remove "subject" from `headers` because currently just signed
                     // messages are shown as unencrypted anyway.
 
+                    timestamp_sent =
+                        Self::get_timestamp_sent(&mail.headers, timestamp_sent, timestamp_rcvd);
                     MimeMessage::merge_headers(
                         context,
                         &mut headers,
@@ -302,7 +298,7 @@ impl MimeMessage {
         // them in signed-only emails, but has no value currently.
         Self::remove_secured_headers(&mut headers);
 
-        let from = from.context("No from in message")?;
+        let mut from = from.context("No from in message")?;
         let private_keyring = load_self_secret_keyring(context).await?;
 
         let mut decryption_info =
@@ -348,6 +344,8 @@ impl MimeMessage {
             content
         });
         if let (Ok(mail), true) = (mail, encrypted) {
+            timestamp_sent =
+                Self::get_timestamp_sent(&mail.headers, timestamp_sent, timestamp_rcvd);
             if !signatures.is_empty() {
                 // Handle any gossip headers if the mail was encrypted. See section
                 // "3.6 Key Gossip" of <https://autocrypt.org/autocrypt-spec-1.1.0.pdf>
@@ -395,12 +393,10 @@ impl MimeMessage {
                 &mail.headers,
             );
 
-            if let (Some(inner_from), true) = (inner_from, !signatures.is_empty()) {
-                if addr_cmp(&inner_from.addr, &from.addr) {
-                    from_is_signed = true;
-                } else {
-                    // There is a From: header in the encrypted &
-                    // signed part, but it doesn't match the outer one.
+            if let Some(inner_from) = inner_from {
+                if !addr_cmp(&inner_from.addr, &from.addr) {
+                    // There is a From: header in the encrypted
+                    // part, but it doesn't match the outer one.
                     // This _might_ be because the sender's mail server
                     // replaced the sending address, e.g. in a mailing list.
                     // Or it's because someone is doing some replay attack.
@@ -409,7 +405,7 @@ impl MimeMessage {
                     // so we return an error below.
                     warn!(
                         context,
-                        "From header in signed part doesn't match the outer one",
+                        "From header in encrypted part doesn't match the outer one",
                     );
 
                     // Return an error from the parser.
@@ -418,6 +414,8 @@ impl MimeMessage {
                     // as if the MIME structure is broken.
                     bail!("From header is forged");
                 }
+                from = inner_from;
+                from_is_signed = !signatures.is_empty();
             }
         }
         if signatures.is_empty() {
@@ -523,6 +521,18 @@ impl MimeMessage {
         Ok(parser)
     }
 
+    fn get_timestamp_sent(
+        hdrs: &[mailparse::MailHeader<'_>],
+        default: i64,
+        timestamp_rcvd: i64,
+    ) -> i64 {
+        hdrs.get_header_value(HeaderDef::Date)
+            .and_then(|v| mailparse::dateparse(&v).ok())
+            .map_or(default, |value| {
+                min(value, timestamp_rcvd + constants::TIMESTAMP_SENT_TOLERANCE)
+            })
+    }
+
     /// Parses system messages.
     fn parse_system_message_headers(&mut self, context: &Context) {
         if self.get_header(HeaderDef::AutocryptSetupMessage).is_some() && !self.incoming {
@@ -772,7 +782,15 @@ impl MimeMessage {
             .collect::<String>()
             .strip_prefix("base64:")
         {
-            match BlobObject::store_from_base64(context, base64, "avatar").await {
+            // Add random suffix to the filename
+            // to prevent the UI from accidentally using
+            // cached "avatar.jpg".
+            let suffix = Alphanumeric
+                .sample_string(&mut rand::thread_rng(), 7)
+                .to_lowercase();
+
+            match BlobObject::store_from_base64(context, base64, &format!("avatar-{suffix}")).await
+            {
                 Ok(path) => Some(AvatarAction::Change(path)),
                 Err(err) => {
                     warn!(
@@ -1233,6 +1251,7 @@ impl MimeMessage {
                 return Ok(());
             }
         }
+        let mut part = Part::default();
         let msg_type = if context
             .is_webxdc_file(filename, decoded_data)
             .await
@@ -1276,6 +1295,13 @@ impl MimeMessage {
                 .unwrap_or_default();
             self.webxdc_status_update = Some(serialized);
             return Ok(());
+        } else if msg_type == Viewtype::Vcard {
+            if let Some(summary) = get_vcard_summary(decoded_data) {
+                part.param.set(Param::Summary1, summary);
+                msg_type
+            } else {
+                Viewtype::File
+            }
         } else {
             msg_type
         };
@@ -1295,8 +1321,6 @@ impl MimeMessage {
         };
         info!(context, "added blobfile: {:?}", blob.as_name());
 
-        /* create and register Mime part referencing the new Blob object */
-        let mut part = Part::default();
         if mime_type.type_() == mime::IMAGE {
             if let Ok((width, height)) = get_filemeta(decoded_data) {
                 part.param.set_int(Param::Width, width as i32);
@@ -1928,7 +1952,10 @@ pub struct Part {
     pub(crate) is_reaction: bool,
 }
 
-/// return mimetype and viewtype for a parsed mail
+/// Returns the mimetype and viewtype for a parsed mail.
+///
+/// This only looks at the metadata, not at the content;
+/// the viewtype may later be corrected in `do_add_single_file_part()`.
 fn get_mime_type(
     mail: &mailparse::ParsedMail<'_>,
     filename: &Option<String>,
@@ -1937,7 +1964,7 @@ fn get_mime_type(
 
     let viewtype = match mimetype.type_() {
         mime::TEXT => match mimetype.subtype() {
-            mime::VCARD if is_valid_deltachat_vcard(mail) => Viewtype::Vcard,
+            mime::VCARD => Viewtype::Vcard,
             mime::PLAIN | mime::HTML if !is_attachment_disposition(mail) => Viewtype::Text,
             _ => Viewtype::File,
         },
@@ -1988,17 +2015,6 @@ fn is_attachment_disposition(mail: &mailparse::ParsedMail<'_>) -> bool {
             .any(|(key, _value)| key.starts_with("filename"))
 }
 
-fn is_valid_deltachat_vcard(mail: &mailparse::ParsedMail) -> bool {
-    let Ok(body) = &mail.get_body() else {
-        return false;
-    };
-    let contacts = deltachat_contact_tools::parse_vcard(body);
-    if let [c] = &contacts[..] {
-        return deltachat_contact_tools::may_be_valid_addr(&c.addr);
-    }
-    false
-}
-
 /// Tries to get attachment filename.
 ///
 /// If filename is explicitly specified in Content-Disposition, it is
@@ -2048,7 +2064,7 @@ fn get_attachment_filename(
         };
     }
 
-    let desired_filename = desired_filename.map(|filename| strip_rtlo_characters(&filename));
+    let desired_filename = desired_filename.map(|filename| sanitize_bidi_characters(&filename));
 
     Ok(desired_filename)
 }
@@ -2138,24 +2154,32 @@ async fn handle_mdn(
         return Ok(());
     }
 
-    let Some((msg_id, chat_id, msg_state)) = context
+    let Some((msg_id, chat_id, has_mdns, is_dup)) = context
         .sql
         .query_row_optional(
             concat!(
                 "SELECT",
                 "    m.id AS msg_id,",
                 "    c.id AS chat_id,",
-                "    m.state AS state",
-                " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+                "    mdns.contact_id AS mdn_contact",
+                " FROM msgs m ",
+                " LEFT JOIN chats c ON m.chat_id=c.id",
+                " LEFT JOIN msgs_mdns mdns ON mdns.msg_id=m.id",
                 " WHERE rfc724_mid=? AND from_id=1",
-                " ORDER BY m.id"
+                " ORDER BY msg_id DESC, mdn_contact=? DESC",
+                " LIMIT 1",
             ),
-            (&rfc724_mid,),
+            (&rfc724_mid, from_id),
             |row| {
                 let msg_id: MsgId = row.get("msg_id")?;
                 let chat_id: ChatId = row.get("chat_id")?;
-                let msg_state: MessageState = row.get("state")?;
-                Ok((msg_id, chat_id, msg_state))
+                let mdn_contact: Option<ContactId> = row.get("mdn_contact")?;
+                Ok((
+                    msg_id,
+                    chat_id,
+                    mdn_contact.is_some(),
+                    mdn_contact == Some(from_id),
+                ))
             },
         )
         .await?
@@ -2167,28 +2191,17 @@ async fn handle_mdn(
         return Ok(());
     };
 
-    if !context
-        .sql
-        .exists(
-            "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?",
-            (msg_id, from_id),
-        )
-        .await?
-    {
-        context
-            .sql
-            .execute(
-                "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?)",
-                (msg_id, from_id, timestamp_sent),
-            )
-            .await?;
+    if is_dup {
+        return Ok(());
     }
-
-    if msg_state == MessageState::OutPreparing
-        || msg_state == MessageState::OutPending
-        || msg_state == MessageState::OutDelivered
-    {
-        update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await?;
+    context
+        .sql
+        .execute(
+            "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?)",
+            (msg_id, from_id, timestamp_sent),
+        )
+        .await?;
+    if !has_mdns {
         context.emit_event(EventType::MsgRead { chat_id, msg_id });
         // note(treefit): only matters if it is the last message in chat (but probably too expensive to check, debounce also solves it)
         chatlist_events::emit_chatlist_item_changed(context, chat_id);
@@ -2296,7 +2309,7 @@ mod tests {
         chat,
         chatlist::Chatlist,
         constants::{Blocked, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS},
-        message::MessengerMessage,
+        message::{MessageState, MessengerMessage},
         receive_imf::receive_imf,
         test_utils::{TestContext, TestContextManager},
         tools::time,

src/net.rs

@@ -1,206 +1,88 @@
 //! # Common network utilities.
-use std::net::{IpAddr, SocketAddr};
-use std::net::{Ipv4Addr, Ipv6Addr};
+use std::net::SocketAddr;
 use std::pin::Pin;
-use std::str::FromStr;
 use std::time::Duration;
 
 use anyhow::{format_err, Context as _, Result};
-use tokio::net::{lookup_host, TcpStream};
+use async_native_tls::TlsStream;
+use tokio::net::TcpStream;
 use tokio::time::timeout;
 use tokio_io_timeout::TimeoutStream;
 
 use crate::context::Context;
 use crate::tools::time;
 
+pub(crate) mod dns;
 pub(crate) mod http;
 pub(crate) mod session;
 pub(crate) mod tls;
 
+use dns::lookup_host_with_cache;
 pub use http::{read_url, read_url_blob, Response as HttpResponse};
+use tls::wrap_tls;
 
-async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result<TcpStream> {
-    let tcp_stream = timeout(timeout_val, TcpStream::connect(addr))
-        .await
-        .context("connection timeout")?
-        .context("connection failure")?;
-    Ok(tcp_stream)
-}
-
-async fn lookup_host_with_timeout(
-    hostname: &str,
-    port: u16,
-    timeout_val: Duration,
-) -> Result<Vec<SocketAddr>> {
-    let res = timeout(timeout_val, lookup_host((hostname, port)))
-        .await
-        .context("DNS lookup timeout")?
-        .context("DNS lookup failure")?;
-    Ok(res.collect())
-}
-
-/// Looks up hostname and port using DNS and updates the address resolution cache.
+/// Connection, write and read timeout.
 ///
-/// If `load_cache` is true, appends cached results not older than 30 days to the end
-/// or entries from fallback cache if there are no cached addresses.
-async fn lookup_host_with_cache(
-    context: &Context,
-    hostname: &str,
-    port: u16,
-    timeout_val: Duration,
-    load_cache: bool,
-) -> Result<Vec<SocketAddr>> {
+/// This constant should be more than the largest expected RTT.
+pub(crate) const TIMEOUT: Duration = Duration::from_secs(60);
+
+/// TTL for caches in seconds.
+pub(crate) const CACHE_TTL: u64 = 30 * 24 * 60 * 60;
+
+/// Removes connection history entries after `CACHE_TTL`.
+pub(crate) async fn prune_connection_history(context: &Context) -> Result<()> {
     let now = time();
-    let mut resolved_addrs = match lookup_host_with_timeout(hostname, port, timeout_val).await {
-        Ok(res) => res,
-        Err(err) => {
-            warn!(
-                context,
-                "DNS resolution for {}:{} failed: {:#}.", hostname, port, err
-            );
-            Vec::new()
-        }
-    };
+    context
+        .sql
+        .execute(
+            "DELETE FROM connection_history
+             WHERE ? > timestamp + ?",
+            (now, CACHE_TTL),
+        )
+        .await?;
+    Ok(())
+}
 
-    for addr in &resolved_addrs {
-        let ip_string = addr.ip().to_string();
-        if ip_string == hostname {
-            // IP address resolved into itself, not interesting to cache.
-            continue;
-        }
+pub(crate) async fn update_connection_history(
+    context: &Context,
+    alpn: &str,
+    host: &str,
+    port: u16,
+    addr: &str,
+    now: i64,
+) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "INSERT INTO connection_history (host, port, alpn, addr, timestamp)
+             VALUES (?, ?, ?, ?, ?)
+             ON CONFLICT (host, port, alpn, addr)
+             DO UPDATE SET timestamp=excluded.timestamp",
+            (host, port, alpn, addr, now),
+        )
+        .await?;
+    Ok(())
+}
 
-        info!(context, "Resolved {}:{} into {}.", hostname, port, &addr);
-
-        // Update the cache.
-        context
-            .sql
-            .execute(
-                "INSERT INTO dns_cache
-                 (hostname, address, timestamp)
-                 VALUES (?, ?, ?)
-                 ON CONFLICT (hostname, address)
-                 DO UPDATE SET timestamp=excluded.timestamp",
-                (hostname, ip_string, now),
-            )
-            .await?;
-    }
-
-    if load_cache {
-        for cached_address in context
-            .sql
-            .query_map(
-                "SELECT address
-                 FROM dns_cache
-                 WHERE hostname = ?
-                 AND ? < timestamp + 30 * 24 * 3600
-                 ORDER BY timestamp DESC",
-                (hostname, now),
-                |row| {
-                    let address: String = row.get(0)?;
-                    Ok(address)
-                },
-                |rows| {
-                    rows.collect::<std::result::Result<Vec<_>, _>>()
-                        .map_err(Into::into)
-                },
-            )
-            .await?
-        {
-            match IpAddr::from_str(&cached_address) {
-                Ok(ip_addr) => {
-                    let addr = SocketAddr::new(ip_addr, port);
-                    if !resolved_addrs.contains(&addr) {
-                        resolved_addrs.push(addr);
-                    }
-                }
-                Err(err) => {
-                    warn!(
-                        context,
-                        "Failed to parse cached address {:?}: {:#}.", cached_address, err
-                    );
-                }
-            }
-        }
-
-        if resolved_addrs.is_empty() {
-            // Load hardcoded cache if everything else fails.
-            //
-            // See <https://support.delta.chat/t/no-dns-resolution-result/2778> and
-            // <https://github.com/deltachat/deltachat-core-rust/issues/4920> for reasons.
-            //
-            // In the future we may pre-resolve all provider database addresses
-            // and build them in.
-            match hostname {
-                "mail.sangham.net" => {
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0xc17, 0x798c, 0, 0, 0, 1)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(159, 69, 186, 85)),
-                        port,
-                    ));
-                }
-                "nine.testrun.org" => {
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0x241, 0x4ce8, 0, 0, 0, 2)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(116, 202, 233, 236)),
-                        port,
-                    ));
-                }
-                "disroot.org" => {
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(178, 21, 23, 139)),
-                        port,
-                    ));
-                }
-                "mail.riseup.net" => {
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(198, 252, 153, 70)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(198, 252, 153, 71)),
-                        port,
-                    ));
-                }
-                "imap.gmail.com" => {
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V6(Ipv6Addr::new(0x2a00, 0x1450, 0x400c, 0xc1f, 0, 0, 0, 0x6c)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V6(Ipv6Addr::new(0x2a00, 0x1450, 0x400c, 0xc1f, 0, 0, 0, 0x6d)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(142, 250, 110, 109)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(142, 250, 110, 108)),
-                        port,
-                    ));
-                }
-                "smtp.gmail.com" => {
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V6(Ipv6Addr::new(0x2a00, 0x1450, 0x4013, 0xc04, 0, 0, 0, 0x6c)),
-                        port,
-                    ));
-                    resolved_addrs.push(SocketAddr::new(
-                        IpAddr::V4(Ipv4Addr::new(142, 250, 110, 109)),
-                        port,
-                    ));
-                }
-                _ => {}
-            }
-        }
-    }
-
-    Ok(resolved_addrs)
+pub(crate) async fn load_connection_timestamp(
+    context: &Context,
+    alpn: &str,
+    host: &str,
+    port: u16,
+    addr: &str,
+) -> Result<Option<i64>> {
+    let timestamp = context
+        .sql
+        .query_get_value(
+            "SELECT timestamp FROM connection_history
+             WHERE host = ?
+               AND port = ?
+               AND alpn = ?
+               AND addr = ?",
+            (host, port, alpn, addr),
+        )
+        .await?;
+    Ok(timestamp)
 }
 
 /// Returns a TCP connection stream with read/write timeouts set
@@ -208,7 +90,37 @@ async fn lookup_host_with_cache(
 ///
 /// `TCP_NODELAY` ensures writing to the stream always results in immediate sending of the packet
 /// to the network, which is important to reduce the latency of interactive protocols such as IMAP.
-///
+pub(crate) async fn connect_tcp_inner(
+    addr: SocketAddr,
+) -> Result<Pin<Box<TimeoutStream<TcpStream>>>> {
+    let tcp_stream = timeout(TIMEOUT, TcpStream::connect(addr))
+        .await
+        .context("connection timeout")?
+        .context("connection failure")?;
+
+    // Disable Nagle's algorithm.
+    tcp_stream.set_nodelay(true)?;
+
+    let mut timeout_stream = TimeoutStream::new(tcp_stream);
+    timeout_stream.set_write_timeout(Some(TIMEOUT));
+    timeout_stream.set_read_timeout(Some(TIMEOUT));
+
+    Ok(Box::pin(timeout_stream))
+}
+
+/// Attempts to establish TLS connection
+/// given the result of the hostname to address resolution.
+pub(crate) async fn connect_tls_inner(
+    addr: SocketAddr,
+    host: &str,
+    strict_tls: bool,
+    alpn: &str,
+) -> Result<TlsStream<Pin<Box<TimeoutStream<TcpStream>>>>> {
+    let tcp_stream = connect_tcp_inner(addr).await?;
+    let tls_stream = wrap_tls(strict_tls, host, alpn, tcp_stream).await?;
+    Ok(tls_stream)
+}
+
 /// If `load_cache` is true, may use cached DNS results.
 /// Because the cache may be poisoned with incorrect results by networks hijacking DNS requests,
 /// this option should only be used when connection is authenticated,
@@ -219,57 +131,24 @@ pub(crate) async fn connect_tcp(
     context: &Context,
     host: &str,
     port: u16,
-    timeout_val: Duration,
     load_cache: bool,
 ) -> Result<Pin<Box<TimeoutStream<TcpStream>>>> {
-    let mut tcp_stream = None;
-    let mut last_error = None;
+    let mut first_error = None;
 
-    for resolved_addr in
-        lookup_host_with_cache(context, host, port, timeout_val, load_cache).await?
-    {
-        match connect_tcp_inner(resolved_addr, timeout_val).await {
+    for resolved_addr in lookup_host_with_cache(context, host, port, "", load_cache).await? {
+        match connect_tcp_inner(resolved_addr).await {
             Ok(stream) => {
-                tcp_stream = Some(stream);
-
-                // Maximize priority of this cached entry.
-                context
-                    .sql
-                    .execute(
-                        "UPDATE dns_cache
-                         SET timestamp = ?
-                         WHERE address = ?",
-                        (time(), resolved_addr.ip().to_string()),
-                    )
-                    .await?;
-                break;
+                return Ok(stream);
             }
             Err(err) => {
                 warn!(
                     context,
                     "Failed to connect to {}: {:#}.", resolved_addr, err
                 );
-                last_error = Some(err);
+                first_error.get_or_insert(err);
             }
         }
     }
 
-    let tcp_stream = match tcp_stream {
-        Some(tcp_stream) => tcp_stream,
-        None => {
-            return Err(
-                last_error.unwrap_or_else(|| format_err!("no DNS resolution results for {host}"))
-            );
-        }
-    };
-
-    // Disable Nagle's algorithm.
-    tcp_stream.set_nodelay(true)?;
-
-    let mut timeout_stream = TimeoutStream::new(tcp_stream);
-    timeout_stream.set_write_timeout(Some(timeout_val));
-    timeout_stream.set_read_timeout(Some(timeout_val));
-    let pinned_stream = Box::pin(timeout_stream);
-
-    Ok(pinned_stream)
+    Err(first_error.unwrap_or_else(|| format_err!("no DNS resolution results for {host}")))
 }

src/net/dns.rs

@@ -0,0 +1,526 @@
+//! DNS resolution and cache.
+
+use anyhow::{Context as _, Result};
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
+use std::str::FromStr;
+use tokio::net::lookup_host;
+use tokio::time::timeout;
+
+use super::load_connection_timestamp;
+use crate::context::Context;
+use crate::tools::time;
+
+/// Inserts entry into DNS cache
+/// or updates existing one with a new timestamp.
+async fn update_cache(context: &Context, host: &str, addr: &str, now: i64) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "INSERT INTO dns_cache
+             (hostname, address, timestamp)
+             VALUES (?, ?, ?)
+             ON CONFLICT (hostname, address)
+             DO UPDATE SET timestamp=excluded.timestamp",
+            (host, addr, now),
+        )
+        .await?;
+    Ok(())
+}
+
+pub(crate) async fn prune_dns_cache(context: &Context) -> Result<()> {
+    let now = time();
+    context
+        .sql
+        .execute(
+            "DELETE FROM dns_cache
+             WHERE ? > timestamp + ?",
+            (now, super::CACHE_TTL),
+        )
+        .await?;
+    Ok(())
+}
+
+/// Looks up the hostname and updates DNS cache
+/// on success.
+async fn lookup_host_and_update_cache(
+    context: &Context,
+    hostname: &str,
+    port: u16,
+    now: i64,
+) -> Result<Vec<SocketAddr>> {
+    let res: Vec<SocketAddr> = timeout(super::TIMEOUT, lookup_host((hostname, port)))
+        .await
+        .context("DNS lookup timeout")?
+        .context("DNS lookup failure")?
+        .collect();
+
+    for addr in &res {
+        let ip_string = addr.ip().to_string();
+        if ip_string == hostname {
+            // IP address resolved into itself, not interesting to cache.
+            continue;
+        }
+
+        info!(context, "Resolved {hostname}:{port} into {addr}.");
+
+        // Update the cache.
+        update_cache(context, hostname, &ip_string, now).await?;
+    }
+
+    Ok(res)
+}
+
+// Updates timestamp of the cached entry
+// or inserts a new one if cached entry does not exist.
+//
+// This function should be called when a successful TLS
+// connection is established with strict TLS checks.
+//
+// This increases priority of existing cached entries
+// and copies fallback addresses from built-in cache
+// into database cache on successful use.
+//
+// Unlike built-in cache,
+// database cache is used even if DNS
+// resolver returns a non-empty
+// (but potentially incorrect and unusable) result.
+pub(crate) async fn update_connect_timestamp(
+    context: &Context,
+    host: &str,
+    address: &str,
+) -> Result<()> {
+    if host == address {
+        return Ok(());
+    }
+
+    context
+        .sql
+        .execute(
+            "INSERT INTO dns_cache (hostname, address, timestamp)
+                 VALUES (?, ?, ?)
+                 ON CONFLICT (hostname, address)
+                 DO UPDATE SET timestamp=excluded.timestamp",
+            (host, address, time()),
+        )
+        .await?;
+    Ok(())
+}
+
+/// Load hardcoded cache if everything else fails.
+///
+/// See <https://support.delta.chat/t/no-dns-resolution-result/2778> and
+/// <https://github.com/deltachat/deltachat-core-rust/issues/4920> for reasons.
+///
+/// In the future we may pre-resolve all provider database addresses
+/// and build them in.
+fn load_hardcoded_cache(hostname: &str, port: u16) -> Vec<SocketAddr> {
+    match hostname {
+        "mail.sangham.net" => {
+            vec![
+                SocketAddr::new(
+                    IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0xc17, 0x798c, 0, 0, 0, 1)),
+                    port,
+                ),
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(159, 69, 186, 85)), port),
+            ]
+        }
+        "nine.testrun.org" => {
+            vec![
+                SocketAddr::new(
+                    IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0x241, 0x4ce8, 0, 0, 0, 2)),
+                    port,
+                ),
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(116, 202, 233, 236)), port),
+            ]
+        }
+        "disroot.org" => {
+            vec![SocketAddr::new(
+                IpAddr::V4(Ipv4Addr::new(178, 21, 23, 139)),
+                port,
+            )]
+        }
+        "mail.riseup.net" => {
+            vec![
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 252, 153, 70)), port),
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(198, 252, 153, 71)), port),
+            ]
+        }
+        "imap.gmail.com" => {
+            vec![
+                SocketAddr::new(
+                    IpAddr::V6(Ipv6Addr::new(0x2a00, 0x1450, 0x400c, 0xc1f, 0, 0, 0, 0x6c)),
+                    port,
+                ),
+                SocketAddr::new(
+                    IpAddr::V6(Ipv6Addr::new(0x2a00, 0x1450, 0x400c, 0xc1f, 0, 0, 0, 0x6d)),
+                    port,
+                ),
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(142, 250, 110, 109)), port),
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(142, 250, 110, 108)), port),
+            ]
+        }
+        "smtp.gmail.com" => {
+            vec![
+                SocketAddr::new(
+                    IpAddr::V6(Ipv6Addr::new(0x2a00, 0x1450, 0x4013, 0xc04, 0, 0, 0, 0x6c)),
+                    port,
+                ),
+                SocketAddr::new(IpAddr::V4(Ipv4Addr::new(142, 250, 110, 109)), port),
+            ]
+        }
+        _ => Vec::new(),
+    }
+}
+
+async fn lookup_cache(
+    context: &Context,
+    host: &str,
+    port: u16,
+    alpn: &str,
+    now: i64,
+) -> Result<Vec<SocketAddr>> {
+    let mut res = Vec::new();
+    for cached_address in context
+        .sql
+        .query_map(
+            "SELECT dns_cache.address
+             FROM dns_cache
+             LEFT JOIN connection_history
+               ON dns_cache.hostname = connection_history.host
+               AND dns_cache.address = connection_history.addr
+               AND connection_history.port = ?
+               AND connection_history.alpn = ?
+             WHERE dns_cache.hostname = ?
+             AND ? < dns_cache.timestamp + ?
+             ORDER BY IFNULL(connection_history.timestamp, dns_cache.timestamp) DESC
+             LIMIT 50",
+            (port, alpn, host, now, super::CACHE_TTL),
+            |row| {
+                let address: String = row.get(0)?;
+                Ok(address)
+            },
+            |rows| {
+                rows.collect::<std::result::Result<Vec<String>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await?
+    {
+        match IpAddr::from_str(&cached_address) {
+            Ok(ip_addr) => {
+                let addr = SocketAddr::new(ip_addr, port);
+                res.push(addr);
+            }
+            Err(err) => {
+                warn!(
+                    context,
+                    "Failed to parse cached address {:?}: {:#}.", cached_address, err
+                );
+            }
+        }
+    }
+    Ok(res)
+}
+
+/// Sorts DNS resolution results by connection timestamp in descending order
+/// so IP addresses that we recently connected to successfully are tried first.
+async fn sort_by_connection_timestamp(
+    context: &Context,
+    input: Vec<SocketAddr>,
+    alpn: &str,
+    host: &str,
+) -> Result<Vec<SocketAddr>> {
+    let mut res: Vec<(Option<i64>, SocketAddr)> = Vec::new();
+    for addr in input {
+        let timestamp =
+            load_connection_timestamp(context, alpn, host, addr.port(), &addr.ip().to_string())
+                .await?;
+        res.push((timestamp, addr));
+    }
+    res.sort_by_key(|(ts, _addr)| std::cmp::Reverse(*ts));
+    Ok(res.into_iter().map(|(_ts, addr)| addr).collect())
+}
+
+/// Looks up hostname and port using DNS and updates the address resolution cache.
+///
+/// `alpn` is used to sort DNS results by the time we have successfully
+/// connected to the IP address using given `alpn`.
+/// If result sorting is not needed or `alpn` is unknown,
+/// pass empty string here, e.g. for HTTP requests
+/// or when resolving the IP address of SOCKS proxy.
+///
+/// If `load_cache` is true, appends cached results not older than 30 days to the end
+/// or entries from fallback cache if there are no cached addresses.
+pub(crate) async fn lookup_host_with_cache(
+    context: &Context,
+    hostname: &str,
+    port: u16,
+    alpn: &str,
+    load_cache: bool,
+) -> Result<Vec<SocketAddr>> {
+    let now = time();
+    let mut resolved_addrs = match lookup_host_and_update_cache(context, hostname, port, now).await
+    {
+        Ok(res) => res,
+        Err(err) => {
+            warn!(
+                context,
+                "DNS resolution for {hostname}:{port} failed: {err:#}."
+            );
+            Vec::new()
+        }
+    };
+    if !alpn.is_empty() {
+        resolved_addrs =
+            sort_by_connection_timestamp(context, resolved_addrs, alpn, hostname).await?;
+    }
+
+    if load_cache {
+        for addr in lookup_cache(context, hostname, port, alpn, now).await? {
+            if !resolved_addrs.contains(&addr) {
+                resolved_addrs.push(addr);
+            }
+        }
+
+        if resolved_addrs.is_empty() {
+            return Ok(load_hardcoded_cache(hostname, port));
+        }
+    }
+
+    Ok(resolved_addrs)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::net::update_connection_history;
+    use crate::test_utils::TestContext;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sort_by_connection_timestamp() {
+        let alice = &TestContext::new_alice().await;
+        let now = time();
+
+        let ipv6_addr = IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0x241, 0x4ce8, 0, 0, 0, 2));
+        let ipv4_addr = IpAddr::V4(Ipv4Addr::new(116, 202, 233, 236));
+
+        assert_eq!(
+            sort_by_connection_timestamp(
+                alice,
+                vec![
+                    SocketAddr::new(ipv6_addr, 993),
+                    SocketAddr::new(ipv4_addr, 993)
+                ],
+                "imap",
+                "nine.testrun.org"
+            )
+            .await
+            .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 993),
+                SocketAddr::new(ipv4_addr, 993)
+            ]
+        );
+        update_connection_history(
+            alice,
+            "imap",
+            "nine.testrun.org",
+            993,
+            "116.202.233.236",
+            now,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            sort_by_connection_timestamp(
+                alice,
+                vec![
+                    SocketAddr::new(ipv6_addr, 993),
+                    SocketAddr::new(ipv4_addr, 993)
+                ],
+                "imap",
+                "nine.testrun.org"
+            )
+            .await
+            .unwrap(),
+            vec![
+                SocketAddr::new(ipv4_addr, 993),
+                SocketAddr::new(ipv6_addr, 993),
+            ]
+        );
+
+        assert_eq!(
+            sort_by_connection_timestamp(
+                alice,
+                vec![
+                    SocketAddr::new(ipv6_addr, 465),
+                    SocketAddr::new(ipv4_addr, 465)
+                ],
+                "smtp",
+                "nine.testrun.org"
+            )
+            .await
+            .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 465),
+                SocketAddr::new(ipv4_addr, 465),
+            ]
+        );
+        update_connection_history(
+            alice,
+            "smtp",
+            "nine.testrun.org",
+            465,
+            "116.202.233.236",
+            now,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            sort_by_connection_timestamp(
+                alice,
+                vec![
+                    SocketAddr::new(ipv6_addr, 465),
+                    SocketAddr::new(ipv4_addr, 465)
+                ],
+                "smtp",
+                "nine.testrun.org"
+            )
+            .await
+            .unwrap(),
+            vec![
+                SocketAddr::new(ipv4_addr, 465),
+                SocketAddr::new(ipv6_addr, 465),
+            ]
+        );
+
+        update_connection_history(
+            alice,
+            "imap",
+            "nine.testrun.org",
+            993,
+            "2a01:4f8:241:4ce8::2",
+            now,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            sort_by_connection_timestamp(
+                alice,
+                vec![
+                    SocketAddr::new(ipv6_addr, 993),
+                    SocketAddr::new(ipv4_addr, 993)
+                ],
+                "imap",
+                "nine.testrun.org"
+            )
+            .await
+            .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 993),
+                SocketAddr::new(ipv4_addr, 993)
+            ]
+        );
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_lookup_cache() {
+        let alice = &TestContext::new_alice().await;
+
+        let ipv4_addr = IpAddr::V4(Ipv4Addr::new(116, 202, 233, 236));
+        let ipv6_addr = IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0x241, 0x4ce8, 0, 0, 0, 2));
+
+        let now = time();
+        assert!(lookup_cache(alice, "nine.testrun.org", 587, "smtp", now)
+            .await
+            .unwrap()
+            .is_empty());
+
+        update_cache(alice, "nine.testrun.org", "116.202.233.236", now)
+            .await
+            .unwrap();
+
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 587, "smtp", now)
+                .await
+                .unwrap(),
+            vec![SocketAddr::new(ipv4_addr, 587)]
+        );
+
+        // Cache should be returned for other ports and no ALPN as well,
+        // port and ALPN should only affect the order
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 443, "", now)
+                .await
+                .unwrap(),
+            vec![SocketAddr::new(ipv4_addr, 443)]
+        );
+
+        update_cache(alice, "nine.testrun.org", "2a01:4f8:241:4ce8::2", now + 30)
+            .await
+            .unwrap();
+
+        // New DNS cache entry should go first.
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 443, "", now + 60)
+                .await
+                .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 443),
+                SocketAddr::new(ipv4_addr, 443)
+            ],
+        );
+
+        // After successful connection to SMTP over port 465 using IPv4 address,
+        // IPv4 address has higher priority.
+        update_connection_history(
+            alice,
+            "smtp",
+            "nine.testrun.org",
+            465,
+            "116.202.233.236",
+            now + 100,
+        )
+        .await
+        .unwrap();
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 465, "smtp", now + 120)
+                .await
+                .unwrap(),
+            vec![
+                SocketAddr::new(ipv4_addr, 465),
+                SocketAddr::new(ipv6_addr, 465)
+            ]
+        );
+
+        // For other ports and ALPNs order remains the same.
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 993, "imap", now + 120)
+                .await
+                .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 993),
+                SocketAddr::new(ipv4_addr, 993)
+            ],
+        );
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 465, "imap", now + 120)
+                .await
+                .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 465),
+                SocketAddr::new(ipv4_addr, 465)
+            ],
+        );
+        assert_eq!(
+            lookup_cache(alice, "nine.testrun.org", 993, "smtp", now + 120)
+                .await
+                .unwrap(),
+            vec![
+                SocketAddr::new(ipv6_addr, 993),
+                SocketAddr::new(ipv4_addr, 993)
+            ],
+        );
+    }
+}

src/net/http.rs

@@ -1,16 +1,15 @@
 //! # HTTP module.
 
-use std::time::Duration;
+use std::sync::Arc;
 
 use anyhow::{anyhow, Result};
 use mime::Mime;
 use once_cell::sync::Lazy;
 
 use crate::context::Context;
+use crate::net::lookup_host_with_cache;
 use crate::socks::Socks5Config;
 
-const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
-
 static LETSENCRYPT_ROOT: Lazy<reqwest::tls::Certificate> = Lazy::new(|| {
     reqwest::tls::Certificate::from_der(include_bytes!(
         "../../assets/root-certificates/letsencrypt/isrgrootx1.der"
@@ -24,7 +23,7 @@ pub struct Response {
     /// Response body.
     pub blob: Vec<u8>,
 
-    /// MIME type exntracted from the `Content-Type` header, if any.
+    /// MIME type extracted from the `Content-Type` header, if any.
     pub mimetype: Option<String>,
 
     /// Encoding extracted from the `Content-Type` header, if any.
@@ -60,8 +59,13 @@ pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
 }
 
 async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Response> {
-    let socks5_config = Socks5Config::from_database(&context.sql).await?;
-    let client = get_client(socks5_config)?;
+    // It is safe to use cached IP addresses
+    // for HTTPS URLs, but for HTTP URLs
+    // better resolve from scratch each time to prevent
+    // cache poisoning attacks from having lasting effects.
+    let load_cache = url.starts_with("https://");
+
+    let client = get_client(context, load_cache).await?;
     let mut url = url.to_string();
 
     // Follow up to 10 http-redirects
@@ -86,10 +90,56 @@ async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Respons
     Err(anyhow!("Followed 10 redirections"))
 }
 
-pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
+struct CustomResolver {
+    context: Context,
+
+    /// Whether to return cached results or not.
+    /// If resolver can be used for URLs
+    /// without TLS, e.g. HTTP URLs from HTML email,
+    /// this must be false. If TLS is used
+    /// and certificate hostnames are checked,
+    /// it is safe to load cache.
+    load_cache: bool,
+}
+
+impl CustomResolver {
+    fn new(context: Context, load_cache: bool) -> Self {
+        Self {
+            context,
+            load_cache,
+        }
+    }
+}
+
+impl reqwest::dns::Resolve for CustomResolver {
+    fn resolve(&self, hostname: reqwest::dns::Name) -> reqwest::dns::Resolving {
+        let context = self.context.clone();
+        let load_cache = self.load_cache;
+        Box::pin(async move {
+            let port = 443; // Actual port does not matter.
+
+            let socket_addrs =
+                lookup_host_with_cache(&context, hostname.as_str(), port, "", load_cache).await;
+            match socket_addrs {
+                Ok(socket_addrs) => {
+                    let addrs: reqwest::dns::Addrs = Box::new(socket_addrs.into_iter());
+
+                    Ok(addrs)
+                }
+                Err(err) => Err(err.into()),
+            }
+        })
+    }
+}
+
+pub(crate) async fn get_client(context: &Context, load_cache: bool) -> Result<reqwest::Client> {
+    let socks5_config = Socks5Config::from_database(&context.sql).await?;
+    let resolver = Arc::new(CustomResolver::new(context.clone(), load_cache));
+
     let builder = reqwest::ClientBuilder::new()
-        .timeout(HTTP_TIMEOUT)
-        .add_root_certificate(LETSENCRYPT_ROOT.clone());
+        .timeout(super::TIMEOUT)
+        .add_root_certificate(LETSENCRYPT_ROOT.clone())
+        .dns_resolver(resolver);
 
     let builder = if let Some(socks5_config) = socks5_config {
         let proxy = reqwest::Proxy::all(socks5_config.to_url())?;

src/net/tls.rs

@@ -14,9 +14,10 @@ static LETSENCRYPT_ROOT: Lazy<Certificate> = Lazy::new(|| {
     .unwrap()
 });
 
-pub fn build_tls(strict_tls: bool) -> TlsConnector {
+pub fn build_tls(strict_tls: bool, alpns: &[&str]) -> TlsConnector {
     let tls_builder = TlsConnector::new()
         .min_protocol_version(Some(Protocol::Tlsv12))
+        .request_alpns(alpns)
         .add_root_certificate(LETSENCRYPT_ROOT.clone());
 
     if strict_tls {
@@ -31,9 +32,10 @@ pub fn build_tls(strict_tls: bool) -> TlsConnector {
 pub async fn wrap_tls<T: AsyncRead + AsyncWrite + Unpin>(
     strict_tls: bool,
     hostname: &str,
+    alpn: &str,
     stream: T,
 ) -> Result<TlsStream<T>> {
-    let tls = build_tls(strict_tls);
+    let tls = build_tls(strict_tls, &[alpn]);
     let tls_stream = tls.connect(hostname, stream).await?;
     Ok(tls_stream)
 }
@@ -46,7 +48,7 @@ mod tests {
     fn test_build_tls() {
         // we are using some additional root certificates.
         // make sure, they do not break construction of TlsConnector
-        let _ = build_tls(true);
-        let _ = build_tls(false);
+        let _ = build_tls(true, &[]);
+        let _ = build_tls(false, &[]);
     }
 }

src/oauth2.rs

@@ -10,7 +10,6 @@ use crate::config::Config;
 use crate::context::Context;
 use crate::provider;
 use crate::provider::Oauth2Authorizer;
-use crate::socks::Socks5Config;
 use crate::tools::time;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
@@ -159,8 +158,12 @@ pub(crate) async fn get_oauth2_access_token(
         }
 
         // ... and POST
-        let socks5_config = Socks5Config::from_database(&context.sql).await?;
-        let client = crate::net::http::get_client(socks5_config)?;
+
+        // All OAuth URLs are hardcoded HTTPS URLs,
+        // so it is safe to load DNS cache.
+        let load_cache = true;
+
+        let client = crate::net::http::get_client(context, load_cache).await?;
 
         let response: Response = match client.post(post_url).form(&post_param).send().await {
             Ok(resp) => match resp.json().await {
@@ -290,8 +293,12 @@ impl Oauth2 {
         //   "verified_email": true,
         //   "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg"
         // }
-        let socks5_config = Socks5Config::from_database(&context.sql).await.ok()?;
-        let client = match crate::net::http::get_client(socks5_config) {
+
+        // All OAuth URLs are hardcoded HTTPS URLs,
+        // so it is safe to load DNS cache.
+        let load_cache = true;
+
+        let client = match crate::net::http::get_client(context, load_cache).await {
             Ok(cl) => cl,
             Err(err) => {
                 warn!(context, "failed to get HTTP client: {}", err);

src/param.rs

@@ -88,6 +88,9 @@ pub enum Param {
     /// For Messages: quoted text.
     Quote = b'q',
 
+    /// For Messages: the 1st part of summary text (i.e. before the dash if any).
+    Summary1 = b'4',
+
     /// For Messages
     Cmd = b'S',
 

src/peer_channels.rs

@@ -25,14 +25,17 @@
 
 use anyhow::{anyhow, Context as _, Result};
 use email::Header;
-use iroh_gossip::net::{Gossip, JoinTopicFut, GOSSIP_ALPN};
-use iroh_gossip::proto::{Event as IrohEvent, TopicId};
+use futures_lite::StreamExt;
+use iroh_gossip::net::{Event, Gossip, GossipEvent, JoinOptions, GOSSIP_ALPN};
+use iroh_gossip::proto::TopicId;
+use iroh_net::key::{PublicKey, SecretKey};
 use iroh_net::relay::{RelayMap, RelayUrl};
-use iroh_net::{key::SecretKey, relay::RelayMode, Endpoint};
+use iroh_net::{relay::RelayMode, Endpoint};
 use iroh_net::{NodeAddr, NodeId};
+use parking_lot::Mutex;
 use std::collections::{BTreeSet, HashMap};
 use std::env;
-use tokio::sync::RwLock;
+use tokio::sync::{oneshot, RwLock};
 use tokio::task::JoinHandle;
 use url::Url;
 
@@ -57,11 +60,16 @@ pub struct Iroh {
     /// [Gossip] needed for iroh peer channels.
     pub(crate) gossip: Gossip,
 
+    /// Sequence numbers for gossip channels.
+    pub(crate) sequence_numbers: Mutex<HashMap<TopicId, i32>>,
+
     /// Topics for which an advertisement has already been sent.
     pub(crate) iroh_channels: RwLock<HashMap<TopicId, ChannelState>>,
 
-    /// Currently used Iroh secret key
-    pub(crate) secret_key: SecretKey,
+    /// Currently used Iroh public key.
+    ///
+    /// This is attached to every message to work around `iroh_gossip` deduplication.
+    pub(crate) public_key: PublicKey,
 }
 
 impl Iroh {
@@ -79,8 +87,10 @@ impl Iroh {
         &self,
         ctx: &Context,
         msg_id: MsgId,
-    ) -> Result<Option<JoinTopicFut>> {
-        let topic = get_iroh_topic_for_msg(ctx, msg_id).await?;
+    ) -> Result<Option<oneshot::Receiver<()>>> {
+        let topic = get_iroh_topic_for_msg(ctx, msg_id)
+            .await?
+            .with_context(|| format!("Message {msg_id} has no gossip topic"))?;
 
         // Take exclusive lock to make sure
         // no other thread can create a second gossip subscription
@@ -88,56 +98,54 @@ impl Iroh {
         // Otherwise we would receive every message twice or more times.
         let mut iroh_channels = self.iroh_channels.write().await;
 
-        let seq = if let Some(channel_state) = iroh_channels.get(&topic) {
-            if channel_state.subscribe_loop.is_some() {
-                return Ok(None);
-            }
-            channel_state.seq_number
-        } else {
-            0
-        };
-
-        let peers = get_iroh_gossip_peers(ctx, msg_id).await?;
-        info!(
-            ctx,
-            "IROH_REALTIME: Joining gossip with peers: {:?}",
-            peers.iter().map(|p| p.node_id).collect::<Vec<_>>()
-        );
-
-        // Connect to all peers
-        for peer in &peers {
-            self.endpoint.add_node_addr(peer.clone())?;
+        if iroh_channels.contains_key(&topic) {
+            return Ok(None);
         }
 
-        let connect_future = self
+        let peers = get_iroh_gossip_peers(ctx, msg_id).await?;
+        let node_ids = peers.iter().map(|p| p.node_id).collect::<Vec<_>>();
+
+        info!(
+            ctx,
+            "IROH_REALTIME: Joining gossip with peers: {:?}", node_ids,
+        );
+
+        // Inform iroh of potentially new node addresses
+        for node_addr in &peers {
+            if !node_addr.info.is_empty() {
+                self.endpoint.add_node_addr(node_addr.clone())?;
+            }
+        }
+
+        let (join_tx, join_rx) = oneshot::channel();
+
+        let (gossip_sender, gossip_receiver) = self
             .gossip
-            .join(topic, peers.into_iter().map(|addr| addr.node_id).collect())
-            .await?;
+            .join_with_opts(topic, JoinOptions::with_bootstrap(node_ids))
+            .split();
 
         let ctx = ctx.clone();
-        let gossip = self.gossip.clone();
         let subscribe_loop = tokio::spawn(async move {
-            if let Err(e) = subscribe_loop(&ctx, gossip, topic, msg_id).await {
+            if let Err(e) = subscribe_loop(&ctx, gossip_receiver, topic, msg_id, join_tx).await {
                 warn!(ctx, "subscribe_loop failed: {e}")
             }
         });
 
-        iroh_channels.insert(topic, ChannelState::new(seq, subscribe_loop));
+        iroh_channels.insert(topic, ChannelState::new(subscribe_loop, gossip_sender));
 
-        Ok(Some(connect_future))
+        Ok(Some(join_rx))
     }
 
     /// Add gossip peers to realtime channel if it is already active.
     pub async fn maybe_add_gossip_peers(&self, topic: TopicId, peers: Vec<NodeAddr>) -> Result<()> {
-        if let Some(state) = self.iroh_channels.read().await.get(&topic) {
-            if state.subscribe_loop.is_some() {
-                for peer in &peers {
-                    self.endpoint.add_node_addr(peer.clone())?;
-                }
-                self.gossip
-                    .join(topic, peers.into_iter().map(|peer| peer.node_id).collect())
-                    .await?;
+        if self.iroh_channels.read().await.get(&topic).is_some() {
+            for peer in &peers {
+                self.endpoint.add_node_addr(peer.clone())?;
             }
+
+            self.gossip
+                .join(topic, peers.into_iter().map(|peer| peer.node_id).collect())
+                .await?;
         }
         Ok(())
     }
@@ -149,14 +157,21 @@ impl Iroh {
         msg_id: MsgId,
         mut data: Vec<u8>,
     ) -> Result<()> {
-        let topic = get_iroh_topic_for_msg(ctx, msg_id).await?;
+        let topic = get_iroh_topic_for_msg(ctx, msg_id)
+            .await?
+            .with_context(|| format!("Message {msg_id} has no gossip topic"))?;
         self.join_and_subscribe_gossip(ctx, msg_id).await?;
 
-        let seq_num = self.get_and_incr(&topic).await;
-        data.extend(seq_num.to_le_bytes());
-        data.extend(self.secret_key.public().as_bytes());
+        let seq_num = self.get_and_incr(&topic);
 
-        self.gossip.broadcast(topic, data.into()).await?;
+        let mut iroh_channels = self.iroh_channels.write().await;
+        let state = iroh_channels
+            .get_mut(&topic)
+            .context("Just created state does not exist")?;
+        data.extend(seq_num.to_le_bytes());
+        data.extend(self.public_key.as_bytes());
+
+        state.sender.broadcast(data.into()).await?;
 
         if env::var("REALTIME_DEBUG").is_ok() {
             info!(ctx, "Sent realtime data");
@@ -165,30 +180,33 @@ impl Iroh {
         Ok(())
     }
 
-    async fn get_and_incr(&self, topic: &TopicId) -> i32 {
-        let mut seq = 0;
-        if let Some(state) = self.iroh_channels.write().await.get_mut(topic) {
-            seq = state.seq_number;
-            state.seq_number = state.seq_number.wrapping_add(1)
-        }
-        seq
+    fn get_and_incr(&self, topic: &TopicId) -> i32 {
+        let mut sequence_numbers = self.sequence_numbers.lock();
+        let entry = sequence_numbers.entry(*topic).or_default();
+        *entry = entry.wrapping_add(1);
+        *entry
     }
 
     /// Get the iroh [NodeAddr] without direct IP addresses.
     pub(crate) async fn get_node_addr(&self) -> Result<NodeAddr> {
-        let mut addr = self.endpoint.my_addr().await?;
+        let mut addr = self.endpoint.node_addr().await?;
         addr.info.direct_addresses = BTreeSet::new();
         Ok(addr)
     }
 
     /// Leave the realtime channel for a given topic.
     pub(crate) async fn leave_realtime(&self, topic: TopicId) -> Result<()> {
-        if let Some(channel) = &mut self.iroh_channels.write().await.get_mut(&topic) {
-            if let Some(subscribe_loop) = channel.subscribe_loop.take() {
-                subscribe_loop.abort();
-            }
+        if let Some(channel) = self.iroh_channels.write().await.remove(&topic) {
+            // Dropping the last GossipTopic results in quitting the topic.
+            // It is split into GossipReceiver and GossipSender.
+            // GossipSender (`channel.sender`) is dropped automatically.
+
+            // Subscribe loop owns GossipReceiver.
+            // Aborting it and waiting for it to be dropped
+            // drops the receiver.
+            channel.subscribe_loop.abort();
+            let _ = channel.subscribe_loop.await;
         }
-        self.gossip.quit(topic).await?;
         Ok(())
     }
 }
@@ -196,25 +214,26 @@ impl Iroh {
 /// Single gossip channel state.
 #[derive(Debug)]
 pub(crate) struct ChannelState {
-    /// Sequence number for the gossip channel.
-    seq_number: i32,
     /// The subscribe loop handle.
-    subscribe_loop: Option<JoinHandle<()>>,
+    subscribe_loop: JoinHandle<()>,
+
+    sender: iroh_gossip::net::GossipSender,
 }
 
 impl ChannelState {
-    fn new(seq_number: i32, subscribe_loop: JoinHandle<()>) -> Self {
+    fn new(subscribe_loop: JoinHandle<()>, sender: iroh_gossip::net::GossipSender) -> Self {
         Self {
-            seq_number,
-            subscribe_loop: Some(subscribe_loop),
+            subscribe_loop,
+            sender,
         }
     }
 }
 
 impl Context {
-    /// Create magic endpoint and gossip.
+    /// Create iroh endpoint and gossip.
     async fn init_peer_channels(&self) -> Result<Iroh> {
-        let secret_key: SecretKey = SecretKey::generate();
+        let secret_key = SecretKey::generate();
+        let public_key = secret_key.public();
 
         let relay_mode = if let Some(relay_url) = self
             .metadata
@@ -231,14 +250,14 @@ impl Context {
         };
 
         let endpoint = Endpoint::builder()
-            .secret_key(secret_key.clone())
+            .secret_key(secret_key)
             .alpns(vec![GOSSIP_ALPN.to_vec()])
             .relay_mode(relay_mode)
             .bind(0)
             .await?;
 
         // create gossip
-        let my_addr = endpoint.my_addr().await?;
+        let my_addr = endpoint.node_addr().await?;
         let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default(), &my_addr.info);
 
         // spawn endpoint loop that forwards incoming connections to the gossiper
@@ -246,12 +265,14 @@ impl Context {
 
         // Shuts down on deltachat shutdown
         tokio::spawn(endpoint_loop(context, endpoint.clone(), gossip.clone()));
+        tokio::spawn(gossip_direct_address_loop(endpoint.clone(), gossip.clone()));
 
         Ok(Iroh {
             endpoint,
             gossip,
+            sequence_numbers: Mutex::new(HashMap::new()),
             iroh_channels: RwLock::new(HashMap::new()),
-            secret_key,
+            public_key,
         })
     }
 
@@ -264,6 +285,15 @@ impl Context {
     }
 }
 
+/// Loop to update direct addresses of the gossip.
+async fn gossip_direct_address_loop(endpoint: Endpoint, gossip: Gossip) -> Result<()> {
+    let mut stream = endpoint.direct_addresses();
+    while let Some(addrs) = stream.next().await {
+        gossip.update_direct_addresses(&addrs)?;
+    }
+    Ok(())
+}
+
 /// Cache a peers [NodeId] for one topic.
 pub(crate) async fn iroh_add_peer_for_topic(
     ctx: &Context,
@@ -321,16 +351,28 @@ async fn get_iroh_gossip_peers(ctx: &Context, msg_id: MsgId) -> Result<Vec<NodeA
 }
 
 /// Get the topic for a given [MsgId].
-pub(crate) async fn get_iroh_topic_for_msg(ctx: &Context, msg_id: MsgId) -> Result<TopicId> {
-    let bytes: Vec<u8> = ctx
+pub(crate) async fn get_iroh_topic_for_msg(
+    ctx: &Context,
+    msg_id: MsgId,
+) -> Result<Option<TopicId>> {
+    if let Some(bytes) = ctx
         .sql
-        .query_get_value(
+        .query_get_value::<Vec<u8>>(
             "SELECT topic FROM iroh_gossip_peers WHERE msg_id = ? LIMIT 1",
             (msg_id,),
         )
-        .await?
-        .context("couldn't restore topic from db")?;
-    Ok(TopicId::from_bytes(bytes.try_into().unwrap()))
+        .await
+        .context("Couldn't restore topic from db")?
+    {
+        let topic_id = TopicId::from_bytes(
+            bytes
+                .try_into()
+                .map_err(|_| anyhow!("Could not convert stored topic ID"))?,
+        );
+        Ok(Some(topic_id))
+    } else {
+        Ok(None)
+    }
 }
 
 /// Send a gossip advertisement to the chat that [MsgId] belongs to.
@@ -338,7 +380,7 @@ pub(crate) async fn get_iroh_topic_for_msg(ctx: &Context, msg_id: MsgId) -> Resu
 pub async fn send_webxdc_realtime_advertisement(
     ctx: &Context,
     msg_id: MsgId,
-) -> Result<Option<JoinTopicFut>> {
+) -> Result<Option<oneshot::Receiver<()>>> {
     if !ctx.get_config_bool(Config::WebxdcRealtimeEnabled).await? {
         return Ok(None);
     }
@@ -372,10 +414,11 @@ pub async fn leave_webxdc_realtime(ctx: &Context, msg_id: MsgId) -> Result<()> {
     if !ctx.get_config_bool(Config::WebxdcRealtimeEnabled).await? {
         return Ok(());
     }
-
+    let topic = get_iroh_topic_for_msg(ctx, msg_id)
+        .await?
+        .with_context(|| format!("Message {msg_id} has no gossip topic"))?;
     let iroh = ctx.get_or_try_init_peer_channel().await?;
-    iroh.leave_realtime(get_iroh_topic_for_msg(ctx, msg_id).await?)
-        .await?;
+    iroh.leave_realtime(topic).await?;
     info!(ctx, "IROH_REALTIME: Left gossip for message {msg_id}");
 
     Ok(())
@@ -419,11 +462,11 @@ async fn handle_connection(
     let conn = conn.await?;
     let peer_id = iroh_net::endpoint::get_remote_node_id(&conn)?;
 
-    match alpn.as_bytes() {
+    match alpn.as_slice() {
         GOSSIP_ALPN => gossip
             .handle_connection(conn)
             .await
-            .context(format!("Connection to {peer_id} with ALPN {alpn} failed"))?,
+            .context(format!("Gossip connection to {peer_id} failed"))?,
         _ => warn!(
             context,
             "Ignoring connection from {peer_id}: unsupported ALPN protocol"
@@ -434,32 +477,50 @@ async fn handle_connection(
 
 async fn subscribe_loop(
     context: &Context,
-    gossip: Gossip,
+    mut stream: iroh_gossip::net::GossipReceiver,
     topic: TopicId,
     msg_id: MsgId,
+    join_tx: oneshot::Sender<()>,
 ) -> Result<()> {
-    let mut stream = gossip.subscribe(topic).await?;
-    loop {
-        let event = stream.recv().await?;
+    let mut join_tx = Some(join_tx);
+
+    while let Some(event) = stream.try_next().await? {
         match event {
-            IrohEvent::NeighborUp(node) => {
-                info!(context, "IROH_REALTIME: NeighborUp: {}", node.to_string());
-                iroh_add_peer_for_topic(context, msg_id, topic, node, None).await?;
+            Event::Gossip(event) => match event {
+                GossipEvent::Joined(nodes) => {
+                    if let Some(join_tx) = join_tx.take() {
+                        // Try to notify that at least one peer joined,
+                        // but ignore the error if receiver is dropped and nobody listens.
+                        join_tx.send(()).ok();
+                    }
+
+                    for node in nodes {
+                        iroh_add_peer_for_topic(context, msg_id, topic, node, None).await?;
+                    }
+                }
+                GossipEvent::NeighborUp(node) => {
+                    info!(context, "IROH_REALTIME: NeighborUp: {}", node.to_string());
+                    iroh_add_peer_for_topic(context, msg_id, topic, node, None).await?;
+                }
+                GossipEvent::NeighborDown(_node) => {}
+                GossipEvent::Received(message) => {
+                    info!(context, "IROH_REALTIME: Received realtime data");
+                    context.emit_event(EventType::WebxdcRealtimeData {
+                        msg_id,
+                        data: message
+                            .content
+                            .get(0..message.content.len() - 4 - PUBLIC_KEY_LENGTH)
+                            .context("too few bytes in iroh message")?
+                            .into(),
+                    });
+                }
+            },
+            Event::Lagged => {
+                warn!(context, "Gossip lost some messages");
             }
-            IrohEvent::Received(event) => {
-                info!(context, "IROH_REALTIME: Received realtime data");
-                context.emit_event(EventType::WebxdcRealtimeData {
-                    msg_id,
-                    data: event
-                        .content
-                        .get(0..event.content.len() - 4 - PUBLIC_KEY_LENGTH)
-                        .context("too few bytes in iroh message")?
-                        .into(),
-                });
-            }
-            _ => (),
         };
     }
+    Ok(())
 }
 
 #[cfg(test)]
@@ -507,10 +568,10 @@ mod tests {
         assert_eq!(alice_webxdc.get_viewtype(), Viewtype::Webxdc);
 
         let webxdc = alice.pop_sent_msg().await;
-        let bob_webdxc = bob.recv_msg(&webxdc).await;
-        assert_eq!(bob_webdxc.get_viewtype(), Viewtype::Webxdc);
+        let bob_webxdc = bob.recv_msg(&webxdc).await;
+        assert_eq!(bob_webxdc.get_viewtype(), Viewtype::Webxdc);
 
-        bob_webdxc.chat_id.accept(bob).await.unwrap();
+        bob_webxdc.chat_id.accept(bob).await.unwrap();
 
         // Alice advertises herself.
         send_webxdc_realtime_advertisement(alice, alice_webxdc.id)
@@ -521,7 +582,7 @@ mod tests {
         let bob_iroh = bob.get_or_try_init_peer_channel().await.unwrap();
 
         // Bob adds alice to gossip peers.
-        let members = get_iroh_gossip_peers(bob, bob_webdxc.id)
+        let members = get_iroh_gossip_peers(bob, bob_webxdc.id)
             .await
             .unwrap()
             .into_iter()
@@ -535,7 +596,7 @@ mod tests {
         );
 
         bob_iroh
-            .join_and_subscribe_gossip(bob, bob_webdxc.id)
+            .join_and_subscribe_gossip(bob, bob_webxdc.id)
             .await
             .unwrap()
             .unwrap()
@@ -563,7 +624,7 @@ mod tests {
         }
         // Bob sends ephemeral message
         bob_iroh
-            .send_webxdc_realtime_data(bob, bob_webdxc.id, "bob -> alice".as_bytes().to_vec())
+            .send_webxdc_realtime_data(bob, bob_webxdc.id, "bob -> alice".as_bytes().to_vec())
             .await
             .unwrap();
 
@@ -595,7 +656,7 @@ mod tests {
         );
 
         bob_iroh
-            .send_webxdc_realtime_data(bob, bob_webdxc.id, "bob -> alice 2".as_bytes().to_vec())
+            .send_webxdc_realtime_data(bob, bob_webxdc.id, "bob -> alice 2".as_bytes().to_vec())
             .await
             .unwrap();
 
@@ -653,10 +714,10 @@ mod tests {
         assert_eq!(alice_webxdc.get_viewtype(), Viewtype::Webxdc);
 
         let webxdc = alice.pop_sent_msg().await;
-        let bob_webdxc = bob.recv_msg(&webxdc).await;
-        assert_eq!(bob_webdxc.get_viewtype(), Viewtype::Webxdc);
+        let bob_webxdc = bob.recv_msg(&webxdc).await;
+        assert_eq!(bob_webxdc.get_viewtype(), Viewtype::Webxdc);
 
-        bob_webdxc.chat_id.accept(bob).await.unwrap();
+        bob_webxdc.chat_id.accept(bob).await.unwrap();
 
         // Alice advertises herself.
         send_webxdc_realtime_advertisement(alice, alice_webxdc.id)
@@ -667,7 +728,7 @@ mod tests {
         let bob_iroh = bob.get_or_try_init_peer_channel().await.unwrap();
 
         // Bob adds alice to gossip peers.
-        let members = get_iroh_gossip_peers(bob, bob_webdxc.id)
+        let members = get_iroh_gossip_peers(bob, bob_webxdc.id)
             .await
             .unwrap()
             .into_iter()
@@ -681,7 +742,7 @@ mod tests {
         );
 
         bob_iroh
-            .join_and_subscribe_gossip(bob, bob_webdxc.id)
+            .join_and_subscribe_gossip(bob, bob_webxdc.id)
             .await
             .unwrap()
             .unwrap()
@@ -708,11 +769,32 @@ mod tests {
             }
         }
 
-        // TODO: check that seq number is persisted
-        leave_webxdc_realtime(bob, bob_webdxc.id).await.unwrap();
+        let bob_topic = get_iroh_topic_for_msg(bob, bob_webxdc.id)
+            .await
+            .unwrap()
+            .unwrap();
+        let bob_sequence_number = bob
+            .iroh
+            .get()
+            .unwrap()
+            .sequence_numbers
+            .lock()
+            .get(&bob_topic)
+            .copied();
+        leave_webxdc_realtime(bob, bob_webxdc.id).await.unwrap();
+        let bob_sequence_number_after = bob
+            .iroh
+            .get()
+            .unwrap()
+            .sequence_numbers
+            .lock()
+            .get(&bob_topic)
+            .copied();
+        // Check that sequence number is persisted when leaving the channel.
+        assert_eq!(bob_sequence_number, bob_sequence_number_after);
 
         bob_iroh
-            .join_and_subscribe_gossip(bob, bob_webdxc.id)
+            .join_and_subscribe_gossip(bob, bob_webxdc.id)
             .await
             .unwrap()
             .unwrap()
@@ -720,7 +802,7 @@ mod tests {
             .unwrap();
 
         bob_iroh
-            .send_webxdc_realtime_data(bob, bob_webdxc.id, "bob -> alice".as_bytes().to_vec())
+            .send_webxdc_realtime_data(bob, bob_webxdc.id, "bob -> alice".as_bytes().to_vec())
             .await
             .unwrap();
 
@@ -748,8 +830,9 @@ mod tests {
         leave_webxdc_realtime(alice, alice_webxdc.id).await.unwrap();
         let topic = get_iroh_topic_for_msg(alice, alice_webxdc.id)
             .await
+            .unwrap()
             .unwrap();
-        assert!(if let Some(state) = alice
+        assert!(alice
             .iroh
             .get()
             .unwrap()
@@ -757,11 +840,7 @@ mod tests {
             .read()
             .await
             .get(&topic)
-        {
-            state.subscribe_loop.is_none()
-        } else {
-            false
-        });
+            .is_none());
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

src/pgp.rs

@@ -11,6 +11,7 @@ use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
     SignedPublicSubKey, SignedSecretKey, StandaloneSignature, SubkeyParamsBuilder,
 };
+use pgp::crypto::ecc_curve::ECCCurve;
 use pgp::crypto::hash::HashAlgorithm;
 use pgp::crypto::sym::SymmetricKeyAlgorithm;
 use pgp::types::{
@@ -115,7 +116,14 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, Str
     let headers = dearmor
         .headers
         .into_iter()
-        .map(|(key, value)| (key.trim().to_lowercase(), value.trim().to_string()))
+        .map(|(key, values)| {
+            (
+                key.trim().to_lowercase(),
+                values
+                    .last()
+                    .map_or_else(String::new, |s| s.trim().to_string()),
+            )
+        })
         .collect();
 
     Ok((typ, headers, bytes))
@@ -145,7 +153,9 @@ pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Res
     let (signing_key_type, encryption_key_type) = match keygen_type {
         KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
         KeyGenType::Rsa4096 => (PgpKeyType::Rsa(4096), PgpKeyType::Rsa(4096)),
-        KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
+        KeyGenType::Ed25519 | KeyGenType::Default => {
+            (PgpKeyType::EdDSA, PgpKeyType::ECDH(ECCCurve::Curve25519))
+        }
     };
 
     let user_id = format!("<{addr}>");
@@ -262,7 +272,7 @@ pub async fn pk_encrypt(
                 lit_msg.encrypt_to_keys(&mut rng, SYMMETRIC_KEY_ALGORITHM, &pkeys_refs)?
             };
 
-            let encoded_msg = encrypted_msg.to_armored_string(None)?;
+            let encoded_msg = encrypted_msg.to_armored_string(Default::default())?;
 
             Ok(encoded_msg)
         })
@@ -279,7 +289,7 @@ pub fn pk_calc_signature(
         || "".into(),
         HASH_ALGORITHM,
     )?;
-    let signature = msg.into_signature().to_armored_string(None)?;
+    let signature = msg.into_signature().to_armored_string(Default::default())?;
     Ok(signature)
 }
 
@@ -304,31 +314,26 @@ pub fn pk_decrypt(
 
     let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.iter().collect();
 
-    let (decryptor, _) = msg.decrypt(|| "".into(), &skeys[..])?;
-    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
+    let (msg, _) = msg.decrypt(|| "".into(), &skeys[..])?;
 
-    if let Some(msg) = msgs.into_iter().next() {
-        // get_content() will decompress the message if needed,
-        // but this avoids decompressing it again to check signatures
-        let msg = msg.decompress()?;
+    // get_content() will decompress the message if needed,
+    // but this avoids decompressing it again to check signatures
+    let msg = msg.decompress()?;
 
-        let content = match msg.get_content()? {
-            Some(content) => content,
-            None => bail!("The decrypted message is empty"),
-        };
+    let content = match msg.get_content()? {
+        Some(content) => content,
+        None => bail!("The decrypted message is empty"),
+    };
 
-        if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
-            for pkey in public_keys_for_validation {
-                if signed_msg.verify(&pkey.primary_key).is_ok() {
-                    let fp = DcKey::fingerprint(pkey);
-                    ret_signature_fingerprints.insert(fp);
-                }
+    if let signed_msg @ pgp::composed::Message::Signed { .. } = msg {
+        for pkey in public_keys_for_validation {
+            if signed_msg.verify(&pkey.primary_key).is_ok() {
+                let fp = DcKey::fingerprint(pkey);
+                ret_signature_fingerprints.insert(fp);
             }
         }
-        Ok((content, ret_signature_fingerprints))
-    } else {
-        bail!("No valid messages found");
     }
+    Ok((content, ret_signature_fingerprints))
 }
 
 /// Validates detached signature.
@@ -368,7 +373,7 @@ pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
         let msg =
             lit_msg.encrypt_with_password(&mut rng, s2k, SYMMETRIC_KEY_ALGORITHM, || passphrase)?;
 
-        let encoded_msg = msg.to_armored_string(None)?;
+        let encoded_msg = msg.to_armored_string(Default::default())?;
 
         Ok(encoded_msg)
     })
@@ -384,16 +389,11 @@ pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
 
     let passphrase = passphrase.to_string();
     tokio::task::spawn_blocking(move || {
-        let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
+        let msg = enc_msg.decrypt_with_password(|| passphrase)?;
 
-        let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
-        if let Some(msg) = msgs.first() {
-            match msg.get_content()? {
-                Some(content) => Ok(content),
-                None => bail!("Decrypted message is empty"),
-            }
-        } else {
-            bail!("No valid messages found")
+        match msg.get_content()? {
+            Some(content) => Ok(content),
+            None => bail!("Decrypted message is empty"),
         }
     })
     .await?
@@ -410,7 +410,7 @@ mod tests {
     #[test]
     fn test_split_armored_data_1() {
         let (typ, _headers, base64) = split_armored_data(
-            b"-----BEGIN PGP MESSAGE-----\nNoVal:\n\naGVsbG8gd29ybGQ=\n-----END PGP MESSAGE----",
+            b"-----BEGIN PGP MESSAGE-----\nNoVal:\n\naGVsbG8gd29ybGQ=\n-----END PGP MESSAGE-----",
         )
         .unwrap();
 

src/provider/data.rs

@@ -66,6 +66,34 @@ static P_AKTIVIX_ORG: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// aliyun.md: aliyun.com
+static P_ALIYUN: Provider = Provider {
+    id: "aliyun",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/aliyun",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "imap.aliyun.com",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "smtp.aliyun.com",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // aol.md: aol.com
 static P_AOL: Provider = Provider {
     id: "aol",
@@ -222,99 +250,6 @@ static P_BUZON_UY: Provider = Provider {
     oauth2_authorizer: None,
 };
 
-// c1.testrun.org.md: c1.testrun.org
-static P_C1_TESTRUN_ORG: Provider = Provider {
-    id: "c1.testrun.org",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/c1-testrun-org",
-    server: &[
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "c1.testrun.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "c1.testrun.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    opt: ProviderOptions::new(),
-    config_defaults: Some(&[ConfigDefault {
-        key: Config::MvboxMove,
-        value: "0",
-    }]),
-    oauth2_authorizer: None,
-};
-
-// c2.testrun.org.md: c2.testrun.org
-static P_C2_TESTRUN_ORG: Provider = Provider {
-    id: "c2.testrun.org",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/c2-testrun-org",
-    server: &[
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "c2.testrun.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "c2.testrun.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    opt: ProviderOptions::new(),
-    config_defaults: Some(&[ConfigDefault {
-        key: Config::MvboxMove,
-        value: "0",
-    }]),
-    oauth2_authorizer: None,
-};
-
-// c3.testrun.org.md: c3.testrun.org
-static P_C3_TESTRUN_ORG: Provider = Provider {
-    id: "c3.testrun.org",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/c3-testrun-org",
-    server: &[
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "c3.testrun.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "c3.testrun.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    opt: ProviderOptions::new(),
-    config_defaults: Some(&[ConfigDefault {
-        key: Config::MvboxMove,
-        value: "0",
-    }]),
-    oauth2_authorizer: None,
-};
-
 // chello.at.md: chello.at
 static P_CHELLO_AT: Provider = Provider {
     id: "chello.at",
@@ -356,6 +291,48 @@ static P_COMCAST: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// daleth.cafe.md: daleth.cafe
+static P_DALETH_CAFE: Provider = Provider {
+    id: "daleth.cafe",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/daleth-cafe",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "daleth.cafe",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "daleth.cafe",
+            port: 465,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Imap,
+            socket: Starttls,
+            hostname: "daleth.cafe",
+            port: 143,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Starttls,
+            hostname: "daleth.cafe",
+            port: 587,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // dismail.de.md: dismail.de
 static P_DISMAIL_DE: Provider = Provider {
     id: "dismail.de",
@@ -382,14 +359,14 @@ static P_DISROOT: Provider = Provider {
             socket: Ssl,
             hostname: "disroot.org",
             port: 993,
-            username_pattern: Email,
+            username_pattern: Emaillocalpart,
         },
         Server {
             protocol: Smtp,
             socket: Starttls,
             hostname: "disroot.org",
             port: 587,
-            username_pattern: Email,
+            username_pattern: Emaillocalpart,
         },
     ],
     opt: ProviderOptions::new(),
@@ -446,7 +423,7 @@ static P_EXAMPLE_COM: Provider = Provider {
     after_login_hint: "This provider doesn't really exist, so you can't use it :/ If you need an email provider for Delta Chat, take a look at providers.delta.chat!",
     overview_page: "https://providers.delta.chat/example-com",
     server: &[
-        Server { protocol: Imap, socket: Ssl, hostname: "imap.example.com", port: 1337, username_pattern: Email },
+        Server { protocol: Imap, socket: Ssl, hostname: "imap.example.com", port: 1337, username_pattern: Emaillocalpart },
         Server { protocol: Smtp, socket: Starttls, hostname: "smtp.example.com", port: 1337, username_pattern: Email },
     ],
     opt: ProviderOptions::new(),
@@ -543,7 +520,7 @@ static P_FREENET_DE: Provider = Provider {
 static P_GMAIL: Provider = Provider {
     id: "gmail",
     status: Status::Preparation,
-    before_login_hint: "For Gmail accounts, you need to create an app-password if you have \"2-Step Verification\" enabled. If this setting is not available, you need to enable \"less secure apps\".",
+    before_login_hint: "For Gmail accounts, you need to have \"2-Step Verification\" enabled and create an app-password.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/gmail",
     server: &[
@@ -747,6 +724,20 @@ static P_KONTENT_COM: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// mail.com.md: email.com, groupmail.com, post.com, homemail.com, housemail.com, writeme.com, mail.com, mail-me.com, workmail.com, accountant.com, activist.com, adexec.com, allergist.com, alumni.com, alumnidirector.com, archaeologist.com, auctioneer.net, bartender.net, brew-master.com, chef.net, chemist.com, collector.org, columnist.com, comic.com, consultant.com, contractor.net, counsellor.com, deliveryman.com, diplomats.com, dr.com, engineer.com, financier.com, fireman.net, gardener.com, geologist.com, graphic-designer.com, graduate.org, hairdresser.net, instructor.net, insurer.com, journalist.com, legislator.com, lobbyist.com, minister.com, musician.org, optician.com, orthodontist.net, pediatrician.com, photographer.net, physicist.net, politician.com, presidency.com, priest.com, programmer.net, publicist.com, radiologist.net, realtyagent.com, registerednurses.com, repairman.com, representative.com, salesperson.net, secretary.net, socialworker.net, sociologist.com, songwriter.net, teachers.org, techie.com, technologist.com, therapist.net, umpire.com, worker.com, artlover.com, bikerider.com, birdlover.com, blader.com, kittymail.com, lovecat.com, marchmail.com, boardermail.com, catlover.com, clubmember.org, nonpartisan.com, petlover.com, doglover.com, greenmail.net, hackermail.com, theplate.com, bsdmail.com, computer4u.com, coolsite.net, cyberdude.com, cybergal.com, cyberservices.com, cyber-wizard.com, linuxmail.org, null.net, solution4u.com, tech-center.com, webname.com, acdcfan.com, angelic.com, discofan.com, elvisfan.com, hiphopfan.com, kissfans.com, madonnafan.com, metalfan.com, ninfan.com, ravemail.com, reggaefan.com, snakebite.com, bellair.net, californiamail.com, dallasmail.com, nycmail.com, pacific-ocean.com, pacificwest.com, sanfranmail.com, usa.com, africamail.com, asia-mail.com, australiamail.com, berlin.com, brazilmail.com, chinamail.com, dublin.com, dutchmail.com, englandmail.com, europe.com, arcticmail.com, europemail.com, germanymail.com, irelandmail.com, israelmail.com, italymail.com, koreamail.com, mexicomail.com, moscowmail.com, munich.com, asia.com, polandmail.com, safrica.com, samerica.com, scotlandmail.com, spainmail.com, swedenmail.com, swissmail.com, torontomail.com, aircraftmail.com, cash4u.com, disposable.com, execs.com, fastservice.com, instruction.com, job4u.com, net-shopping.com, planetmail.com, planetmail.net, qualityservice.com, rescueteam.com, surgical.net, atheist.com, disciples.com, muslim.com, protestant.com, reborn.com, reincarnate.com, religious.com, saintly.com, brew-meister.com, cutey.com, dbzmail.com, doramail.com, galaxyhit.com, hilarious.com, humanoid.net, hot-shot.com, inorbit.com, iname.com, innocent.com, keromail.com, myself.com, rocketship.com, toothfairy.com, toke.com, tvstar.com, uymail.com, 2trom.com
+static P_MAIL_COM: Provider = Provider {
+    id: "mail.com",
+    status: Status::Preparation,
+    before_login_hint: "To log in with Delta Chat, you first need to activate POP3/IMAP in your mail.com settings. Note that this is a mail.com Premium feature only.",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/mail-com",
+    server: &[
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // mail.de.md: mail.de
 static P_MAIL_DE: Provider = Provider {
     id: "mail.de",
@@ -875,6 +866,64 @@ static P_MAILO_COM: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// mehl.cloud.md: mehl.cloud
+static P_MEHL_CLOUD: Provider = Provider {
+    id: "mehl.cloud",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/mehl-cloud",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "mehl.cloud",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "mehl.cloud",
+            port: 465,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Imap,
+            socket: Starttls,
+            hostname: "mehl.cloud",
+            port: 143,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Starttls,
+            hostname: "mehl.cloud",
+            port: 587,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
+// mehl.store.md: mehl.store, ende.in.net, l2i.top, szh.homes, sls.post.in, ente.quest, ente.cfd, nein.jetzt
+static P_MEHL_STORE: Provider = Provider {
+    id: "mehl.store",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "This account provides 3GB storage for eMails and the possibility to access a NEXTCLOUD-instance by using the email-credits!",
+    overview_page: "https://providers.delta.chat/mehl-store",
+    server: &[
+        Server { protocol: Imap, socket: Ssl, hostname: "mail.ende.in.net", port: 993, username_pattern: Email },
+        Server { protocol: Smtp, socket: Starttls, hostname: "mail.ende.in.net", port: 587, username_pattern: Email },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // nauta.cu.md: nauta.cu
 static P_NAUTA_CU: Provider = Provider {
     id: "nauta.cu",
@@ -908,10 +957,6 @@ static P_NAUTA_CU: Provider = Provider {
             key: Config::DeleteServerAfter,
             value: "1",
         },
-        ConfigDefault {
-            key: Config::BccSelf,
-            value: "0",
-        },
         ConfigDefault {
             key: Config::SentboxWatch,
             value: "0",
@@ -924,10 +969,6 @@ static P_NAUTA_CU: Provider = Provider {
             key: Config::MediaQuality,
             value: "1",
         },
-        ConfigDefault {
-            key: Config::FetchExistingMsgs,
-            value: "0",
-        },
     ]),
     oauth2_authorizer: None,
 };
@@ -982,6 +1023,34 @@ static P_NINE_TESTRUN_ORG: Provider = Provider {
             port: 465,
             username_pattern: Email,
         },
+        Server {
+            protocol: Imap,
+            socket: Starttls,
+            hostname: "nine.testrun.org",
+            port: 143,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Starttls,
+            hostname: "nine.testrun.org",
+            port: 587,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "nine.testrun.org",
+            port: 443,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "nine.testrun.org",
+            port: 443,
+            username_pattern: Email,
+        },
     ],
     opt: ProviderOptions::new(),
     config_defaults: Some(&[ConfigDefault {
@@ -1131,6 +1200,34 @@ static P_PROTONMAIL: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// purelymail.com.md: purelymail.com, cheapermail.com, placeq.com, rethinkmail.com, worldofmail.com
+static P_PURELYMAIL_COM: Provider = Provider {
+    id: "purelymail.com",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/purelymail-com",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "imap.purelymail.com",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "smtp.purelymail.com",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // qq.md: qq.com, foxmail.com
 static P_QQ: Provider = Provider {
     id: "qq",
@@ -1147,6 +1244,23 @@ static P_QQ: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// rambler.ru.md: rambler.ru, autorambler.ru, myrambler.ru, rambler.ua, lenta.ru, ro.ru, r0.ru
+static P_RAMBLER_RU: Provider = Provider {
+    id: "rambler.ru",
+    status: Status::Preparation,
+    before_login_hint: "Чтобы войти в Рамблер/почта через Delta Chat, необходимо предварительно включить доступ с помощью почтовых клиентов на сайте mail.rambler.ru",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/rambler-ru",
+    server: &[
+        Server { protocol: Imap, socket: Ssl, hostname: "imap.rambler.ru", port: 993, username_pattern: Email },
+        Server { protocol: Smtp, socket: Ssl, hostname: "smtp.rambler.ru", port: 465, username_pattern: Email },
+        Server { protocol: Imap, socket: Starttls, hostname: "imap.rambler.ru", port: 143, username_pattern: Email },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // riseup.net.md: riseup.net
 static P_RISEUP_NET: Provider = Provider {
     id: "riseup.net",
@@ -1201,6 +1315,37 @@ static P_SONIC: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// stinpriza.net.md: stinpriza.net, stinpriza.eu, el-hoyo.net
+static P_STINPRIZA_NET: Provider = Provider {
+    id: "stinpriza.net",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/stinpriza-net",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Starttls,
+            hostname: "stinpriza.net",
+            port: 143,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Starttls,
+            hostname: "stinpriza.net",
+            port: 587,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions {
+        strict_tls: true,
+        ..ProviderOptions::new()
+    },
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // systemausfall.org.md: systemausfall.org, solidaris.me
 static P_SYSTEMAUSFALL_ORG: Provider = Provider {
     id: "systemausfall.org",
@@ -1288,6 +1433,13 @@ static P_TESTRUN: Provider = Provider {
             port: 993,
             username_pattern: Email,
         },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
         Server {
             protocol: Imap,
             socket: Starttls,
@@ -1445,6 +1597,22 @@ static P_VIVALDI: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// vk.com.md: vk.com
+static P_VK_COM: Provider = Provider {
+    id: "vk.com",
+    status: Status::Preparation,
+    before_login_hint: "Вам необходимо сгенерировать \"пароль для внешнего приложения\" в веб-интерфейсе mail.ru https://account.mail.ru/user/2-step-auth/passwords/ чтобы vk.com работал с Delta Chat.",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/vk-com",
+    server: &[
+        Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.ru", port: 993, username_pattern: Email },
+        Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.ru", port: 465, username_pattern: Email },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // vodafone.de.md: vodafone.de, vodafonemail.de
 static P_VODAFONE_DE: Provider = Provider {
     id: "vodafone.de",
@@ -1490,6 +1658,34 @@ static P_WEB_DE: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// wkpb.de.md: wkpb.de
+static P_WKPB_DE: Provider = Provider {
+    id: "wkpb.de",
+    status: Status::Preparation,
+    before_login_hint: "Dies sind die gleichen Anmeldedaten wie bei Moodle und Abitur-Online.",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/wkpb-de",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "pimap.schulon.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "psmtp.schulon.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: None,
+    oauth2_authorizer: None,
+};
+
 // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
 static P_YAHOO: Provider = Provider {
     id: "yahoo",
@@ -1608,9 +1804,10 @@ static P_ZOHO: Provider = Provider {
     oauth2_authorizer: None,
 };
 
-pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
+pub(crate) static PROVIDER_DATA: [(&str, &Provider); 531] = [
     ("163.com", &P_163),
     ("aktivix.org", &P_AKTIVIX_ORG),
+    ("aliyun.com", &P_ALIYUN),
     ("aol.com", &P_AOL),
     ("arcor.de", &P_ARCOR_DE),
     ("autistici.org", &P_AUTISTICI_ORG),
@@ -1618,12 +1815,10 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
     ("delta.blindzeln.org", &P_BLINDZELN_ORG),
     ("bluewin.ch", &P_BLUEWIN_CH),
     ("buzon.uy", &P_BUZON_UY),
-    ("c1.testrun.org", &P_C1_TESTRUN_ORG),
-    ("c2.testrun.org", &P_C2_TESTRUN_ORG),
-    ("c3.testrun.org", &P_C3_TESTRUN_ORG),
     ("chello.at", &P_CHELLO_AT),
     ("xfinity.com", &P_COMCAST),
     ("comcast.net", &P_COMCAST),
+    ("daleth.cafe", &P_DALETH_CAFE),
     ("dismail.de", &P_DISMAIL_DE),
     ("disroot.org", &P_DISROOT),
     ("e.email", &P_E_EMAIL),
@@ -1775,6 +1970,194 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
     ("ik.me", &P_INFOMANIAK_COM),
     ("kolst.com", &P_KOLST_COM),
     ("kontent.com", &P_KONTENT_COM),
+    ("email.com", &P_MAIL_COM),
+    ("groupmail.com", &P_MAIL_COM),
+    ("post.com", &P_MAIL_COM),
+    ("homemail.com", &P_MAIL_COM),
+    ("housemail.com", &P_MAIL_COM),
+    ("writeme.com", &P_MAIL_COM),
+    ("mail.com", &P_MAIL_COM),
+    ("mail-me.com", &P_MAIL_COM),
+    ("workmail.com", &P_MAIL_COM),
+    ("accountant.com", &P_MAIL_COM),
+    ("activist.com", &P_MAIL_COM),
+    ("adexec.com", &P_MAIL_COM),
+    ("allergist.com", &P_MAIL_COM),
+    ("alumni.com", &P_MAIL_COM),
+    ("alumnidirector.com", &P_MAIL_COM),
+    ("archaeologist.com", &P_MAIL_COM),
+    ("auctioneer.net", &P_MAIL_COM),
+    ("bartender.net", &P_MAIL_COM),
+    ("brew-master.com", &P_MAIL_COM),
+    ("chef.net", &P_MAIL_COM),
+    ("chemist.com", &P_MAIL_COM),
+    ("collector.org", &P_MAIL_COM),
+    ("columnist.com", &P_MAIL_COM),
+    ("comic.com", &P_MAIL_COM),
+    ("consultant.com", &P_MAIL_COM),
+    ("contractor.net", &P_MAIL_COM),
+    ("counsellor.com", &P_MAIL_COM),
+    ("deliveryman.com", &P_MAIL_COM),
+    ("diplomats.com", &P_MAIL_COM),
+    ("dr.com", &P_MAIL_COM),
+    ("engineer.com", &P_MAIL_COM),
+    ("financier.com", &P_MAIL_COM),
+    ("fireman.net", &P_MAIL_COM),
+    ("gardener.com", &P_MAIL_COM),
+    ("geologist.com", &P_MAIL_COM),
+    ("graphic-designer.com", &P_MAIL_COM),
+    ("graduate.org", &P_MAIL_COM),
+    ("hairdresser.net", &P_MAIL_COM),
+    ("instructor.net", &P_MAIL_COM),
+    ("insurer.com", &P_MAIL_COM),
+    ("journalist.com", &P_MAIL_COM),
+    ("legislator.com", &P_MAIL_COM),
+    ("lobbyist.com", &P_MAIL_COM),
+    ("minister.com", &P_MAIL_COM),
+    ("musician.org", &P_MAIL_COM),
+    ("optician.com", &P_MAIL_COM),
+    ("orthodontist.net", &P_MAIL_COM),
+    ("pediatrician.com", &P_MAIL_COM),
+    ("photographer.net", &P_MAIL_COM),
+    ("physicist.net", &P_MAIL_COM),
+    ("politician.com", &P_MAIL_COM),
+    ("presidency.com", &P_MAIL_COM),
+    ("priest.com", &P_MAIL_COM),
+    ("programmer.net", &P_MAIL_COM),
+    ("publicist.com", &P_MAIL_COM),
+    ("radiologist.net", &P_MAIL_COM),
+    ("realtyagent.com", &P_MAIL_COM),
+    ("registerednurses.com", &P_MAIL_COM),
+    ("repairman.com", &P_MAIL_COM),
+    ("representative.com", &P_MAIL_COM),
+    ("salesperson.net", &P_MAIL_COM),
+    ("secretary.net", &P_MAIL_COM),
+    ("socialworker.net", &P_MAIL_COM),
+    ("sociologist.com", &P_MAIL_COM),
+    ("songwriter.net", &P_MAIL_COM),
+    ("teachers.org", &P_MAIL_COM),
+    ("techie.com", &P_MAIL_COM),
+    ("technologist.com", &P_MAIL_COM),
+    ("therapist.net", &P_MAIL_COM),
+    ("umpire.com", &P_MAIL_COM),
+    ("worker.com", &P_MAIL_COM),
+    ("artlover.com", &P_MAIL_COM),
+    ("bikerider.com", &P_MAIL_COM),
+    ("birdlover.com", &P_MAIL_COM),
+    ("blader.com", &P_MAIL_COM),
+    ("kittymail.com", &P_MAIL_COM),
+    ("lovecat.com", &P_MAIL_COM),
+    ("marchmail.com", &P_MAIL_COM),
+    ("boardermail.com", &P_MAIL_COM),
+    ("catlover.com", &P_MAIL_COM),
+    ("clubmember.org", &P_MAIL_COM),
+    ("nonpartisan.com", &P_MAIL_COM),
+    ("petlover.com", &P_MAIL_COM),
+    ("doglover.com", &P_MAIL_COM),
+    ("greenmail.net", &P_MAIL_COM),
+    ("hackermail.com", &P_MAIL_COM),
+    ("theplate.com", &P_MAIL_COM),
+    ("bsdmail.com", &P_MAIL_COM),
+    ("computer4u.com", &P_MAIL_COM),
+    ("coolsite.net", &P_MAIL_COM),
+    ("cyberdude.com", &P_MAIL_COM),
+    ("cybergal.com", &P_MAIL_COM),
+    ("cyberservices.com", &P_MAIL_COM),
+    ("cyber-wizard.com", &P_MAIL_COM),
+    ("linuxmail.org", &P_MAIL_COM),
+    ("null.net", &P_MAIL_COM),
+    ("solution4u.com", &P_MAIL_COM),
+    ("tech-center.com", &P_MAIL_COM),
+    ("webname.com", &P_MAIL_COM),
+    ("acdcfan.com", &P_MAIL_COM),
+    ("angelic.com", &P_MAIL_COM),
+    ("discofan.com", &P_MAIL_COM),
+    ("elvisfan.com", &P_MAIL_COM),
+    ("hiphopfan.com", &P_MAIL_COM),
+    ("kissfans.com", &P_MAIL_COM),
+    ("madonnafan.com", &P_MAIL_COM),
+    ("metalfan.com", &P_MAIL_COM),
+    ("ninfan.com", &P_MAIL_COM),
+    ("ravemail.com", &P_MAIL_COM),
+    ("reggaefan.com", &P_MAIL_COM),
+    ("snakebite.com", &P_MAIL_COM),
+    ("bellair.net", &P_MAIL_COM),
+    ("californiamail.com", &P_MAIL_COM),
+    ("dallasmail.com", &P_MAIL_COM),
+    ("nycmail.com", &P_MAIL_COM),
+    ("pacific-ocean.com", &P_MAIL_COM),
+    ("pacificwest.com", &P_MAIL_COM),
+    ("sanfranmail.com", &P_MAIL_COM),
+    ("usa.com", &P_MAIL_COM),
+    ("africamail.com", &P_MAIL_COM),
+    ("asia-mail.com", &P_MAIL_COM),
+    ("australiamail.com", &P_MAIL_COM),
+    ("berlin.com", &P_MAIL_COM),
+    ("brazilmail.com", &P_MAIL_COM),
+    ("chinamail.com", &P_MAIL_COM),
+    ("dublin.com", &P_MAIL_COM),
+    ("dutchmail.com", &P_MAIL_COM),
+    ("englandmail.com", &P_MAIL_COM),
+    ("europe.com", &P_MAIL_COM),
+    ("arcticmail.com", &P_MAIL_COM),
+    ("europemail.com", &P_MAIL_COM),
+    ("germanymail.com", &P_MAIL_COM),
+    ("irelandmail.com", &P_MAIL_COM),
+    ("israelmail.com", &P_MAIL_COM),
+    ("italymail.com", &P_MAIL_COM),
+    ("koreamail.com", &P_MAIL_COM),
+    ("mexicomail.com", &P_MAIL_COM),
+    ("moscowmail.com", &P_MAIL_COM),
+    ("munich.com", &P_MAIL_COM),
+    ("asia.com", &P_MAIL_COM),
+    ("polandmail.com", &P_MAIL_COM),
+    ("safrica.com", &P_MAIL_COM),
+    ("samerica.com", &P_MAIL_COM),
+    ("scotlandmail.com", &P_MAIL_COM),
+    ("spainmail.com", &P_MAIL_COM),
+    ("swedenmail.com", &P_MAIL_COM),
+    ("swissmail.com", &P_MAIL_COM),
+    ("torontomail.com", &P_MAIL_COM),
+    ("aircraftmail.com", &P_MAIL_COM),
+    ("cash4u.com", &P_MAIL_COM),
+    ("disposable.com", &P_MAIL_COM),
+    ("execs.com", &P_MAIL_COM),
+    ("fastservice.com", &P_MAIL_COM),
+    ("instruction.com", &P_MAIL_COM),
+    ("job4u.com", &P_MAIL_COM),
+    ("net-shopping.com", &P_MAIL_COM),
+    ("planetmail.com", &P_MAIL_COM),
+    ("planetmail.net", &P_MAIL_COM),
+    ("qualityservice.com", &P_MAIL_COM),
+    ("rescueteam.com", &P_MAIL_COM),
+    ("surgical.net", &P_MAIL_COM),
+    ("atheist.com", &P_MAIL_COM),
+    ("disciples.com", &P_MAIL_COM),
+    ("muslim.com", &P_MAIL_COM),
+    ("protestant.com", &P_MAIL_COM),
+    ("reborn.com", &P_MAIL_COM),
+    ("reincarnate.com", &P_MAIL_COM),
+    ("religious.com", &P_MAIL_COM),
+    ("saintly.com", &P_MAIL_COM),
+    ("brew-meister.com", &P_MAIL_COM),
+    ("cutey.com", &P_MAIL_COM),
+    ("dbzmail.com", &P_MAIL_COM),
+    ("doramail.com", &P_MAIL_COM),
+    ("galaxyhit.com", &P_MAIL_COM),
+    ("hilarious.com", &P_MAIL_COM),
+    ("humanoid.net", &P_MAIL_COM),
+    ("hot-shot.com", &P_MAIL_COM),
+    ("inorbit.com", &P_MAIL_COM),
+    ("iname.com", &P_MAIL_COM),
+    ("innocent.com", &P_MAIL_COM),
+    ("keromail.com", &P_MAIL_COM),
+    ("myself.com", &P_MAIL_COM),
+    ("rocketship.com", &P_MAIL_COM),
+    ("toothfairy.com", &P_MAIL_COM),
+    ("toke.com", &P_MAIL_COM),
+    ("tvstar.com", &P_MAIL_COM),
+    ("uymail.com", &P_MAIL_COM),
+    ("2trom.com", &P_MAIL_COM),
     ("mail.de", &P_MAIL_DE),
     ("mail.ru", &P_MAIL_RU),
     ("inbox.ru", &P_MAIL_RU),
@@ -1785,6 +2168,15 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
     ("mailbox.org", &P_MAILBOX_ORG),
     ("secure.mailbox.org", &P_MAILBOX_ORG),
     ("mailo.com", &P_MAILO_COM),
+    ("mehl.cloud", &P_MEHL_CLOUD),
+    ("mehl.store", &P_MEHL_STORE),
+    ("ende.in.net", &P_MEHL_STORE),
+    ("l2i.top", &P_MEHL_STORE),
+    ("szh.homes", &P_MEHL_STORE),
+    ("sls.post.in", &P_MEHL_STORE),
+    ("ente.quest", &P_MEHL_STORE),
+    ("ente.cfd", &P_MEHL_STORE),
+    ("nein.jetzt", &P_MEHL_STORE),
     ("nauta.cu", &P_NAUTA_CU),
     ("naver.com", &P_NAVER),
     ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
@@ -1850,11 +2242,26 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
     ("protonmail.com", &P_PROTONMAIL),
     ("protonmail.ch", &P_PROTONMAIL),
     ("pm.me", &P_PROTONMAIL),
+    ("purelymail.com", &P_PURELYMAIL_COM),
+    ("cheapermail.com", &P_PURELYMAIL_COM),
+    ("placeq.com", &P_PURELYMAIL_COM),
+    ("rethinkmail.com", &P_PURELYMAIL_COM),
+    ("worldofmail.com", &P_PURELYMAIL_COM),
     ("qq.com", &P_QQ),
     ("foxmail.com", &P_QQ),
+    ("rambler.ru", &P_RAMBLER_RU),
+    ("autorambler.ru", &P_RAMBLER_RU),
+    ("myrambler.ru", &P_RAMBLER_RU),
+    ("rambler.ua", &P_RAMBLER_RU),
+    ("lenta.ru", &P_RAMBLER_RU),
+    ("ro.ru", &P_RAMBLER_RU),
+    ("r0.ru", &P_RAMBLER_RU),
     ("riseup.net", &P_RISEUP_NET),
     ("rogers.com", &P_ROGERS_COM),
     ("sonic.net", &P_SONIC),
+    ("stinpriza.net", &P_STINPRIZA_NET),
+    ("stinpriza.eu", &P_STINPRIZA_NET),
+    ("el-hoyo.net", &P_STINPRIZA_NET),
     ("systemausfall.org", &P_SYSTEMAUSFALL_ORG),
     ("solidaris.me", &P_SYSTEMAUSFALL_ORG),
     ("systemli.org", &P_SYSTEMLI_ORG),
@@ -1871,6 +2278,7 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
     ("undernet.uy", &P_UNDERNET_UY),
     ("vfemail.net", &P_VFEMAIL),
     ("vivaldi.net", &P_VIVALDI),
+    ("vk.com", &P_VK_COM),
     ("vodafone.de", &P_VODAFONE_DE),
     ("vodafonemail.de", &P_VODAFONE_DE),
     ("web.de", &P_WEB_DE),
@@ -1900,6 +2308,7 @@ pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
     ("joker.ms", &P_WEB_DE),
     ("planet.ms", &P_WEB_DE),
     ("power.ms", &P_WEB_DE),
+    ("wkpb.de", &P_WKPB_DE),
     ("yahoo.com", &P_YAHOO),
     ("yahoo.de", &P_YAHOO),
     ("yahoo.it", &P_YAHOO),
@@ -1933,17 +2342,16 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
     HashMap::from([
         ("163", &P_163),
         ("aktivix.org", &P_AKTIVIX_ORG),
+        ("aliyun", &P_ALIYUN),
         ("aol", &P_AOL),
         ("arcor.de", &P_ARCOR_DE),
         ("autistici.org", &P_AUTISTICI_ORG),
         ("blindzeln.org", &P_BLINDZELN_ORG),
         ("bluewin.ch", &P_BLUEWIN_CH),
         ("buzon.uy", &P_BUZON_UY),
-        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
-        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
-        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
         ("chello.at", &P_CHELLO_AT),
         ("comcast", &P_COMCAST),
+        ("daleth.cafe", &P_DALETH_CAFE),
         ("dismail.de", &P_DISMAIL_DE),
         ("disroot", &P_DISROOT),
         ("e.email", &P_E_EMAIL),
@@ -1963,11 +2371,14 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("infomaniak.com", &P_INFOMANIAK_COM),
         ("kolst.com", &P_KOLST_COM),
         ("kontent.com", &P_KONTENT_COM),
+        ("mail.com", &P_MAIL_COM),
         ("mail.de", &P_MAIL_DE),
         ("mail.ru", &P_MAIL_RU),
         ("mail2tor", &P_MAIL2TOR),
         ("mailbox.org", &P_MAILBOX_ORG),
         ("mailo.com", &P_MAILO_COM),
+        ("mehl.cloud", &P_MEHL_CLOUD),
+        ("mehl.store", &P_MEHL_STORE),
         ("nauta.cu", &P_NAUTA_CU),
         ("naver", &P_NAVER),
         ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
@@ -1976,10 +2387,13 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("ouvaton.coop", &P_OUVATON_COOP),
         ("posteo", &P_POSTEO),
         ("protonmail", &P_PROTONMAIL),
+        ("purelymail.com", &P_PURELYMAIL_COM),
         ("qq", &P_QQ),
+        ("rambler.ru", &P_RAMBLER_RU),
         ("riseup.net", &P_RISEUP_NET),
         ("rogers.com", &P_ROGERS_COM),
         ("sonic", &P_SONIC),
+        ("stinpriza.net", &P_STINPRIZA_NET),
         ("systemausfall.org", &P_SYSTEMAUSFALL_ORG),
         ("systemli.org", &P_SYSTEMLI_ORG),
         ("t-online", &P_T_ONLINE),
@@ -1990,8 +2404,10 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("undernet.uy", &P_UNDERNET_UY),
         ("vfemail", &P_VFEMAIL),
         ("vivaldi", &P_VIVALDI),
+        ("vk.com", &P_VK_COM),
         ("vodafone.de", &P_VODAFONE_DE),
         ("web.de", &P_WEB_DE),
+        ("wkpb.de", &P_WKPB_DE),
         ("yahoo", &P_YAHOO),
         ("yandex.ru", &P_YANDEX_RU),
         ("yggmail", &P_YGGMAIL),
@@ -2001,4 +2417,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static _PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2024, 2, 5).unwrap());
+    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2024, 8, 14).unwrap());

src/push.rs

@@ -5,7 +5,6 @@ use anyhow::Result;
 use tokio::sync::RwLock;
 
 use crate::context::Context;
-use crate::net::http;
 
 /// Manages subscription to Apple Push Notification services.
 ///
@@ -48,7 +47,10 @@ impl PushSubscriber {
     }
 
     /// Subscribes for heartbeat notifications with previously set device token.
-    pub(crate) async fn subscribe(&self) -> Result<()> {
+    #[cfg(target_os = "ios")]
+    pub(crate) async fn subscribe(&self, context: &Context) -> Result<()> {
+        use crate::net::http;
+
         let mut state = self.inner.write().await;
 
         if state.heartbeat_subscribed {
@@ -59,8 +61,9 @@ impl PushSubscriber {
             return Ok(());
         };
 
-        let socks5_config = None;
-        let response = http::get_client(socks5_config)?
+        let load_cache = true;
+        let response = http::get_client(context, load_cache)
+            .await?
             .post("https://notifications.delta.chat/register")
             .body(format!("{{\"token\":\"{token}\"}}"))
             .send()
@@ -73,6 +76,14 @@ impl PushSubscriber {
         Ok(())
     }
 
+    /// Placeholder to skip subscribing to heartbeat notifications outside iOS.
+    #[cfg(not(target_os = "ios"))]
+    pub(crate) async fn subscribe(&self, _context: &Context) -> Result<()> {
+        let mut state = self.inner.write().await;
+        state.heartbeat_subscribed = true;
+        Ok(())
+    }
+
     pub(crate) async fn heartbeat_subscribed(&self) -> bool {
         self.inner.read().await.heartbeat_subscribed
     }

src/qr.rs

@@ -20,7 +20,6 @@ use crate::events::EventType;
 use crate::key::Fingerprint;
 use crate::message::Message;
 use crate::peerstate::Peerstate;
-use crate::socks::Socks5Config;
 use crate::token;
 use crate::tools::validate_id;
 use iroh_old as iroh;
@@ -37,8 +36,13 @@ const VCARD_SCHEME: &str = "BEGIN:VCARD";
 const SMTP_SCHEME: &str = "SMTP:";
 const HTTP_SCHEME: &str = "http://";
 const HTTPS_SCHEME: &str = "https://";
+
+/// Legacy backup transfer based on iroh 0.4.
 pub(crate) const DCBACKUP_SCHEME: &str = "DCBACKUP:";
 
+/// Backup transfer based on iroh-net.
+pub(crate) const DCBACKUP2_SCHEME: &str = "DCBACKUP2:";
+
 /// Scanned QR code.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Qr {
@@ -106,7 +110,7 @@ pub enum Qr {
         domain: String,
     },
 
-    /// Provides a backup that can be retrieve.
+    /// Provides a backup that can be retrieved using legacy iroh 0.4.
     ///
     /// This contains all the data needed to connect to a device and download a backup from
     /// it to configure the receiving device with the same account.
@@ -120,6 +124,15 @@ pub enum Qr {
         ticket: iroh::provider::Ticket,
     },
 
+    /// Provides a backup that can be retrieved using iroh-net based backup transfer protocol.
+    Backup2 {
+        /// Iroh node address.
+        node_addr: iroh_net::NodeAddr,
+
+        /// Authentication token.
+        auth_token: String,
+    },
+
     /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         /// Server domain name.
@@ -266,6 +279,8 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result<Qr> {
         decode_webrtc_instance(context, qr)?
     } else if starts_with_ignore_case(qr, DCBACKUP_SCHEME) {
         decode_backup(qr)?
+    } else if starts_with_ignore_case(qr, DCBACKUP2_SCHEME) {
+        decode_backup2(qr)?
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr).await?
     } else if qr.starts_with(SMTP_SCHEME) {
@@ -295,6 +310,13 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result<Qr> {
 pub fn format_backup(qr: &Qr) -> Result<String> {
     match qr {
         Qr::Backup { ref ticket } => Ok(format!("{DCBACKUP_SCHEME}{ticket}")),
+        Qr::Backup2 {
+            ref node_addr,
+            ref auth_token,
+        } => {
+            let node_addr = serde_json::to_string(node_addr)?;
+            Ok(format!("{DCBACKUP2_SCHEME}{auth_token}&{node_addr}"))
+        }
         _ => Err(anyhow!("Not a backup QR code")),
     }
 }
@@ -487,7 +509,7 @@ fn decode_account(qr: &str) -> Result<Qr> {
         Ok(Qr::Account {
             domain: url
                 .host_str()
-                .context("can't extract WebRTC instance domain")?
+                .context("can't extract account setup domain")?
                 .to_string(),
         })
     } else {
@@ -529,6 +551,24 @@ fn decode_backup(qr: &str) -> Result<Qr> {
     Ok(Qr::Backup { ticket })
 }
 
+/// Decodes a [`DCBACKUP2_SCHEME`] QR code.
+fn decode_backup2(qr: &str) -> Result<Qr> {
+    let payload = qr
+        .strip_prefix(DCBACKUP2_SCHEME)
+        .ok_or_else(|| anyhow!("invalid DCBACKUP scheme"))?;
+    let (auth_token, node_addr) = payload
+        .split_once('&')
+        .context("Backup QR code has no separator")?;
+    let auth_token = auth_token.to_string();
+    let node_addr = serde_json::from_str::<iroh_net::NodeAddr>(node_addr)
+        .context("Invalid node addr in backup QR code")?;
+
+    Ok(Qr::Backup2 {
+        node_addr,
+        auth_token,
+    })
+}
+
 #[derive(Debug, Deserialize)]
 struct CreateAccountSuccessResponse {
     /// Email address.
@@ -549,8 +589,16 @@ struct CreateAccountErrorResponse {
 #[allow(clippy::indexing_slicing)]
 async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> {
     let url_str = &qr[DCACCOUNT_SCHEME.len()..];
-    let socks5_config = Socks5Config::from_database(&context.sql).await?;
-    let response = crate::net::http::get_client(socks5_config)?
+
+    if !url_str.starts_with(HTTPS_SCHEME) {
+        bail!("DCACCOUNT QR codes must use HTTPS scheme");
+    }
+
+    // As only HTTPS is used, it is safe to load DNS cache.
+    let load_cache = true;
+
+    let response = crate::net::http::get_client(context, load_cache)
+        .await?
         .post(url_str)
         .send()
         .await?;
@@ -611,7 +659,6 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
             context
                 .sync_qr_code_token_deletion(invitenumber, authcode)
                 .await?;
-            context.send_sync_msg().await?;
         }
         Qr::WithdrawVerifyGroup {
             invitenumber,
@@ -623,7 +670,6 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
             context
                 .sync_qr_code_token_deletion(invitenumber, authcode)
                 .await?;
-            context.send_sync_msg().await?;
         }
         Qr::ReviveVerifyContact {
             invitenumber,
@@ -633,7 +679,7 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
             token::save(context, token::Namespace::InviteNumber, None, &invitenumber).await?;
             token::save(context, token::Namespace::Auth, None, &authcode).await?;
             context.sync_qr_code_tokens(None).await?;
-            context.send_sync_msg().await?;
+            context.scheduler.interrupt_smtp().await;
         }
         Qr::ReviveVerifyGroup {
             invitenumber,
@@ -653,7 +699,7 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
             .await?;
             token::save(context, token::Namespace::Auth, chat_id, &authcode).await?;
             context.sync_qr_code_tokens(chat_id).await?;
-            context.send_sync_msg().await?;
+            context.scheduler.interrupt_smtp().await;
         }
         Qr::Login { address, options } => {
             configure_from_login_qr(context, &address, options).await?
@@ -1359,9 +1405,12 @@ mod tests {
             ctx.ctx.get_config(Config::SendUser).await?,
             Some("SendUser".to_owned())
         );
+
+        // `sc` option is actually ignored and `ic` is used instead
+        // because `smtp_certificate_checks` is deprecated.
         assert_eq!(
             ctx.ctx.get_config(Config::SmtpCertificateChecks).await?,
-            Some("3".to_owned())
+            Some("1".to_owned())
         );
         assert_eq!(
             ctx.ctx.get_config(Config::SendSecurity).await?,

src/qr/dclogin_scheme.rs

@@ -39,9 +39,6 @@ pub enum LoginOptions {
         /// IMAP socket security.
         imap_security: Option<Socket>,
 
-        /// IMAP certificate checks.
-        imap_certificate_checks: Option<CertificateChecks>,
-
         /// SMTP host.
         smtp_host: Option<String>,
 
@@ -57,8 +54,8 @@ pub enum LoginOptions {
         /// SMTP socket security.
         smtp_security: Option<Socket>,
 
-        /// SMTP certificate checks.
-        smtp_certificate_checks: Option<CertificateChecks>,
+        /// Certificate checks.
+        certificate_checks: Option<CertificateChecks>,
     },
 }
 
@@ -107,14 +104,13 @@ pub(super) fn decode_login(qr: &str) -> Result<Qr> {
                 imap_username: parameter_map.get("iu").map(|s| s.to_owned()),
                 imap_password: parameter_map.get("ipw").map(|s| s.to_owned()),
                 imap_security: parse_socket_security(parameter_map.get("is"))?,
-                imap_certificate_checks: parse_certificate_checks(parameter_map.get("ic"))?,
                 smtp_host: parameter_map.get("sh").map(|s| s.to_owned()),
                 smtp_port: parse_port(parameter_map.get("sp"))
                     .context("could not parse smtp port")?,
                 smtp_username: parameter_map.get("su").map(|s| s.to_owned()),
                 smtp_password: parameter_map.get("spw").map(|s| s.to_owned()),
                 smtp_security: parse_socket_security(parameter_map.get("ss"))?,
-                smtp_certificate_checks: parse_certificate_checks(parameter_map.get("sc"))?,
+                certificate_checks: parse_certificate_checks(parameter_map.get("ic"))?,
             },
             Some(Ok(v)) => LoginOptions::UnsuportedVersion(v),
             Some(Err(_)) => bail!("version could not be parsed as number E6"),
@@ -177,13 +173,12 @@ pub(crate) async fn configure_from_login_qr(
             imap_username,
             imap_password,
             imap_security,
-            imap_certificate_checks,
             smtp_host,
             smtp_port,
             smtp_username,
             smtp_password,
             smtp_security,
-            smtp_certificate_checks,
+            certificate_checks,
         } => {
             context
                 .set_config_internal(Config::MailPw, Some(&mail_pw))
@@ -216,14 +211,6 @@ pub(crate) async fn configure_from_login_qr(
                     .set_config_internal(Config::MailSecurity, Some(&code.to_string()))
                     .await?;
             }
-            if let Some(value) = imap_certificate_checks {
-                let code = value
-                    .to_u32()
-                    .context("could not convert imap certificate checks value to number")?;
-                context
-                    .set_config_internal(Config::ImapCertificateChecks, Some(&code.to_string()))
-                    .await?;
-            }
             if let Some(value) = smtp_host {
                 context
                     .set_config_internal(Config::SendServer, Some(&value))
@@ -252,10 +239,13 @@ pub(crate) async fn configure_from_login_qr(
                     .set_config_internal(Config::SendSecurity, Some(&code.to_string()))
                     .await?;
             }
-            if let Some(value) = smtp_certificate_checks {
+            if let Some(value) = certificate_checks {
                 let code = value
                     .to_u32()
-                    .context("could not convert smtp certificate checks value to number")?;
+                    .context("could not convert certificate checks value to number")?;
+                context
+                    .set_config_internal(Config::ImapCertificateChecks, Some(&code.to_string()))
+                    .await?;
                 context
                     .set_config_internal(Config::SmtpCertificateChecks, Some(&code.to_string()))
                     .await?;
@@ -284,13 +274,12 @@ mod test {
                 imap_username: None,
                 imap_password: None,
                 imap_security: None,
-                imap_certificate_checks: None,
                 smtp_host: None,
                 smtp_port: None,
                 smtp_username: None,
                 smtp_password: None,
                 smtp_security: None,
-                smtp_certificate_checks: None,
+                certificate_checks: None,
             }
         };
     }
@@ -392,13 +381,12 @@ mod test {
                     imap_username: Some("max".to_owned()),
                     imap_password: Some("87654".to_owned()),
                     imap_security: Some(Socket::Ssl),
-                    imap_certificate_checks: Some(CertificateChecks::Strict),
                     smtp_host: Some("mail.host.tld".to_owned()),
                     smtp_port: Some(3000),
                     smtp_username: Some("max@host.tld".to_owned()),
                     smtp_password: Some("3242HS".to_owned()),
                     smtp_security: Some(Socket::Plain),
-                    smtp_certificate_checks: Some(CertificateChecks::AcceptInvalidCertificates),
+                    certificate_checks: Some(CertificateChecks::Strict),
                 }
             );
         } else {

src/qr_code_generator.rs

@@ -58,7 +58,7 @@ async fn generate_verification_qr(context: &Context) -> Result<String> {
     )
 }
 
-/// Renders a [`Qr::Backup`] QR code as an SVG image.
+/// Renders a [`Qr::Backup2`] QR code as an SVG image.
 pub async fn generate_backup_qr(context: &Context, qr: &Qr) -> Result<String> {
     let content = qr::format_backup(qr)?;
     let (avatar, displayname, _addr, color) = self_info(context).await?;

src/quota.rs

@@ -1,6 +1,7 @@
 //! # Support for IMAP QUOTA extension.
 
 use std::collections::BTreeMap;
+use std::time::Duration;
 
 use anyhow::{anyhow, Context as _, Result};
 use async_imap::types::{Quota, QuotaResource};
@@ -11,7 +12,7 @@ use crate::context::Context;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::session::Session as ImapSession;
 use crate::message::{Message, Viewtype};
-use crate::tools;
+use crate::tools::{self, time_elapsed};
 use crate::{stock_str, EventType};
 
 /// warn about a nearly full mailbox after this usage percentage is reached.
@@ -102,6 +103,16 @@ pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> b
 }
 
 impl Context {
+    /// Returns whether the quota value needs an update. If so, `update_recent_quota()` should be
+    /// called.
+    pub(crate) async fn quota_needs_update(&self, ratelimit_secs: u64) -> bool {
+        let quota = self.quota.read().await;
+        quota
+            .as_ref()
+            .filter(|quota| time_elapsed(&quota.modified) < Duration::from_secs(ratelimit_secs))
+            .is_none()
+    }
+
     /// Updates `quota.recent`, sets `quota.modified` to the current time
     /// and emits an event to let the UIs update connectivity view.
     ///
@@ -155,6 +166,7 @@ impl Context {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::test_utils::TestContextManager;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_needs_quota_warning() -> Result<()> {
@@ -183,4 +195,24 @@ mod tests {
         assert!(QUOTA_ERROR_THRESHOLD_PERCENTAGE < 100);
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_quota_needs_update() {
+        let mut tcm = TestContextManager::new();
+        let t = &tcm.unconfigured().await;
+        const TIMEOUT: u64 = 60;
+        assert!(t.quota_needs_update(TIMEOUT).await);
+
+        *t.quota.write().await = Some(QuotaInfo {
+            recent: Ok(Default::default()),
+            modified: tools::Time::now() - Duration::from_secs(TIMEOUT + 1),
+        });
+        assert!(t.quota_needs_update(TIMEOUT).await);
+
+        *t.quota.write().await = Some(QuotaInfo {
+            recent: Ok(Default::default()),
+            modified: tools::Time::now(),
+        });
+        assert!(!t.quota_needs_update(TIMEOUT).await);
+    }
 }

src/receive_imf.rs

@@ -4,9 +4,7 @@ use std::collections::HashSet;
 use std::str::FromStr;
 
 use anyhow::{Context as _, Result};
-use deltachat_contact_tools::{
-    addr_cmp, may_be_valid_addr, normalize_name, strip_rtlo_characters, ContactAddress,
-};
+use deltachat_contact_tools::{addr_cmp, may_be_valid_addr, sanitize_single_line, ContactAddress};
 use iroh_gossip::proto::TopicId;
 use mailparse::{parse_mail, SingleInfo};
 use num_traits::FromPrimitive;
@@ -27,7 +25,7 @@ use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::imap::{markseen_on_imap_table, GENERATED_PREFIX};
 use crate::log::LogExt;
 use crate::message::{
-    self, rfc724_mid_exists, rfc724_mid_exists_and, Message, MessageState, MessengerMessage, MsgId,
+    self, rfc724_mid_exists, rfc724_mid_exists_ex, Message, MessageState, MessengerMessage, MsgId,
     Viewtype,
 };
 use crate::mimeparser::{parse_message_ids, AvatarAction, MimeMessage, SystemMessage};
@@ -40,7 +38,7 @@ use crate::simplify;
 use crate::sql;
 use crate::stock_str;
 use crate::sync::Sync::*;
-use crate::tools::{self, buf_compress};
+use crate::tools::{self, buf_compress, remove_subject_prefix};
 use crate::{chatlist_events, location};
 use crate::{contact, imap};
 use iroh_net::NodeAddr;
@@ -489,7 +487,9 @@ pub(crate) async fn receive_imf_inner(
             can_info_msg = false;
             Some(Message::load_from_db(context, insert_msg_id).await?)
         } else if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) {
-            if let Some(instance) = get_rfc724_mid_in_list(context, field).await? {
+            if let Some(instance) =
+                message::get_by_rfc724_mids(context, &parse_message_ids(field)).await?
+            {
                 can_info_msg = instance.download_state() == DownloadState::Done;
                 Some(instance)
             } else {
@@ -632,9 +632,9 @@ pub(crate) async fn receive_imf_inner(
 /// Also returns whether it is blocked or not and its origin.
 ///
 /// * `prevent_rename`: if true, the display_name of this contact will not be changed. Useful for
-/// mailing lists: In some mailing lists, many users write from the same address but with different
-/// display names. We don't want the display name to change every time the user gets a new email from
-/// a mailing list.
+///   mailing lists: In some mailing lists, many users write from the same address but with different
+///   display names. We don't want the display name to change every time the user gets a new email from
+///   a mailing list.
 ///
 /// Returns `None` if From field does not contain a valid contact address.
 pub async fn from_field_to_contact_id(
@@ -658,10 +658,10 @@ pub async fn from_field_to_contact_id(
         }
     };
 
-    let from_id = add_or_lookup_contact_by_addr(
+    let (from_id, _) = Contact::add_or_lookup(
         context,
-        display_name,
-        from_addr,
+        display_name.unwrap_or_default(),
+        &from_addr,
         Origin::IncomingUnknownFrom,
     )
     .await?;
@@ -694,6 +694,9 @@ async fn add_parts(
     prevent_rename: bool,
     verified_encryption: VerifiedEncryption,
 ) -> Result<ReceivedMsg> {
+    let is_bot = context.get_config_bool(Config::Bot).await?;
+    // Bots handle existing messages the same way as new ones.
+    let fetching_existing_messages = fetching_existing_messages && !is_bot;
     let rfc724_mid_orig = &mime_parser
         .get_rfc724_mid()
         .unwrap_or(rfc724_mid.to_string());
@@ -707,9 +710,13 @@ async fn add_parts(
         better_msg = Some(stock_str::msg_location_enabled_by(context, from_id).await);
     }
 
-    let parent = get_parent_message(context, mime_parser)
-        .await?
-        .filter(|p| Some(p.id) != replace_msg_id);
+    let parent = get_parent_message(
+        context,
+        mime_parser.get_header(HeaderDef::References),
+        mime_parser.get_header(HeaderDef::InReplyTo),
+    )
+    .await?
+    .filter(|p| Some(p.id) != replace_msg_id);
 
     let is_dc_message = if mime_parser.has_chat_version() {
         MessengerMessage::Yes
@@ -782,9 +789,6 @@ async fn add_parts(
             info!(context, "Message is an MDN (TRASH).",);
         }
 
-        // signals whether the current user is a bot
-        let is_bot = context.get_config_bool(Config::Bot).await?;
-
         let create_blocked_default = if is_bot {
             Blocked::Not
         } else {
@@ -1440,12 +1444,19 @@ async fn add_parts(
             Ok(node_addr) => {
                 info!(context, "Adding iroh peer with address {node_addr:?}.");
                 let instance_id = parent.context("Failed to get parent message")?.id;
-                let node_id = node_addr.node_id;
-                let relay_server = node_addr.relay_url().map(|relay| relay.as_str());
-                let topic = get_iroh_topic_for_msg(context, instance_id).await?;
-                iroh_add_peer_for_topic(context, instance_id, topic, node_id, relay_server).await?;
-                let iroh = context.get_or_try_init_peer_channel().await?;
-                iroh.maybe_add_gossip_peers(topic, vec![node_addr]).await?;
+                if let Some(topic) = get_iroh_topic_for_msg(context, instance_id).await? {
+                    let node_id = node_addr.node_id;
+                    let relay_server = node_addr.relay_url().map(|relay| relay.as_str());
+                    iroh_add_peer_for_topic(context, instance_id, topic, node_id, relay_server)
+                        .await?;
+                    let iroh = context.get_or_try_init_peer_channel().await?;
+                    iroh.maybe_add_gossip_peers(topic, vec![node_addr]).await?;
+                } else {
+                    warn!(
+                        context,
+                        "Could not add iroh peer because {instance_id} has no topic"
+                    );
+                }
                 chat_id = DC_CHAT_ID_TRASH;
             }
             Err(err) => {
@@ -1534,7 +1545,7 @@ INSERT INTO msgs
     rfc724_mid, chat_id,
     from_id, to_id, timestamp, timestamp_sent, 
     timestamp_rcvd, type, state, msgrmsg, 
-    txt, subject, txt_raw, param, hidden,
+    txt, txt_normalized, subject, txt_raw, param, hidden,
     bytes, mime_headers, mime_compressed, mime_in_reply_to,
     mime_references, mime_modified, error, ephemeral_timer,
     ephemeral_timestamp, download_state, hop_info
@@ -1544,7 +1555,7 @@ INSERT INTO msgs
     ?, ?, ?, ?,
     ?, ?, ?, ?,
     ?, ?, ?, ?, ?,
-    ?, ?, ?, ?, 1,
+    ?, ?, ?, ?, ?, 1,
     ?, ?, ?, ?,
     ?, ?, ?, ?
   )
@@ -1552,7 +1563,8 @@ ON CONFLICT (id) DO UPDATE
 SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
     from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent,
     type=excluded.type, msgrmsg=excluded.msgrmsg,
-    txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param,
+    txt=excluded.txt, txt_normalized=excluded.txt_normalized, subject=excluded.subject,
+    txt_raw=excluded.txt_raw, param=excluded.param,
     hidden=excluded.hidden,bytes=excluded.bytes, mime_headers=excluded.mime_headers,
     mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to,
     mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer,
@@ -1572,6 +1584,7 @@ RETURNING id
                     state,
                     is_dc_message,
                     if trash { "" } else { msg },
+                    if trash { None } else { message::normalize_text(msg) },
                     if trash { "" } else { &subject },
                     // txt_raw might contain invalid utf8
                     if trash { "" } else { &txt_raw },
@@ -1644,11 +1657,20 @@ RETURNING id
     }
 
     if let Some(replace_msg_id) = replace_msg_id {
-        // "Replace" placeholder with a message that has no parts.
-        replace_msg_id.trash(context).await?;
+        // Trash the "replace" placeholder with a message that has no parts. If it has the original
+        // "Message-ID", mark the placeholder for server-side deletion so as if the user deletes the
+        // fully downloaded message later, the server-side deletion is issued.
+        let on_server = rfc724_mid == rfc724_mid_orig;
+        replace_msg_id.trash(context, on_server).await?;
     }
 
-    chat_id.unarchive_if_not_muted(context, state).await?;
+    let unarchive = match mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
+        Some(addr) => context.is_self_addr(addr).await?,
+        None => true,
+    };
+    if unarchive {
+        chat_id.unarchive_if_not_muted(context, state).await?;
+    }
 
     info!(
         context,
@@ -1808,7 +1830,7 @@ async fn lookup_chat_or_create_adhoc_group(
         Ok(Some((new_chat_id, new_chat_id_blocked)))
     } else if allow_creation {
         // Try to create an ad hoc group.
-        if let Some(new_chat_id) = create_adhoc_group(
+        create_adhoc_group(
             context,
             mime_parser,
             create_blocked,
@@ -1817,12 +1839,7 @@ async fn lookup_chat_or_create_adhoc_group(
             is_partial_download,
         )
         .await
-        .context("Could not create ad hoc group")?
-        {
-            Ok(Some((new_chat_id, create_blocked)))
-        } else {
-            Ok(None)
-        }
+        .context("Could not create ad hoc group")
     } else {
         Ok(None)
     }
@@ -1930,8 +1947,9 @@ async fn create_group(
         let grpname = mime_parser
             .get_header(HeaderDef::ChatGroupName)
             .context("Chat-Group-Name vanished")?
-            // W/a for "Space added before long group names after MIME serialization/deserialization
-            // #3650" issue. DC itself never creates group names with leading/trailing whitespace.
+            // Workaround for the "Space added before long group names after MIME
+            // serialization/deserialization #3650" issue. DC itself never creates group names with
+            // leading/trailing whitespace.
             .trim();
         let new_chat_id = ChatId::create_multiuser_record(
             context,
@@ -2049,8 +2067,9 @@ async fn apply_group_changes(
             || match mime_parser.get_header(HeaderDef::InReplyTo) {
                 // If we don't know the referenced message, we missed some messages.
                 // Maybe they added/removed members, so we need to recreate our member list.
-                Some(reply_to) => rfc724_mid_exists_and(context, reply_to, "download_state=0")
+                Some(reply_to) => rfc724_mid_exists_ex(context, reply_to, "download_state=0")
                     .await?
+                    .filter(|(_, _, downloaded)| *downloaded)
                     .is_none(),
                 None => false,
             }
@@ -2121,15 +2140,15 @@ async fn apply_group_changes(
         }
     } else if let Some(old_name) = mime_parser
         .get_header(HeaderDef::ChatGroupNameChanged)
-        // See create_or_lookup_group() for explanation
         .map(|s| s.trim())
     {
         if let Some(grpname) = mime_parser
             .get_header(HeaderDef::ChatGroupName)
-            // See create_or_lookup_group() for explanation
             .map(|grpname| grpname.trim())
             .filter(|grpname| grpname.len() < 200)
         {
+            let grpname = &sanitize_single_line(grpname);
+            let old_name = &sanitize_single_line(old_name);
             if chat_id
                 .update_timestamp(
                     context,
@@ -2141,10 +2160,7 @@ async fn apply_group_changes(
                 info!(context, "Updating grpname for chat {chat_id}.");
                 context
                     .sql
-                    .execute(
-                        "UPDATE chats SET name=? WHERE id=?;",
-                        (strip_rtlo_characters(grpname), chat_id),
-                    )
+                    .execute("UPDATE chats SET name=? WHERE id=?;", (grpname, chat_id))
                     .await?;
                 send_event_chat_modified = true;
             }
@@ -2417,7 +2433,7 @@ fn compute_mailinglist_name(
         }
     }
 
-    strip_rtlo_characters(&name)
+    sanitize_single_line(&name)
 }
 
 /// Set ListId param on the contact and ListPost param the chat.
@@ -2497,7 +2513,7 @@ async fn create_adhoc_group(
     from_id: ContactId,
     to_ids: &[ContactId],
     is_partial_download: bool,
-) -> Result<Option<ChatId>> {
+) -> Result<Option<(ChatId, Blocked)>> {
     if is_partial_download {
         // Partial download may be an encrypted message with protected Subject header.
         //
@@ -2536,15 +2552,24 @@ async fn create_adhoc_group(
         );
         return Ok(None);
     }
-
+    if mime_parser
+        .get_header(HeaderDef::ChatGroupMemberRemoved)
+        .is_some()
+    {
+        info!(
+            context,
+            "Message removes member from unknown ad-hoc group (TRASH)."
+        );
+        return Ok(Some((DC_CHAT_ID_TRASH, Blocked::Not)));
+    }
     if member_ids.len() < 3 {
         return Ok(None);
     }
 
-    // use subject as initial chat name
     let grpname = mime_parser
         .get_subject()
-        .unwrap_or_else(|| "Unnamed group".to_string());
+        .map(|s| remove_subject_prefix(&s))
+        .unwrap_or_else(|| "👥📧".to_string());
 
     let new_chat_id: ChatId = ChatId::create_multiuser_record(
         context,
@@ -2568,7 +2593,7 @@ async fn create_adhoc_group(
     chatlist_events::emit_chatlist_changed(context);
     chatlist_events::emit_chatlist_item_changed(context, new_chat_id);
 
-    Ok(Some(new_chat_id))
+    Ok(Some((new_chat_id, create_blocked)))
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -2792,53 +2817,35 @@ async fn get_previous_message(
     Ok(None)
 }
 
-/// Given a list of Message-IDs, returns the latest message found in the database.
-///
-/// Only messages that are not in the trash chat are considered.
-async fn get_rfc724_mid_in_list(context: &Context, mid_list: &str) -> Result<Option<Message>> {
-    message::get_latest_by_rfc724_mids(context, &parse_message_ids(mid_list)).await
-}
-
 /// Returns the last message referenced from References: header found in the database.
 ///
 /// If none found, tries In-Reply-To: as a fallback for classic MUAs that don't set the
 /// References: header.
 async fn get_parent_message(
     context: &Context,
-    mime_parser: &MimeMessage,
+    references: Option<&str>,
+    in_reply_to: Option<&str>,
 ) -> Result<Option<Message>> {
-    if let Some(field) = mime_parser.get_header(HeaderDef::References) {
-        if let Some(msg) = get_rfc724_mid_in_list(context, field).await? {
-            return Ok(Some(msg));
-        }
+    let mut mids = Vec::new();
+    if let Some(field) = in_reply_to {
+        mids = parse_message_ids(field);
     }
-
-    if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) {
-        if let Some(msg) = get_rfc724_mid_in_list(context, field).await? {
-            return Ok(Some(msg));
-        }
+    if let Some(field) = references {
+        mids.append(&mut parse_message_ids(field));
     }
-
-    Ok(None)
+    message::get_by_rfc724_mids(context, &mids).await
 }
 
 pub(crate) async fn get_prefetch_parent_message(
     context: &Context,
     headers: &[mailparse::MailHeader<'_>],
 ) -> Result<Option<Message>> {
-    if let Some(field) = headers.get_header_value(HeaderDef::References) {
-        if let Some(msg) = get_rfc724_mid_in_list(context, &field).await? {
-            return Ok(Some(msg));
-        }
-    }
-
-    if let Some(field) = headers.get_header_value(HeaderDef::InReplyTo) {
-        if let Some(msg) = get_rfc724_mid_in_list(context, &field).await? {
-            return Ok(Some(msg));
-        }
-    }
-
-    Ok(None)
+    get_parent_message(
+        context,
+        headers.get_header_value(HeaderDef::References).as_deref(),
+        headers.get_header_value(HeaderDef::InReplyTo).as_deref(),
+    )
+    .await
 }
 
 /// Looks up contact IDs from the database given the list of recipients.
@@ -2857,8 +2864,9 @@ async fn add_or_lookup_contacts_by_address_list(
         }
         let display_name = info.display_name.as_deref();
         if let Ok(addr) = ContactAddress::new(addr) {
-            let contact_id =
-                add_or_lookup_contact_by_addr(context, display_name, addr, origin).await?;
+            let (contact_id, _) =
+                Contact::add_or_lookup(context, display_name.unwrap_or_default(), &addr, origin)
+                    .await?;
             contact_ids.insert(contact_id);
         } else {
             warn!(context, "Contact with address {:?} cannot exist.", addr);
@@ -2868,22 +2876,5 @@ async fn add_or_lookup_contacts_by_address_list(
     Ok(contact_ids.into_iter().collect::<Vec<ContactId>>())
 }
 
-/// Add contacts to database on receiving messages.
-async fn add_or_lookup_contact_by_addr(
-    context: &Context,
-    display_name: Option<&str>,
-    addr: ContactAddress,
-    origin: Origin,
-) -> Result<ContactId> {
-    if context.is_self_addr(&addr).await? {
-        return Ok(ContactId::SELF);
-    }
-    let display_name_normalized = display_name.map(normalize_name).unwrap_or_default();
-
-    let (contact_id, _modified) =
-        Contact::add_or_lookup(context, &display_name_normalized, &addr, origin).await?;
-    Ok(contact_id)
-}
-
 #[cfg(test)]
 mod tests;