Prepare 1.105.0 release

remove unused imports
and fail without revealing the token

.gitattributes

@@ -4,7 +4,7 @@
 
 # This directory contains email messages verbatim, and changing CRLF to
 # LF will corrupt them.
-test-data/* text=false
+test-data/** text=false
 
 # binary files should be detected by git, however, to be sure, you can add them here explicitly
 *.png  binary

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
           override: true
       - run: rustup component add rustfmt
       - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
+        uses: swatinem/rust-cache@v2
       - run: cargo fmt --all -- --check
 
   run_clippy:
@@ -38,7 +38,7 @@ jobs:
             components: clippy
             override: true
       - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
+        uses: swatinem/rust-cache@v2
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -60,7 +60,7 @@ jobs:
           components: rust-docs
           override: true
       - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
+        uses: swatinem/rust-cache@v2
       - name: Rustdoc
         run: cargo doc --document-private-items --no-deps
 
@@ -69,21 +69,21 @@ jobs:
     strategy:
       matrix:
         include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
+          # Currently used Rust version.
           - os: ubuntu-latest
-            rust: 1.65.0
+            rust: 1.64.0
             python: 3.9
           - os: windows-latest
-            rust: 1.65.0
+            rust: 1.64.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.61.0
+          # Minimum Supported Rust Version = 1.63.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.61.0
+            rust: 1.63.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
@@ -96,7 +96,7 @@ jobs:
         override: true
 
     - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v1
+      uses: swatinem/rust-cache@v2
 
     - name: check
       run: cargo check --all --bins --examples --tests --features repl --benches

.github/workflows/jsonrpc.yml

@@ -19,13 +19,8 @@ jobs:
         uses: actions/setup-node@v3
         with:
           node-version: 16.x
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
       - name: Add Rust cache
-        uses: Swatinem/rust-cache@v1.3.0
+        uses: Swatinem/rust-cache@v2
       - name: npm install
         run: |
           cd deltachat-jsonrpc/typescript

.github/workflows/node-tests.yml

@@ -59,6 +59,7 @@ jobs:
           npm run test
         env:
           DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'
       - name: Run tests on Windows, except lint
         timeout-minutes: 10
         if: runner.os == 'Windows'
@@ -67,3 +68,4 @@ jobs:
           npm run test:mocha
         env:
           DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'

.github/workflows/repl.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Install Rust
       uses: actions-rs/toolchain@v1
       with:
-        toolchain: 1.50.0
+        toolchain: 1.66.0
         override: true
 
     - name: build

.github/workflows/upload-docs.yml

@@ -13,7 +13,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - uses: actions-rs/toolchain@v1
     - name: Build the documentation with cargo
       run: |
         cargo doc --package deltachat --no-deps

.github/workflows/upload-ffi-docs.yml

@@ -13,7 +13,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - uses: actions-rs/toolchain@v1
     - name: Build the documentation with cargo
       run: |
         cargo doc --package deltachat_ffi --no-deps

CHANGELOG.md

@@ -3,12 +3,62 @@
 ## Unreleased
 
 ### Changes
-- Don't use deprecated `chrono` functions #3798
 
 ### API-Changes
 
+### Fixes
+
+
+## 1.105.0
+
+### Changes
+- Validate signatures in try_decrypt() even if the message isn't encrypted #3859
+- Don't parse the message again after detached signatures validation #3862
+- Move format=flowed support to a separate crate #3869
+- cargo: bump quick-xml from 0.23.0 to 0.26.0 #3722
+- Add fuzzing tests #3853
+- Add mappings for some file types to Viewtype / MIME type #3881
+- Buffer IMAP client writes #3888
+- move `DC_CHAT_ID_ARCHIVED_LINK` to the top of chat lists
+  and make `dc_get_fresh_msg_cnt()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3918
+- make `dc_marknoticed_chat()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3919
+- Update provider database
+
+### API-Changes
+- jsonrpc: add python API for webxdc updates #3872
+- jsonrpc: add fresh message count to ChatListItemFetchResult::ArchiveLink
+- Add ffi functions to retrieve `verified by` information #3786
+- resultify `Message::get_filebytes()` #3925
+
+### Fixes
+- Do not add an error if the message is encrypted but not signed #3860
+- Do not strip leading spaces from message lines #3867
+- Fix uncaught exception in JSON-RPC tests #3884
+- Fix STARTTLS connection and add a test for it #3907
+- Trigger reconnection when failing to fetch existing messages #3911
+- Do not retry fetching existing messages after failure, prevents infinite reconnection loop #3913
+- Ensure format=flowed formatting is always reversible on the receiver side #3880
+
+
+## 1.104.0
+
+### Changes
+- Don't use deprecated `chrono` functions #3798
+- Document accounts manager #3837
+- If a classical-email-user sends an email to a group and adds new recipients,
+  add the new recipients as group members #3781
+- Remove `pytest-async` plugin #3846
+- Only send the message about ephemeral timer change if the chat is promoted #3847
+- Use relative paths in `accounts.toml` #3838
+
 ### Fixes
 - Set read/write timeouts for IMAP over SOCKS5 #3833
+- Treat attached PGP keys as peer keys with mutual encryption preference #3832
+- fix migration of old databases #3842
+- Fix cargo clippy and doc errors after Rust update to 1.66 #3850
+- Don't send GroupNameChanged message if the group name doesn't change in terms of
+  `improve_single_line_input()` #3852
+- Prefer encryption for the peer if the message is encrypted or signed with the known key #3849
 
 
 ## 1.103.0

Cargo.lock

@@ -10,9 +10,9 @@ checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
 
 [[package]]
 name = "addr2line"
-version = "0.17.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
+checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
 dependencies = [
  "gimli",
 ]
@@ -86,9 +86,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.66"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
+checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
 
 [[package]]
 name = "ascii_utils"
@@ -340,15 +340,15 @@ dependencies = [
 
 [[package]]
 name = "backtrace"
-version = "0.3.66"
+version = "0.3.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
+checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
 dependencies = [
  "addr2line",
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide 0.5.3",
+ "miniz_oxide 0.6.2",
  "object",
  "rustc-demangle",
 ]
@@ -371,6 +371,12 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
+[[package]]
+name = "base64"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
+
 [[package]]
 name = "base64ct"
 version = "1.5.1"
@@ -861,13 +867,13 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.3.2"
+version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
+checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb"
 
 [[package]]
 name = "deltachat"
-version = "1.103.0"
+version = "1.105.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -877,7 +883,7 @@ dependencies = [
  "async-smtp",
  "async_zip",
  "backtrace",
- "base64 0.13.1",
+ "base64 0.20.0",
  "bitflags",
  "chrono",
  "criterion",
@@ -887,6 +893,7 @@ dependencies = [
  "encoded-words",
  "escaper",
  "fast-socks5",
+ "format-flowed",
  "futures",
  "futures-lite",
  "hex",
@@ -940,7 +947,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.103.0"
+version = "1.105.0"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -962,7 +969,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.103.0"
+version = "1.105.0"
 dependencies = [
  "anyhow",
  "deltachat-jsonrpc",
@@ -985,7 +992,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.103.0"
+version = "1.105.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1445,6 +1452,10 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "format-flowed"
+version = "1.0.0"
+
 [[package]]
 name = "futures"
 version = "0.3.25"
@@ -1593,9 +1604,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.26.2"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
+checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793"
 
 [[package]]
 name = "h2"
@@ -1747,9 +1758,9 @@ dependencies = [
 
 [[package]]
 name = "humansize"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e682e2bd70ecbcce5209f11a992a4ba001fea8e60acf7860ce007629e6d2756"
+checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
 dependencies = [
  "libm",
 ]
@@ -2025,9 +2036,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.137"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
 [[package]]
 name = "libm"
@@ -2095,9 +2106,9 @@ dependencies = [
 
 [[package]]
 name = "mailparse"
-version = "0.13.8"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cae768a50835557749599277fc59f7c728118724eb34185e8feb633ef266a32"
+checksum = "6b56570f5f8c0047260d1c8b5b331f62eb9c660b9dd4071a8c46f8c7d3f280aa"
 dependencies = [
  "charset",
  "data-encoding",
@@ -2331,28 +2342,28 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
- "hermit-abi 0.1.19",
+ "hermit-abi 0.2.6",
  "libc",
 ]
 
 [[package]]
 name = "object"
-version = "0.29.0"
+version = "0.30.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
+checksum = "239da7f290cfa979f43f85a8efeee9a8a76d0827c356d37f9d3d7254d6b537fb"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
 [[package]]
 name = "oorandom"
@@ -2734,27 +2745,27 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
 
 [[package]]
 name = "quick-xml"
-version = "0.23.0"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9279fbdacaad3baf559d8cabe0acc3d06e30ea14931af31af79578ac0946decc"
+checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
 name = "quoted_printable"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fee2dce59f7a43418e3382c766554c614e06a552d53a8f07ef499ea4b332c0f"
+checksum = "20f14e071918cbeefc5edc986a7aa92c425dae244e003a35e1cdddb5ca39b5cb"
 
 [[package]]
 name = "r2d2"
@@ -3179,18 +3190,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.148"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.148"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3199,9 +3210,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.89"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
+checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
 dependencies = [
  "itoa",
  "ryu",
@@ -3408,9 +3419,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.105"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3477,18 +3488,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.37"
+version = "1.0.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3533,9 +3544,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.22.0"
+version = "1.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3"
+checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"
 dependencies = [
  "autocfg",
  "bytes",
@@ -3548,7 +3559,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -3636,9 +3647,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.5.9"
+version = "0.5.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f"
 dependencies = [
  "serde",
 ]

Cargo.toml

@@ -1,10 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.103.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+version = "1.105.0"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.61"
+rust-version = "1.63"
 
 [profile.dev]
 debug = 0
@@ -20,6 +19,7 @@ panic = 'abort'
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
+format-flowed = { path = "./format-flowed" }
 
 ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
@@ -30,7 +30,7 @@ trust-dns-resolver = "0.22"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 backtrace = "0.3"
-base64 = "0.13"
+base64 = "0.20"
 bitflags = "1.3"
 chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 dirs = { version = "4", optional=true }
@@ -44,16 +44,16 @@ kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
 log = {version = "0.4.16", optional = true }
-mailparse = "0.13"
+mailparse = "0.14"
 native-tls = "0.2"
-num_cpus = "1.14"
+num_cpus = "1.15"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.16.0"
+once_cell = "1.17.0"
 percent-encoding = "2.2"
 pgp = { version = "0.9", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.23"
+quick-xml = "0.27"
 r2d2 = "0.8"
 r2d2_sqlite = "0.20"
 rand = "0.8"
@@ -100,7 +100,8 @@ members = [
   "deltachat-ffi",
   "deltachat_derive",
   "deltachat-jsonrpc",
-  "deltachat-rpc-server"
+  "deltachat-rpc-server",
+  "format-flowed",
 ]
 
 [[example]]

README.md

@@ -115,6 +115,29 @@ use the `--ignored` argument to the test binary (not to cargo itself):
 $ cargo test -- --ignored
 ```
 
+### Fuzzing
+
+Install [`cargo-bolero`](https://github.com/camshaft/bolero) with
+```sh
+$ cargo install cargo-bolero
+```
+
+Run fuzzing tests with
+```sh
+$ cd fuzz
+$ cargo bolero test fuzz_mailparse --release=false -s NONE
+```
+
+Corpus is created at `fuzz/fuzz_targets/corpus`,
+you can add initial inputs there.
+For `fuzz_mailparse` target corpus can be populated with
+`../test-data/message/*.eml`.
+
+To run with AFL instead of libFuzzer:
+```sh
+$ cargo bolero test fuzz_format_flowed --release=false -e afl -s NONE
+```
+
 ## Features
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.

assets/icon-archive.svg

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="60"
+   height="60"
+   viewBox="0 0 60 60"
+   fill="none"
+   stroke="currentColor"
+   stroke-width="2"
+   stroke-linecap="round"
+   stroke-linejoin="round"
+   class="feather feather-archive"
+   version="1.1"
+   id="svg8"
+   sodipodi:docname="icon-archive.svg"
+   inkscape:version="1.2.2 (b0a84865, 2022-12-01)"
+   inkscape:export-filename="icon-archive.png"
+   inkscape:export-xdpi="409.60001"
+   inkscape:export-ydpi="409.60001"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs12" />
+  <sodipodi:namedview
+     id="namedview10"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     showgrid="false"
+     inkscape:zoom="6.4597151"
+     inkscape:cx="24.459283"
+     inkscape:cy="32.509174"
+     inkscape:window-width="1457"
+     inkscape:window-height="860"
+     inkscape:window-x="55"
+     inkscape:window-y="38"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg8" />
+  <g
+     id="g846"
+     transform="translate(0.558605,0.464417)">
+    <path
+       style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:1.78186;stroke-dasharray:none;stroke-opacity:1"
+       d="M 38.749006,25.398867 V 38.843194 H 20.133784 V 25.398867"
+       id="path847" />
+    <path
+       style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:1.78186;stroke-dasharray:none;stroke-opacity:1"
+       d="m 18.065427,20.227972 h 22.751936 v 5.170894 H 18.065427 Z"
+       id="path845" />
+    <path
+       style="fill:#ff0000;fill-opacity:1;stroke:#808080;stroke-width:1.78186;stroke-dasharray:none;stroke-opacity:1"
+       d="m 27.373036,29.535581 h 4.136718"
+       id="line6" />
+  </g>
+</svg>

deltachat-ffi/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
 name = "deltachat_ffi"
-version = "1.103.0"
+version = "1.105.0"
 description = "Deltachat FFI"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
 license = "MPL-2.0"
@@ -25,7 +24,7 @@ tokio = { version = "1", features = ["rt-multi-thread"] }
 anyhow = "1"
 thiserror = "1"
 rand = "0.7"
-once_cell = "1.16.0"
+once_cell = "1.17.0"
 
 [features]
 default = ["vendored"]

deltachat-ffi/deltachat.h

@@ -1227,7 +1227,11 @@ int             dc_get_msg_cnt               (dc_context_t* context, uint32_t ch
  * Get the number of _fresh_ messages in a chat.
  * Typically used to implement a badge with a number in the chatlist.
  *
- * If the specified chat is muted,
+ * As muted archived chats are not unarchived automatically,
+ * a similar information is needed for the @ref dc_get_chatlist() "archive link" as well:
+ * here, the number of archived chats containing fresh messages is returned.
+ *
+ * If the specified chat is muted or the @ref dc_get_chatlist() "archive link",
  * the UI should show the badge counter "less obtrusive",
  * e.g. using "gray" instead of "red" color.
  *
@@ -4735,6 +4739,37 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
+
+/**
+ * Return the address that verified a contact
+ *
+ * The UI may use this in addition to a checkmark showing the verification status
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 
+ *    A string containing the verifiers address. If it is the same address as the contact itself,
+ *    we verified the contact ourself. If it is an empty string, we don't have verifier 
+ *    information or the contact is not verified.
+ */
+char*           dc_contact_get_verifier_addr       (dc_contact_t* contact);
+
+
+/**
+ * Return the `ContactId` that verified a contact
+ *
+ * The UI may use this in addition to a checkmark showing the verification status
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 
+ *    The `ContactId` of the verifiers address. If it is the same address as the contact itself,
+ *    we verified the contact ourself. If it is 0, we don't have verifier information or 
+ *    the contact is not verified.
+ */
+uint32_t       dc_contact_get_verifier_id      (dc_contact_t* contact);
+
+
 /**
  * @class dc_provider_t
  *

deltachat-ffi/src/lib.rs

@@ -2182,7 +2182,7 @@ pub unsafe extern "C" fn dc_imex(
         eprintln!("ignoring careless call to dc_imex()");
         return;
     }
-    let what = match imex::ImexMode::from_i32(what_raw as i32) {
+    let what = match imex::ImexMode::from_i32(what_raw) {
         Some(what) => what,
         None => {
             eprintln!("ignoring invalid argument {} to dc_imex", what_raw);
@@ -2253,10 +2253,7 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
     msg_id: u32,
     setup_code: *const libc::c_char,
 ) -> libc::c_int {
-    if context.is_null()
-        || msg_id <= constants::DC_MSG_ID_LAST_SPECIAL as u32
-        || setup_code.is_null()
-    {
+    if context.is_null() || msg_id <= constants::DC_MSG_ID_LAST_SPECIAL || setup_code.is_null() {
         eprintln!("ignoring careless call to dc_continue_key_transfer()");
         return 0;
     }
@@ -2447,15 +2444,9 @@ pub unsafe extern "C" fn dc_get_locations(
     };
 
     block_on(async move {
-        let res = location::get_range(
-            ctx,
-            chat_id,
-            contact_id,
-            timestamp_begin as i64,
-            timestamp_end as i64,
-        )
-        .await
-        .unwrap_or_log_default(ctx, "Failed get_locations");
+        let res = location::get_range(ctx, chat_id, contact_id, timestamp_begin, timestamp_end)
+            .await
+            .unwrap_or_log_default(ctx, "Failed get_locations");
         Box::into_raw(Box::new(dc_array_t::from(res)))
     })
 }
@@ -2702,7 +2693,7 @@ pub unsafe extern "C" fn dc_chatlist_get_chat_id(
     }
     let ffi_list = &*chatlist;
     let ctx = &*ffi_list.context;
-    match ffi_list.list.get_chat_id(index as usize) {
+    match ffi_list.list.get_chat_id(index) {
         Ok(chat_id) => chat_id.to_u32(),
         Err(err) => {
             warn!(ctx, "get_chat_id failed: {}", err);
@@ -2722,7 +2713,7 @@ pub unsafe extern "C" fn dc_chatlist_get_msg_id(
     }
     let ffi_list = &*chatlist;
     let ctx = &*ffi_list.context;
-    match ffi_list.list.get_msg_id(index as usize) {
+    match ffi_list.list.get_msg_id(index) {
         Ok(msg_id) => msg_id.map_or(0, |msg_id| msg_id.to_u32()),
         Err(err) => {
             warn!(ctx, "get_msg_id failed: {}", err);
@@ -2753,7 +2744,7 @@ pub unsafe extern "C" fn dc_chatlist_get_summary(
     block_on(async move {
         let summary = ffi_list
             .list
-            .get_summary(ctx, index as usize, maybe_chat)
+            .get_summary(ctx, index, maybe_chat)
             .await
             .log_err(ctx, "get_summary failed")
             .unwrap_or_default();
@@ -3318,6 +3309,8 @@ pub unsafe extern "C" fn dc_msg_get_filebytes(msg: *mut dc_msg_t) -> u64 {
     let ctx = &*ffi_msg.context;
 
     block_on(ffi_msg.message.get_filebytes(ctx))
+        .unwrap_or_log_default(ctx, "Cannot get file size")
+        .unwrap_or_default()
 }
 
 #[no_mangle]
@@ -3972,6 +3965,40 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
         .unwrap_or_default() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_verifier_addr(
+    contact: *mut dc_contact_t,
+) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_verifier_addr()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    let ctx = &*ffi_contact.context;
+    block_on(Contact::get_verifier_addr(
+        ctx,
+        &ffi_contact.contact.get_id(),
+    ))
+    .log_err(ctx, "failed to get verifier for contact")
+    .unwrap_or_default()
+    .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) -> u32 {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_verifier_id()");
+        return 0;
+    }
+    let ffi_contact = &*contact;
+    let ctx = &*ffi_contact.context;
+    let contact_id = block_on(Contact::get_verifier_id(ctx, &ffi_contact.contact.get_id()))
+        .log_err(ctx, "failed to get verifier")
+        .unwrap_or_default()
+        .unwrap_or_default();
+
+    contact_id.to_u32()
+}
 // dc_lot_t
 
 pub type dc_lot_t = lot::Lot;

deltachat-jsonrpc/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.103.0"
+version = "1.105.0"
 description = "DeltaChat JSON-RPC API"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
 license = "MPL-2.0"
@@ -21,10 +20,10 @@ tempfile = "3.3.0"
 log = "0.4"
 async-channel = { version = "1.8.0" }
 futures = { version = "0.3.25" }
-serde_json = "1.0.89"
+serde_json = "1.0.91"
 yerpc = { version = "^0.3.1", features = ["anyhow_expose"] }
 typescript-type-def = { version = "0.5.5", features = ["json_value"] }
-tokio = { version = "1.22.0" }
+tokio = { version = "1.23.1" }
 sanitize-filename = "0.4"
 walkdir = "2.3.2"
 
@@ -33,7 +32,7 @@ axum = { version = "0.6.1", optional = true, features = ["ws"] }
 env_logger = { version = "0.10.0", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.22.0", features = ["full", "rt-multi-thread"] }
+tokio = { version = "1.23.1", features = ["full", "rt-multi-thread"] }
 
 
 [features]

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -48,12 +48,10 @@ pub enum ChatListItemFetchResult {
         dm_chat_contact: Option<u32>,
         was_seen_recently: bool,
     },
-    ArchiveLink,
     #[serde(rename_all = "camelCase")]
-    Error {
-        id: u32,
-        error: String,
-    },
+    ArchiveLink { fresh_message_counter: usize },
+    #[serde(rename_all = "camelCase")]
+    Error { id: u32, error: String },
 }
 
 pub(crate) async fn get_chat_list_item_by_id(
@@ -66,8 +64,12 @@ pub(crate) async fn get_chat_list_item_by_id(
         _ => Some(MsgId::new(entry.1)),
     };
 
+    let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?;
+
     if chat_id.is_archived_link() {
-        return Ok(ChatListItemFetchResult::ArchiveLink);
+        return Ok(ChatListItemFetchResult::ArchiveLink {
+            fresh_message_counter,
+        });
     }
 
     let chat = Chat::load_from_db(ctx, chat_id).await?;
@@ -111,7 +113,6 @@ pub(crate) async fn get_chat_list_item_by_id(
         (None, false)
     };
 
-    let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?;
     let color = color_int_to_hex_string(chat.get_color(ctx).await?);
 
     Ok(ChatListItemFetchResult::ChatListItem {

deltachat-jsonrpc/src/api/types/message.rs

@@ -105,7 +105,7 @@ impl MessageObject {
 
         let sender_contact = Contact::load_from_db(context, message.get_from_id()).await?;
         let sender = ContactObject::try_from_dc_contact(context, sender_contact).await?;
-        let file_bytes = message.get_filebytes(context).await;
+        let file_bytes = message.get_filebytes(context).await?.unwrap_or_default();
         let override_sender_name = message.get_override_sender_name();
 
         let webxdc_info = if message.get_viewtype() == Viewtype::Webxdc {

deltachat-jsonrpc/typescript/package.json

@@ -48,5 +48,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.103.0"
+  "version": "1.105.0"
 }

deltachat-jsonrpc/typescript/test/online.ts

@@ -12,7 +12,7 @@ describe("online tests", function () {
   let accountId1: number, accountId2: number;
 
   before(async function () {
-    this.timeout(12000);
+    this.timeout(60000);
     if (!process.env.DCC_NEW_TMP_EMAIL) {
       if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
         console.error(

deltachat-rpc-client/examples/echobot.py

@@ -17,8 +17,9 @@ async def log_event(event):
 
 
 @hooks.on(events.NewMessage)
-async def echo(msg):
-    await msg.chat.send_text(msg.text)
+async def echo(event):
+    snapshot = event.message_snapshot
+    await snapshot.chat.send_text(snapshot.text)
 
 
 if __name__ == "__main__":

deltachat-rpc-client/examples/echobot_advanced.py

@@ -25,14 +25,34 @@ async def log_error(event):
     logging.error(event.msg)
 
 
-@hooks.on(events.NewMessage(r".+", func=lambda msg: not msg.text.startswith("/")))
-async def echo(msg):
-    await msg.chat.send_text(msg.text)
+@hooks.on(events.MemberListChanged)
+async def on_memberlist_changed(event):
+    logging.info(
+        "member %s was %s", event.member, "added" if event.member_added else "removed"
+    )
 
 
-@hooks.on(events.NewMessage(r"/help"))
-async def help_command(msg):
-    await msg.chat.send_text("Send me any text message and I will echo it back")
+@hooks.on(events.GroupImageChanged)
+async def on_group_image_changed(event):
+    logging.info("group image %s", "deleted" if event.image_deleted else "changed")
+
+
+@hooks.on(events.GroupNameChanged)
+async def on_group_name_changed(event):
+    logging.info("group name changed, old name: %s", event.old_name)
+
+
+@hooks.on(events.NewMessage(func=lambda e: not e.command))
+async def echo(event):
+    snapshot = event.message_snapshot
+    if snapshot.text or snapshot.file:
+        await snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
+
+
+@hooks.on(events.NewMessage(command="/help"))
+async def help_command(event):
+    snapshot = event.message_snapshot
+    await snapshot.chat.send_text("Send me any message and I will echo it back")
 
 
 async def main():

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -1,4 +1,5 @@
 """Delta Chat asynchronous high-level API"""
+from ._utils import AttrDict, run_bot_cli, run_client_cli
 from .account import Account
 from .chat import Chat
 from .client import Bot, Client
@@ -7,4 +8,3 @@ from .contact import Contact
 from .deltachat import DeltaChat
 from .message import Message
 from .rpc import Rpc
-from .utils import AttrDict, run_bot_cli, run_client_cli

deltachat-rpc-client/src/deltachat_rpc_client/_utils.py

@@ -17,6 +17,8 @@ def _camel_to_snake(name: str) -> str:
 
 
 def _to_attrdict(obj):
+    if isinstance(obj, AttrDict):
+        return obj
     if isinstance(obj, dict):
         return AttrDict(obj)
     if isinstance(obj, list):
@@ -112,3 +114,64 @@ async def _run_cli(
                 client.configure(email=args.email, password=args.password)
             )
         await client.run_forever()
+
+
+def extract_addr(text: str) -> str:
+    """extract email address from the given text."""
+    match = re.match(r".*\((.+@.+)\)", text)
+    if match:
+        text = match.group(1)
+    text = text.rstrip(".")
+    return text.strip()
+
+
+def parse_system_image_changed(text: str) -> Optional[Tuple[str, bool]]:
+    """return image changed/deleted info from parsing the given system message text."""
+    text = text.lower()
+    match = re.match(r"group image (changed|deleted) by (.+).", text)
+    if match:
+        action, actor = match.groups()
+        return (extract_addr(actor), action == "deleted")
+    return None
+
+
+def parse_system_title_changed(text: str) -> Optional[Tuple[str, str]]:
+    text = text.lower()
+    match = re.match(r'group name changed from "(.+)" to ".+" by (.+).', text)
+    if match:
+        old_title, actor = match.groups()
+        return (extract_addr(actor), old_title)
+    return None
+
+
+def parse_system_add_remove(text: str) -> Optional[Tuple[str, str, str]]:
+    """return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) tuple.
+    """
+    # You removed member a@b.
+    # You added member a@b.
+    # Member Me (x@y) removed by a@b.
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
+    text = text.lower()
+
+    match = re.match(r"member (.+) (removed|added) by (.+)", text)
+    if match:
+        affected, action, actor = match.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+
+    match = re.match(r"you (removed|added) member (.+)", text)
+    if match:
+        action, affected = match.groups()
+        return action, extract_addr(affected), "me"
+
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr
+
+    return None

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -1,11 +1,11 @@
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
+from ._utils import AttrDict
 from .chat import Chat
 from .const import ChatlistFlag, ContactFlag, SpecialContactId
 from .contact import Contact
 from .message import Message
 from .rpc import Rpc
-from .utils import AttrDict
 
 if TYPE_CHECKING:
     from .deltachat import DeltaChat
@@ -108,7 +108,7 @@ class Account:
             obj = (await obj.get_snapshot()).address
         return Contact(self, await self._rpc.create_contact(self.id, obj, name))
 
-    async def get_contact_by_id(self, contact_id: int) -> Contact:
+    def get_contact_by_id(self, contact_id: int) -> Contact:
         """Return Contact instance for the given contact ID."""
         return Contact(self, contact_id)
 
@@ -212,7 +212,7 @@ class Account:
         """
         return Chat(self, await self._rpc.create_group_chat(self.id, name, protect))
 
-    async def get_chat_by_id(self, chat_id: int) -> Chat:
+    def get_chat_by_id(self, chat_id: int) -> Chat:
         """Return the Chat instance with the given ID."""
         return Chat(self, chat_id)
 
@@ -237,7 +237,7 @@ class Account:
         """
         return await self._rpc.get_chat_securejoin_qr_code_svg(self.id, None)
 
-    async def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id: int) -> Message:
         """Return the Message instance with the given ID."""
         return Message(self, msg_id)
 

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -2,11 +2,11 @@ import calendar
 from datetime import datetime
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
 
+from ._utils import AttrDict
 from .const import ChatVisibility
 from .contact import Contact
 from .message import Message
 from .rpc import Rpc
-from .utils import AttrDict
 
 if TYPE_CHECKING:
     from .account import Account

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -1,12 +1,35 @@
 """Event loop implementations offering high level event handling/hooking."""
+import inspect
 import logging
-from typing import Callable, Dict, Iterable, Optional, Set, Tuple, Type, Union
+from typing import (
+    Callable,
+    Coroutine,
+    Dict,
+    Iterable,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
 
 from deltachat_rpc_client.account import Account
 
-from .const import EventType
-from .events import EventFilter, NewInfoMessage, NewMessage, RawEvent
-from .utils import AttrDict
+from ._utils import (
+    AttrDict,
+    parse_system_add_remove,
+    parse_system_image_changed,
+    parse_system_title_changed,
+)
+from .const import COMMAND_PREFIX, EventType, SystemMessageType
+from .events import (
+    EventFilter,
+    GroupImageChanged,
+    GroupNameChanged,
+    MemberListChanged,
+    NewMessage,
+    RawEvent,
+)
 
 
 class Client:
@@ -21,6 +44,7 @@ class Client:
         self.account = account
         self.logger = logger or logging
         self._hooks: Dict[type, Set[tuple]] = {}
+        self._should_process_messages = 0
         self.add_hooks(hooks or [])
 
     def add_hooks(
@@ -36,12 +60,24 @@ class Client:
         if isinstance(event, type):
             event = event()
         assert isinstance(event, EventFilter)
+        self._should_process_messages += int(
+            isinstance(
+                event,
+                (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged),
+            )
+        )
         self._hooks.setdefault(type(event), set()).add((hook, event))
 
     def remove_hook(self, hook: Callable, event: Union[type, EventFilter]) -> None:
         """Unregister hook from the given event filter."""
         if isinstance(event, type):
             event = event()
+        self._should_process_messages -= int(
+            isinstance(
+                event,
+                (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged),
+            )
+        )
         self._hooks.get(type(event), set()).remove((hook, event))
 
     async def is_configured(self) -> bool:
@@ -56,6 +92,18 @@ class Client:
         self.logger.debug("Account configured")
 
     async def run_forever(self) -> None:
+        """Process events forever."""
+        await self.run_until(lambda _: False)
+
+    async def run_until(
+        self, func: Callable[[AttrDict], Union[bool, Coroutine]]
+    ) -> AttrDict:
+        """Process events until the given callable evaluates to True.
+
+        The callable should accept an AttrDict object representing the
+        last processed event. The event is returned when the callable
+        evaluates to True.
+        """
         self.logger.debug("Listening to incoming events...")
         if await self.is_configured():
             await self.account.start_io()
@@ -68,6 +116,12 @@ class Client:
             if event.type == EventType.INCOMING_MSG:
                 await self._process_messages()
 
+            stop = func(event)
+            if inspect.isawaitable(stop):
+                stop = await stop
+            if stop:
+                return event
+
     async def _on_event(
         self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent
     ) -> None:
@@ -78,17 +132,82 @@ class Client:
                 except Exception as ex:
                     self.logger.exception(ex)
 
-    def _should_process_messages(self) -> bool:
-        return any(issubclass(filter_type, NewMessage) for filter_type in self._hooks)
+    async def _parse_command(self, event: AttrDict) -> None:
+        cmds = [
+            hook[1].command
+            for hook in self._hooks.get(NewMessage, [])
+            if hook[1].command
+        ]
+        parts = event.message_snapshot.text.split(maxsplit=1)
+        payload = parts[1] if len(parts) > 1 else ""
+        cmd = parts.pop(0)
+
+        if "@" in cmd:
+            suffix = "@" + (await self.account.self_contact.get_snapshot()).address
+            if cmd.endswith(suffix):
+                cmd = cmd[: -len(suffix)]
+            else:
+                return
+
+        parts = cmd.split("_")
+        _payload = payload
+        while parts:
+            _cmd = "_".join(parts)
+            if _cmd in cmds:
+                break
+            _payload = (parts.pop() + " " + _payload).rstrip()
+
+        if parts:
+            cmd = _cmd
+            payload = _payload
+
+        event["command"], event["payload"] = cmd, payload
+
+    async def _on_new_msg(self, snapshot: AttrDict) -> None:
+        event = AttrDict(command="", payload="", message_snapshot=snapshot)
+        if not snapshot.is_info and snapshot.text.startswith(COMMAND_PREFIX):
+            await self._parse_command(event)
+        await self._on_event(event, NewMessage)
+
+    async def _handle_info_msg(self, snapshot: AttrDict) -> None:
+        event = AttrDict(message_snapshot=snapshot)
+
+        img_changed = parse_system_image_changed(snapshot.text)
+        if img_changed:
+            _, event["image_deleted"] = img_changed
+            await self._on_event(event, GroupImageChanged)
+            return
+
+        title_changed = parse_system_title_changed(snapshot.text)
+        if title_changed:
+            _, event["old_name"] = title_changed
+            await self._on_event(event, GroupNameChanged)
+            return
+
+        members_changed = parse_system_add_remove(snapshot.text)
+        if members_changed:
+            action, event["member"], _ = members_changed
+            event["member_added"] = action == "added"
+            await self._on_event(event, MemberListChanged)
+            return
+
+        self.logger.warning(
+            "ignoring unsupported system message id=%s text=%s",
+            snapshot.id,
+            snapshot.text,
+        )
 
     async def _process_messages(self) -> None:
-        if self._should_process_messages():
+        if self._should_process_messages:
             for message in await self.account.get_fresh_messages_in_arrival_order():
                 snapshot = await message.get_snapshot()
-                if snapshot.is_info:
-                    await self._on_event(snapshot, NewInfoMessage)
-                else:
-                    await self._on_event(snapshot, NewMessage)
+                await self._on_new_msg(snapshot)
+                if (
+                    snapshot.is_info
+                    and snapshot.system_message_type
+                    != SystemMessageType.WEBXDC_INFO_MESSAGE
+                ):
+                    await self._handle_info_msg(snapshot)
                 await snapshot.message.mark_seen()
 
 

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -1,5 +1,7 @@
 from enum import Enum, IntEnum
 
+COMMAND_PREFIX = "/"
+
 
 class ContactFlag(IntEnum):
     VERIFIED_ONLY = 0x01

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -1,7 +1,7 @@
 from typing import TYPE_CHECKING
 
+from ._utils import AttrDict
 from .rpc import Rpc
-from .utils import AttrDict
 
 if TYPE_CHECKING:
     from .account import Account

deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py

@@ -1,8 +1,8 @@
 from typing import Dict, List
 
+from ._utils import AttrDict
 from .account import Account
 from .rpc import Rpc
-from .utils import AttrDict
 
 
 class DeltaChat:

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -4,8 +4,8 @@ import re
 from abc import ABC, abstractmethod
 from typing import Callable, Iterable, Iterator, Optional, Set, Tuple, Union
 
+from ._utils import AttrDict
 from .const import EventType
-from .utils import AttrDict
 
 
 def _tuple_of(obj, type_: type) -> tuple:
@@ -91,8 +91,19 @@ class RawEvent(EventFilter):
 class NewMessage(EventFilter):
     """Matches whenever a new message arrives.
 
-    Warning: registering a handler for this event or any subclass will cause the messages
+    Warning: registering a handler for this event will cause the messages
     to be marked as read. Its usage is mainly intended for bots.
+
+    :param pattern: if set, this Pattern will be used to filter the message by its text
+                    content.
+    :param command: If set, only match messages with the given command (ex. /help).
+                    Setting this property implies `is_info==False`.
+    :param is_info: If set to True only match info/system messages, if set to False
+                    only match messages that are not info/system messages. If omitted
+                    info/system messages as well as normal messages will be matched.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
     """
 
     def __init__(
@@ -103,9 +114,17 @@ class NewMessage(EventFilter):
             Callable[[str], bool],
             re.Pattern,
         ] = None,
+        command: Optional[str] = None,
+        is_info: Optional[bool] = None,
         func: Optional[Callable[[AttrDict], bool]] = None,
     ) -> None:
         super().__init__(func=func)
+        self.is_info = is_info
+        if command is not None and not isinstance(command, str):
+            raise TypeError("Invalid command")
+        self.command = command
+        if self.is_info and self.command:
+            raise AttributeError("Can not use command and is_info at the same time.")
         if isinstance(pattern, str):
             pattern = re.compile(pattern)
         if isinstance(pattern, re.Pattern):
@@ -119,13 +138,22 @@ class NewMessage(EventFilter):
         return hash((self.pattern, self.func))
 
     def __eq__(self, other) -> bool:
-        if type(other) is self.__class__:  # noqa
-            return (self.pattern, self.func) == (other.pattern, other.func)
+        if isinstance(other, NewMessage):
+            return (self.pattern, self.command, self.is_info, self.func) == (
+                other.pattern,
+                other.command,
+                other.is_info,
+                other.func,
+            )
         return False
 
     async def filter(self, event: AttrDict) -> bool:
+        if self.is_info is not None and self.is_info != event.message_snapshot.is_info:
+            return False
+        if self.command and self.command != event.command:
+            return False
         if self.pattern:
-            match = self.pattern(event.text)
+            match = self.pattern(event.message_snapshot.text)
             if inspect.isawaitable(match):
                 match = await match
             if not match:
@@ -133,8 +161,91 @@ class NewMessage(EventFilter):
         return await super()._call_func(event)
 
 
-class NewInfoMessage(NewMessage):
-    """Matches whenever a new info/system message arrives."""
+class MemberListChanged(EventFilter):
+    """Matches when a group member is added or removed.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param added: If set to True only match if a member was added, if set to False
+                  only match if a member was removed. If omitted both, member additions
+                  and removals, will be matched.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(self, added: Optional[bool] = None, **kwargs):
+        super().__init__(**kwargs)
+        self.added = added
+
+    def __hash__(self) -> int:
+        return hash((self.added, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, MemberListChanged):
+            return (self.added, self.func) == (other.added, other.func)
+        return False
+
+    async def filter(self, event: AttrDict) -> bool:
+        if self.added is not None and self.added != event.member_added:
+            return False
+        return await self._call_func(event)
+
+
+class GroupImageChanged(EventFilter):
+    """Matches when the group image is changed.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param deleted: If set to True only match if the image was deleted, if set to False
+                    only match if a new image was set. If omitted both, image changes and
+                    removals, will be matched.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(self, deleted: Optional[bool] = None, **kwargs):
+        super().__init__(**kwargs)
+        self.deleted = deleted
+
+    def __hash__(self) -> int:
+        return hash((self.deleted, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, GroupImageChanged):
+            return (self.deleted, self.func) == (other.deleted, other.func)
+        return False
+
+    async def filter(self, event: AttrDict) -> bool:
+        if self.deleted is not None and self.deleted != event.image_deleted:
+            return False
+        return await self._call_func(event)
+
+
+class GroupNameChanged(EventFilter):
+    """Matches when the group name is changed.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __hash__(self) -> int:
+        return hash((GroupNameChanged, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, GroupNameChanged):
+            return self.func == other.func
+        return False
+
+    async def filter(self, event: AttrDict) -> bool:
+        return await self._call_func(event)
 
 
 class HookCollection:

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -1,8 +1,9 @@
-from typing import TYPE_CHECKING
+import json
+from typing import TYPE_CHECKING, Union
 
+from ._utils import AttrDict
 from .contact import Contact
 from .rpc import Rpc
-from .utils import AttrDict
 
 if TYPE_CHECKING:
     from .account import Account
@@ -47,3 +48,23 @@ class Message:
     async def mark_seen(self) -> None:
         """Mark the message as seen."""
         await self._rpc.markseen_msgs(self.account.id, [self.id])
+
+    async def send_webxdc_status_update(
+        self, update: Union[dict, str], description: str
+    ) -> None:
+        """Send a webxdc status update. This message must be a webxdc."""
+        if not isinstance(update, str):
+            update = json.dumps(update)
+        await self._rpc.send_webxdc_status_update(
+            self.account.id, self.id, update, description
+        )
+
+    async def get_webxdc_status_updates(self, last_known_serial: int = 0) -> list:
+        return json.loads(
+            await self._rpc.get_webxdc_status_updates(
+                self.account.id, self.id, last_known_serial
+            )
+        )
+
+    async def get_webxdc_info(self) -> dict:
+        return await self._rpc.get_webxdc_info(self.account.id, self.id)

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -1,21 +1,22 @@
 import json
 import os
-from typing import AsyncGenerator, List
+from typing import AsyncGenerator, List, Optional
 
 import aiohttp
 import pytest_asyncio
 
-from .account import Account
-from .client import Bot
-from .deltachat import DeltaChat
+from . import Account, AttrDict, Bot, Client, DeltaChat, EventType, Message
 from .rpc import Rpc
 
 
 async def get_temp_credentials() -> dict:
     url = os.getenv("DCC_NEW_TMP_EMAIL")
     assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set"
+
+    # Replace default 5 minute timeout with a 1 minute timeout.
+    timeout = aiohttp.ClientTimeout(total=60)
     async with aiohttp.ClientSession() as session:
-        async with session.post(url) as response:
+        async with session.post(url, timeout=timeout) as response:
             return json.loads(await response.text())
 
 
@@ -29,12 +30,17 @@ class ACFactory:
     async def get_unconfigured_bot(self) -> Bot:
         return Bot(await self.get_unconfigured_account())
 
-    async def new_configured_account(self) -> Account:
+    async def new_preconfigured_account(self) -> Account:
+        """Make a new account with configuration options set, but configuration not started."""
         credentials = await get_temp_credentials()
         account = await self.get_unconfigured_account()
-        assert not await account.is_configured()
         await account.set_config("addr", credentials["email"])
         await account.set_config("mail_pw", credentials["password"])
+        assert not await account.is_configured()
+        return account
+
+    async def new_configured_account(self) -> Account:
+        account = await self.new_preconfigured_account()
         await account.configure()
         assert await account.is_configured()
         return account
@@ -51,6 +57,44 @@ class ACFactory:
             await account.start_io()
         return accounts
 
+    async def send_message(
+        self,
+        to_account: Account,
+        from_account: Optional[Account] = None,
+        text: Optional[str] = None,
+        file: Optional[str] = None,
+        group: Optional[str] = None,
+    ) -> Message:
+        if not from_account:
+            from_account = (await self.get_online_accounts(1))[0]
+        to_contact = await from_account.create_contact(
+            await to_account.get_config("addr")
+        )
+        if group:
+            to_chat = await from_account.create_group(group)
+            await to_chat.add_contact(to_contact)
+        else:
+            to_chat = await to_contact.create_chat()
+        return await to_chat.send_message(text=text, file=file)
+
+    async def process_message(
+        self,
+        to_client: Client,
+        from_account: Optional[Account] = None,
+        text: Optional[str] = None,
+        file: Optional[str] = None,
+        group: Optional[str] = None,
+    ) -> AttrDict:
+        await self.send_message(
+            to_account=to_client.account,
+            from_account=from_account,
+            text=text,
+            file=file,
+            group=group,
+        )
+
+        return await to_client.run_until(lambda e: e.type == EventType.INCOMING_MSG)
+
 
 @pytest_asyncio.fixture
 async def rpc(tmp_path) -> AsyncGenerator:

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -14,9 +14,7 @@ class Rpc:
         if accounts_dir:
             kwargs["env"] = {
                 **kwargs.get("env", os.environ),
-                "DC_ACCOUNTS_PATH": os.path.abspath(
-                    os.path.expanduser(str(accounts_dir))
-                ),
+                "DC_ACCOUNTS_PATH": str(accounts_dir),
             }
 
         self._kwargs = kwargs

deltachat-rpc-client/tests/test_something.py

@@ -1,6 +1,8 @@
+from unittest.mock import MagicMock
+
 import pytest
 
-from deltachat_rpc_client import AttrDict, EventType, events
+from deltachat_rpc_client import EventType, events
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
@@ -39,6 +41,16 @@ async def test_acfactory(acfactory) -> None:
     print("Successful configuration")
 
 
+@pytest.mark.asyncio
+async def test_configure_starttls(acfactory) -> None:
+    account = await acfactory.new_preconfigured_account()
+
+    # Use STARTTLS
+    await account.set_config("mail_security", "2")
+    await account.configure()
+    assert await account.is_configured()
+
+
 @pytest.mark.asyncio
 async def test_account(acfactory) -> None:
     alice, bob = await acfactory.get_online_accounts(2)
@@ -55,7 +67,7 @@ async def test_account(acfactory) -> None:
             msg_id = event.msg_id
             break
 
-    message = await bob.get_message_by_id(msg_id)
+    message = bob.get_message_by_id(msg_id)
     snapshot = await message.get_snapshot()
     assert snapshot.chat_id == chat_id
     assert snapshot.text == "Hello!"
@@ -80,8 +92,8 @@ async def test_account(acfactory) -> None:
     group = await alice.create_group("test group")
     await group.add_contact(alice_contact_bob)
     group_msg = await group.send_message(text="hello")
-    assert group_msg == await alice.get_message_by_id(group_msg.id)
-    assert group == await alice.get_chat_by_id(group.id)
+    assert group_msg == alice.get_message_by_id(group_msg.id)
+    assert group == alice.get_chat_by_id(group.id)
     await alice.delete_messages([group_msg])
 
     await alice.set_config("selfstatus", "test")
@@ -114,11 +126,11 @@ async def test_chat(acfactory) -> None:
             chat_id = event.chat_id
             msg_id = event.msg_id
             break
-    message = await bob.get_message_by_id(msg_id)
+    message = bob.get_message_by_id(msg_id)
     snapshot = await message.get_snapshot()
     assert snapshot.chat_id == chat_id
     assert snapshot.text == "Hello!"
-    bob_chat_alice = await bob.get_chat_by_id(chat_id)
+    bob_chat_alice = bob.get_chat_by_id(chat_id)
 
     assert alice_chat_bob != bob_chat_alice
     assert repr(alice_chat_bob)
@@ -172,7 +184,7 @@ async def test_contact(acfactory) -> None:
     bob_addr = await bob.get_config("addr")
     alice_contact_bob = await alice.create_contact(bob_addr, "Bob")
 
-    assert alice_contact_bob == await alice.get_contact_by_id(alice_contact_bob.id)
+    assert alice_contact_bob == alice.get_contact_by_id(alice_contact_bob.id)
     assert repr(alice_contact_bob)
     await alice_contact_bob.block()
     await alice_contact_bob.unblock()
@@ -199,7 +211,7 @@ async def test_message(acfactory) -> None:
             msg_id = event.msg_id
             break
 
-    message = await bob.get_message_by_id(msg_id)
+    message = bob.get_message_by_id(msg_id)
     snapshot = await message.get_snapshot()
     assert snapshot.chat_id == chat_id
     assert snapshot.text == "Hello!"
@@ -216,31 +228,42 @@ async def test_message(acfactory) -> None:
 
 @pytest.mark.asyncio
 async def test_bot(acfactory) -> None:
-    async def callback(e):
-        res.append(e)
-
-    res = []
+    mock = MagicMock()
+    user = (await acfactory.get_online_accounts(1))[0]
     bot = await acfactory.new_configured_bot()
+
     assert await bot.is_configured()
     assert await bot.account.get_config("bot") == "1"
 
-    bot.add_hook(callback, events.RawEvent(EventType.INFO))
-    info_event = AttrDict(account=bot.account, type=EventType.INFO, msg="info")
-    warn_event = AttrDict(account=bot.account, type=EventType.WARNING, msg="warning")
-    await bot._on_event(info_event)
-    await bot._on_event(warn_event)
-    assert info_event in res
-    assert warn_event not in res
-    assert len(res) == 1
+    hook = lambda e: mock.hook(e.msg_id), events.RawEvent(EventType.INCOMING_MSG)
+    bot.add_hook(*hook)
+    event = await acfactory.process_message(
+        from_account=user, to_client=bot, text="Hello!"
+    )
+    mock.hook.assert_called_once_with(event.msg_id)
+    bot.remove_hook(*hook)
 
-    res = []
-    bot.add_hook(callback, events.NewMessage(r"hello"))
-    snapshot1 = AttrDict(text="hello")
-    snapshot2 = AttrDict(text="hello, world")
-    snapshot3 = AttrDict(text="hey!")
-    for snapshot in [snapshot1, snapshot2, snapshot3]:
-        await bot._on_event(snapshot, events.NewMessage)
-    assert len(res) == 2
-    assert snapshot1 in res
-    assert snapshot2 in res
-    assert snapshot3 not in res
+    track = lambda e: mock.hook(e.message_snapshot.id)
+
+    mock.hook.reset_mock()
+    hook = track, events.NewMessage(r"hello")
+    bot.add_hook(*hook)
+    bot.add_hook(track, events.NewMessage(command="/help"))
+    event = await acfactory.process_message(
+        from_account=user, to_client=bot, text="hello"
+    )
+    mock.hook.assert_called_with(event.msg_id)
+    event = await acfactory.process_message(
+        from_account=user, to_client=bot, text="hello!"
+    )
+    mock.hook.assert_called_with(event.msg_id)
+    await acfactory.process_message(from_account=user, to_client=bot, text="hey!")
+    assert len(mock.hook.mock_calls) == 2
+    bot.remove_hook(*hook)
+
+    mock.hook.reset_mock()
+    await acfactory.process_message(from_account=user, to_client=bot, text="hello")
+    event = await acfactory.process_message(
+        from_account=user, to_client=bot, text="/help"
+    )
+    mock.hook.assert_called_once_with(event.msg_id)

deltachat-rpc-client/tests/test_webxdc.py

@@ -0,0 +1,50 @@
+import pytest
+
+from deltachat_rpc_client import EventType
+
+
+@pytest.mark.asyncio
+async def test_webxdc(acfactory) -> None:
+    alice, bob = await acfactory.get_online_accounts(2)
+
+    bob_addr = await bob.get_config("addr")
+    alice_contact_bob = await alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = await alice_contact_bob.create_chat()
+    await alice_chat_bob.send_message(
+        text="Let's play chess!", file="../test-data/webxdc/chess.xdc"
+    )
+
+    while True:
+        event = await bob.wait_for_event()
+        if event.type == EventType.INCOMING_MSG:
+            bob_chat_alice = bob.get_chat_by_id(event.chat_id)
+            message = bob.get_message_by_id(event.msg_id)
+            break
+
+    webxdc_info = await message.get_webxdc_info()
+    assert webxdc_info == {
+        "document": None,
+        "icon": "icon.png",
+        "internetAccess": False,
+        "name": "Chess Board",
+        "sourceCodeUrl": None,
+        "summary": None,
+    }
+
+    status_updates = await message.get_webxdc_status_updates()
+    assert status_updates == []
+
+    await bob_chat_alice.accept()
+    await message.send_webxdc_status_update({"payload": 42}, "")
+    await message.send_webxdc_status_update({"payload": "Second update"}, "description")
+
+    status_updates = await message.get_webxdc_status_updates()
+    assert status_updates == [
+        {"payload": 42, "serial": 1, "max_serial": 2},
+        {"payload": "Second update", "serial": 2, "max_serial": 2},
+    ]
+
+    status_updates = await message.get_webxdc_status_updates(1)
+    assert status_updates == [
+        {"payload": "Second update", "serial": 2, "max_serial": 2},
+    ]

deltachat-rpc-client/tox.ini

@@ -13,7 +13,6 @@ passenv =
     DCC_NEW_TMP_EMAIL
 deps =
     pytest
-    pytest-async
     pytest-asyncio
     aiohttp
     aiodns

deltachat-rpc-server/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.103.0"
+version = "1.105.0"
 description = "DeltaChat JSON-RPC server"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 readme = "README.md"
 license = "MPL-2.0"
@@ -20,7 +19,7 @@ anyhow = "1"
 env_logger = { version = "0.10.0" }
 futures-lite = "1.12.0"
 log = "0.4"
-serde_json = "1.0.89"
+serde_json = "1.0.91"
 serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.22.0", features = ["io-std"] }
+tokio = { version = "1.23.1", features = ["io-std"] }
 yerpc = { version = "0.3.1", features = ["anyhow_expose"] }

deltachat_derive/Cargo.toml

@@ -1,7 +1,6 @@
 [package]
 name = "deltachat_derive"
 version = "2.0.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 

format-flowed/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "format-flowed"
+version = "1.0.0"
+description = "format=flowed support"
+edition = "2021"
+license = "MPL-2.0"
+
+keywords = ["email"]
+categories = ["email"]
+
+[dependencies]

format-flowed/src/lib.rs

@@ -24,10 +24,14 @@
 fn format_line_flowed(line: &str, prefix: &str) -> String {
     let mut result = String::new();
     let mut buffer = prefix.to_string();
-    let mut after_space = false;
+    let mut after_space = prefix.ends_with(' ');
 
     for c in line.chars() {
         if c == ' ' {
+            if buffer.is_empty() {
+                // Space stuffing, see RFC 3676
+                buffer.push(' ');
+            }
             buffer.push(c);
             after_space = true;
         } else if c == '>' {
@@ -51,14 +55,14 @@ fn format_line_flowed(line: &str, prefix: &str) -> String {
     result + &buffer
 }
 
-/// Returns text formatted according to RFC 3767 (format=flowed).
+/// Returns text formatted according to RFC 3676 (format=flowed).
 ///
 /// This function accepts text separated by LF, but returns text
 /// separated by CRLF.
 ///
 /// RFC 2646 technique is used to insert soft line breaks, so DelSp
 /// SHOULD be set to "no" when sending.
-pub(crate) fn format_flowed(text: &str) -> String {
+pub fn format_flowed(text: &str) -> String {
     let mut result = String::new();
 
     for line in text.split('\n') {
@@ -66,21 +70,20 @@ pub(crate) fn format_flowed(text: &str) -> String {
             result += "\r\n";
         }
 
-        let line_no_prefix = line.strip_prefix('>');
-        let is_quote = line_no_prefix.is_some();
-        let line = line_no_prefix.unwrap_or(line).trim();
-        let prefix = if is_quote { "> " } else { "" };
+        let line = line.trim_end();
+        let quote_depth = line.chars().take_while(|&c| c == '>').count();
+        let (prefix, mut line) = line.split_at(quote_depth);
 
-        if prefix.len() + line.len() > 78 {
-            result += &format_line_flowed(line, prefix);
-        } else {
-            result += prefix;
-            if prefix.is_empty() && line.starts_with('>') {
-                // Space stuffing, see RFC 3676
-                result.push(' ');
+        let mut prefix = prefix.to_string();
+
+        if quote_depth > 0 {
+            if let Some(s) = line.strip_prefix(' ') {
+                line = s;
+                prefix += " ";
             }
-            result += line;
         }
+
+        result += &format_line_flowed(line, &prefix);
     }
 
     result
@@ -105,9 +108,6 @@ pub fn format_flowed_quote(text: &str) -> String {
 ///
 /// Lines must be separated by single LF.
 ///
-/// Quote processing is not supported, it is assumed that they are
-/// deleted during simplification.
-///
 /// Signature separator line is not processed here, it is assumed to
 /// be stripped beforehand.
 pub fn unformat_flowed(text: &str, delsp: bool) -> String {
@@ -115,6 +115,12 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String {
     let mut skip_newline = true;
 
     for line in text.split('\n') {
+        let line = if !result.is_empty() && skip_newline {
+            line.trim_start_matches('>')
+        } else {
+            line
+        };
+
         // Revert space-stuffing
         let line = line.strip_prefix(' ').unwrap_or(line);
 
@@ -141,12 +147,23 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::TestContext;
 
     #[test]
     fn test_format_flowed() {
+        let text = "";
+        assert_eq!(format_flowed(text), "");
+
         let text = "Foo bar baz";
-        assert_eq!(format_flowed(text), "Foo bar baz");
+        assert_eq!(format_flowed(text), text);
+
+        let text = ">Foo bar";
+        assert_eq!(format_flowed(text), text);
+
+        let text = "> Foo bar";
+        assert_eq!(format_flowed(text), text);
+
+        let text = ">\n\nA";
+        assert_eq!(format_flowed(text), ">\r\n\r\nA");
 
         let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\
                     \n\
@@ -160,16 +177,45 @@ mod tests {
         let text = "> A quote";
         assert_eq!(format_flowed(text), "> A quote");
 
+        let text = "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A";
+        assert_eq!(
+            format_flowed(text),
+            "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > \r\n> A"
+        );
+
         // Test space stuffing of wrapped lines
         let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
                     >                               \n\
                     > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
         let expected = "> This is the Autocrypt Setup Message used to transfer your key between \r\n\
                         > clients.\r\n\
-                        > \r\n\
+                        >\r\n\
                         > To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
                         > client and enter the setup code presented on the generating device.";
         assert_eq!(format_flowed(text), expected);
+
+        let text = ">> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    >>                               \n\
+                    >> To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = ">> This is the Autocrypt Setup Message used to transfer your key between \r\n\
+                        >> clients.\r\n\
+                        >>\r\n\
+                        >> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        >> client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
+
+        // Test space stuffing of spaces.
+        let text = " Foo bar baz";
+        assert_eq!(format_flowed(text), "  Foo bar baz");
+
+        let text = "   Foo bar baz";
+        assert_eq!(format_flowed(text), "    Foo bar baz");
+
+        let text =
+            "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA AAAAAA";
+        let expected =
+            "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA \r\nAAAAAA";
+        assert_eq!(format_flowed(text), expected);
     }
 
     #[test]
@@ -180,6 +226,16 @@ mod tests {
             "this is a very long message that should be wrapped using format=flowed and \
                         unwrapped on the receiver";
         assert_eq!(unformat_flowed(text, false), expected);
+
+        let text = "  Foo bar";
+        let expected = " Foo bar";
+        assert_eq!(unformat_flowed(text, false), expected);
+
+        let text =
+            "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > \n> A";
+        let expected =
+            "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A";
+        assert_eq!(unformat_flowed(text, false), expected);
     }
 
     #[test]
@@ -202,25 +258,4 @@ mod tests {
             > unwrapped on the receiver";
         assert_eq!(format_flowed_quote(quote), expected);
     }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_send_quotes() -> anyhow::Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-        let chat = alice.create_chat(&bob).await;
-
-        let sent = alice.send_text(chat.id, "> First quote").await;
-        let received = bob.recv_msg(&sent).await;
-        assert_eq!(received.text.as_deref(), Some("> First quote"));
-        assert!(received.quoted_text().is_none());
-        assert!(received.quoted_message(&bob).await?.is_none());
-
-        let sent = alice.send_text(chat.id, "> Second quote").await;
-        let received = bob.recv_msg(&sent).await;
-        assert_eq!(received.text.as_deref(), Some("> Second quote"));
-        assert!(received.quoted_text().is_none());
-        assert!(received.quoted_message(&bob).await?.is_none());
-
-        Ok(())
-    }
 }

fuzz/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "deltachat-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2021"
+
+[dev-dependencies]
+bolero = "0.8"
+
+[dependencies]
+mailparse = "0.13"
+deltachat = { path = ".." }
+format-flowed = { path = "../format-flowed" }
+
+[workspace]
+members = ["."]
+
+[[test]]
+name = "fuzz_dateparse"
+path = "fuzz_targets/fuzz_dateparse.rs"
+harness = false
+
+[[test]]
+name = "fuzz_simplify"
+path = "fuzz_targets/fuzz_simplify.rs"
+harness = false
+
+[[test]]
+name = "fuzz_mailparse"
+path = "fuzz_targets/fuzz_mailparse.rs"
+harness = false
+
+[[test]]
+name = "fuzz_format_flowed"
+path = "fuzz_targets/fuzz_format_flowed.rs"
+harness = false

fuzz/fuzz_targets/fuzz_dateparse.rs

@@ -0,0 +1,10 @@
+use bolero::check;
+
+fn main() {
+    check!().for_each(|data: &[u8]| match std::str::from_utf8(data) {
+        Ok(input) => {
+            mailparse::dateparse(input).ok();
+        }
+        Err(_err) => {}
+    });
+}

fuzz/fuzz_targets/fuzz_format_flowed.rs

@@ -0,0 +1,22 @@
+use bolero::check;
+use format_flowed::{format_flowed, unformat_flowed};
+
+fn round_trip(input: &str) -> String {
+    let mut input = format_flowed(input);
+    input.retain(|c| c != '\r');
+    unformat_flowed(&input, false)
+}
+
+fn main() {
+    check!().for_each(|data: &[u8]| {
+        if let Ok(input) = std::str::from_utf8(data.into()) {
+            let input = input.trim().to_string();
+
+            // Only consider inputs that are the result of unformatting format=flowed text.
+            // At least this means that lines don't contain any trailing whitespace.
+            let input = round_trip(&input);
+            let output = round_trip(&input);
+            assert_eq!(input, output);
+        }
+    });
+}

fuzz/fuzz_targets/fuzz_mailparse.rs

@@ -0,0 +1,7 @@
+use bolero::check;
+
+fn main() {
+    check!().for_each(|data: &[u8]| {
+        mailparse::parse_mail(data).ok();
+    });
+}

fuzz/fuzz_targets/fuzz_simplify.rs

@@ -0,0 +1,13 @@
+use bolero::check;
+
+use deltachat::fuzzing::simplify;
+
+fn main() {
+    check!().for_each(|data: &[u8]| match String::from_utf8(data.to_vec()) {
+        Ok(input) => {
+            simplify(input.clone(), true);
+            simplify(input, false);
+        }
+        Err(_err) => {}
+    });
+}

node/test/test.js

@@ -1,32 +1,29 @@
 // @ts-check
-import DeltaChat, { Message } from '../dist'
-import binding from '../binding'
+import DeltaChat from '../dist'
 
-import { deepEqual, deepStrictEqual, strictEqual } from 'assert'
+import { deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
 import { EventId2EventName, C } from '../dist/constants'
 import { join } from 'path'
-import { mkdtempSync, statSync } from 'fs'
-import { tmpdir } from 'os'
+import { statSync } from 'fs'
 import { Context } from '../dist/context'
+import fetch from 'node-fetch'
 chai.use(chaiAsPromised)
+chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 
 async function createTempUser(url) {
-  const fetch = require('node-fetch')
-
   async function postData(url = '') {
     // Default options are marked with *
     const response = await fetch(url, {
       method: 'POST', // *GET, POST, PUT, DELETE, etc.
-      mode: 'cors', // no-cors, *cors, same-origin
-      cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached
-      credentials: 'same-origin', // include, *same-origin, omit
       headers: {
         'cache-control': 'no-cache',
       },
-      referrerPolicy: 'no-referrer', // no-referrer, *client
     })
+    if (!response.ok) {
+      throw new Error('request failed: ' + response.body.read())
+    }
     return response.json() // parses JSON response into native JavaScript objects
   }
 
@@ -121,7 +118,7 @@ describe('JSON RPC', function () {
     const promises = {}
     dc.startJsonRpcHandler((msg) => {
       const response = JSON.parse(msg)
-      promises[response.id](response)
+      if (response.hasOwnProperty('id')) promises[response.id](response)
       delete promises[response.id]
     })
     const call = (request) => {

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.103.0"
+  "version": "1.105.0"
 }

python/src/deltachat/contact.py

@@ -75,6 +75,10 @@ class Contact(object):
         """Return True if the contact is verified."""
         return lib.dc_contact_is_verified(self._dc_contact)
 
+    def get_verifier(self, contact):
+        """Return the address of the contact that verified the contact"""
+        return from_dc_charpointer(lib.dc_contact_get_verifier_addr(contact._dc_contact))
+
     def get_profile_image(self) -> Optional[str]:
         """Get contact profile image.
 

python/src/deltachat/testplugin.py

@@ -176,7 +176,7 @@ class TestProcess:
                 try:
                     yield self._configlist[index]
                 except IndexError:
-                    res = requests.post(liveconfig_opt)
+                    res = requests.post(liveconfig_opt, timeout=60)
                     if res.status_code != 200:
                         pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text))
                     d = res.json()

python/tests/data/r

@@ -0,0 +1,2 @@
+
+hello

python/tests/test_0_complex_or_slow.py

@@ -123,6 +123,7 @@ class TestGroupStressTests:
 
 def test_qr_verified_group_and_chatting(acfactory, lp):
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+    ac1_addr = ac1.get_self_contact().addr
     lp.sec("ac1: create verified-group QR, ac2 scans and joins")
     chat1 = ac1.create_group_chat("hello", verified=True)
     assert chat1.is_protected()
@@ -141,12 +142,17 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
     msg_out = chat1.send_text("hello")
     assert msg_out.is_encrypted()
 
-    lp.sec("ac2: read message and check it's verified chat")
+    lp.sec("ac2: read message and check that it's a verified chat")
     msg = ac2._evtracker.wait_next_incoming_message()
     assert msg.text == "hello"
     assert msg.chat.is_protected()
     assert msg.is_encrypted()
 
+    lp.sec("ac2: Check that ac2 verified ac1")
+    # If we verified the contact ourselves then verifier addr == contact addr
+    ac2_ac1_contact = ac2.get_contacts()[0]
+    assert ac2.get_self_contact().get_verifier(ac2_ac1_contact) == ac1_addr
+
     lp.sec("ac2: send message and let ac1 read it")
     chat2.send_text("world")
     msg = ac1._evtracker.wait_next_incoming_message()
@@ -168,6 +174,12 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
     assert msg.is_system_message()
     assert not msg.error
 
+    lp.sec("ac2: Check that ac1 verified ac3 for ac2")
+    ac2_ac1_contact = ac2.get_contacts()[0]
+    assert ac2.get_self_contact().get_verifier(ac2_ac1_contact) == ac1_addr
+    ac2_ac3_contact = ac2.get_contacts()[1]
+    assert ac2.get_self_contact().get_verifier(ac2_ac3_contact) == ac1_addr
+
     lp.sec("ac2: send message and let ac3 read it")
     chat2.send_text("hi")
     # Skip system message about added member

python/tests/test_3_offline.py

@@ -450,24 +450,25 @@ class TestOfflineChat:
         assert msg.filemime == "image/png"
 
     @pytest.mark.parametrize(
-        "typein,typeout",
+        "fn,typein,typeout",
         [
-            (None, "application/octet-stream"),
-            ("text/plain", "text/plain"),
-            ("image/png", "image/png"),
+            ("r", None, "application/octet-stream"),
+            ("r.txt", None, "text/plain"),
+            ("r.txt", "text/plain", "text/plain"),
+            ("r.txt", "image/png", "image/png"),
         ],
     )
-    def test_message_file(self, ac1, chat1, data, lp, typein, typeout):
+    def test_message_file(self, ac1, chat1, data, lp, fn, typein, typeout):
         lp.sec("sending file")
-        fn = data.get_path("r.txt")
-        msg = chat1.send_file(fn, typein)
+        fp = data.get_path(fn)
+        msg = chat1.send_file(fp, typein)
         assert msg
         assert msg.id > 0
         assert msg.is_file()
         assert os.path.exists(msg.filename)
         assert msg.filename.endswith(msg.basename)
         assert msg.filemime == typeout
-        msg2 = chat1.send_file(fn, typein)
+        msg2 = chat1.send_file(fp, typein)
         assert msg2 != msg
         assert msg2.filename != msg.filename
 

python/tox.ini

@@ -38,6 +38,8 @@ passenv =
 skipsdist = True
 deps = auditwheel
 passenv =
+    DCC_RS_DEV
+    DCC_RS_TARGET
     AUDITWHEEL_ARCH
     AUDITWHEEL_PLAT
     AUDITWHEEL_POLICY
@@ -49,7 +51,8 @@ skipsdist = True
 skip_install = True
 deps =
     flake8
-    isort
+# isort 5.11.0 is broken: https://github.com/PyCQA/isort/issues/2031
+    isort<5.11.0
     black
     # pygments required by rst-lint
     pygments

rust-toolchain

@@ -1 +0,0 @@
-1.65.0

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.65.0
+RUST_VERSION=1.64.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

src/accounts.rs

@@ -64,7 +64,7 @@ impl Accounts {
         let events = Events::new();
         let stockstrings = StockStrings::new();
         let accounts = config
-            .load_accounts(&events, &stockstrings)
+            .load_accounts(&events, &stockstrings, &dir)
             .await
             .context("failed to load accounts")?;
 
@@ -107,10 +107,11 @@ impl Accounts {
     ///
     /// Returns account ID.
     pub async fn add_account(&mut self) -> Result<u32> {
-        let account_config = self.config.new_account(&self.dir).await?;
+        let account_config = self.config.new_account().await?;
+        let dbfile = account_config.dbfile(&self.dir);
 
         let ctx = Context::new(
-            &account_config.dbfile(),
+            &dbfile,
             account_config.id,
             self.events.clone(),
             self.stockstrings.clone(),
@@ -123,10 +124,10 @@ impl Accounts {
 
     /// Adds a new closed account.
     pub async fn add_closed_account(&mut self) -> Result<u32> {
-        let account_config = self.config.new_account(&self.dir).await?;
+        let account_config = self.config.new_account().await?;
 
         let ctx = Context::new_closed(
-            &account_config.dbfile(),
+            &account_config.dbfile(&self.dir),
             account_config.id,
             self.events.clone(),
             self.stockstrings.clone(),
@@ -147,6 +148,8 @@ impl Accounts {
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id) {
+            let account_path = self.dir.join(cfg.dir);
+
             // Spend up to 1 minute trying to remove the files.
             // Files may remain locked up to 30 seconds due to r2d2 bug:
             // https://github.com/sfackler/r2d2/issues/99
@@ -154,7 +157,7 @@ impl Accounts {
             loop {
                 counter += 1;
 
-                if let Err(err) = fs::remove_dir_all(&cfg.dir)
+                if let Err(err) = fs::remove_dir_all(&account_path)
                     .await
                     .context("failed to remove account data")
                 {
@@ -187,16 +190,16 @@ impl Accounts {
         // create new account
         let account_config = self
             .config
-            .new_account(&self.dir)
+            .new_account()
             .await
             .context("failed to create new account")?;
 
-        let new_dbfile = account_config.dbfile();
+        let new_dbfile = account_config.dbfile(&self.dir);
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
         let new_walfile = Context::derive_walfile(&new_dbfile);
 
         let res = {
-            fs::create_dir_all(&account_config.dir)
+            fs::create_dir_all(self.dir.join(&account_config.dir))
                 .await
                 .context("failed to create dir")?;
             fs::rename(&dbfile, &new_dbfile)
@@ -265,12 +268,14 @@ impl Accounts {
         true
     }
 
+    /// Starts background tasks such as IMAP and SMTP loops for all accounts.
     pub async fn start_io(&self) {
         for account in self.accounts.values() {
             account.start_io().await;
         }
     }
 
+    /// Stops background tasks for all accounts.
     pub async fn stop_io(&self) {
         // Sending an event here wakes up event loop even
         // if there are no accounts.
@@ -280,12 +285,14 @@ impl Accounts {
         }
     }
 
+    /// Notifies all accounts that the network may have become available.
     pub async fn maybe_network(&self) {
         for account in self.accounts.values() {
             account.maybe_network().await;
         }
     }
 
+    /// Notifies all accounts that the network connection may have been lost.
     pub async fn maybe_network_lost(&self) {
         for account in self.accounts.values() {
             account.maybe_network_lost().await;
@@ -303,7 +310,10 @@ impl Accounts {
     }
 }
 
+/// Configuration file name.
 pub const CONFIG_NAME: &str = "accounts.toml";
+
+/// Database file name.
 pub const DB_NAME: &str = "dc.db";
 
 /// Account manager configuration file.
@@ -325,6 +335,7 @@ struct InnerConfig {
 }
 
 impl Config {
+    /// Creates a new configuration file in the given account manager directory.
     pub async fn new(dir: &Path) -> Result<Self> {
         let inner = InnerConfig {
             accounts: Vec::new(),
@@ -350,8 +361,17 @@ impl Config {
 
     /// Read a configuration from the given file into memory.
     pub async fn from_file(file: PathBuf) -> Result<Self> {
+        let dir = file.parent().context("can't get config file directory")?;
         let bytes = fs::read(&file).await.context("failed to read file")?;
-        let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
+        let mut inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
+
+        // Previous versions of the core stored absolute paths in account config.
+        // Convert them to relative paths.
+        for account in &mut inner.accounts {
+            if let Ok(new_dir) = account.dir.strip_prefix(dir) {
+                account.dir = new_dir.to_path_buf();
+            }
+        }
 
         Ok(Config { file, inner })
     }
@@ -364,12 +384,13 @@ impl Config {
         &self,
         events: &Events,
         stockstrings: &StockStrings,
+        dir: &Path,
     ) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
 
         for account_config in &self.inner.accounts {
             let ctx = Context::new(
-                &account_config.dbfile(),
+                &account_config.dbfile(dir),
                 account_config.id,
                 events.clone(),
                 stockstrings.clone(),
@@ -378,7 +399,7 @@ impl Config {
             .with_context(|| {
                 format!(
                     "failed to create context from file {:?}",
-                    account_config.dbfile()
+                    account_config.dbfile(dir)
                 )
             })?;
 
@@ -388,12 +409,12 @@ impl Config {
         Ok(accounts)
     }
 
-    /// Create a new account in the given root directory.
-    async fn new_account(&mut self, dir: &Path) -> Result<AccountConfig> {
+    /// Creates a new account in the account manager directory.
+    async fn new_account(&mut self) -> Result<AccountConfig> {
         let id = {
             let id = self.inner.next_id;
             let uuid = Uuid::new_v4();
-            let target_dir = dir.join(uuid.to_string());
+            let target_dir = PathBuf::from(uuid.to_string());
 
             self.inner.accounts.push(AccountConfig {
                 id,
@@ -432,14 +453,17 @@ impl Config {
         self.sync().await
     }
 
+    /// Returns configuration file section for the given account ID.
     fn get_account(&self, id: u32) -> Option<AccountConfig> {
         self.inner.accounts.iter().find(|e| e.id == id).cloned()
     }
 
+    /// Returns the ID of selected account.
     pub fn get_selected_account(&self) -> u32 {
         self.inner.selected_account
     }
 
+    /// Changes selected account ID.
     pub async fn select_account(&mut self, id: u32) -> Result<()> {
         {
             ensure!(
@@ -462,14 +486,16 @@ struct AccountConfig {
     /// Unique id.
     pub id: u32,
     /// Root directory for all data for this account.
+    ///
+    /// The path is relative to the account manager directory.
     pub dir: std::path::PathBuf,
     pub uuid: Uuid,
 }
 
 impl AccountConfig {
     /// Get the canoncial dbfile name for this configuration.
-    pub fn dbfile(&self) -> std::path::PathBuf {
-        self.dir.join(DB_NAME)
+    pub fn dbfile(&self, accounts_dir: &Path) -> std::path::PathBuf {
+        accounts_dir.join(&self.dir).join(DB_NAME)
     }
 }
 

src/authres.rs

@@ -5,18 +5,14 @@ use std::borrow::Cow;
 use std::collections::BTreeSet;
 use std::fmt;
 
-use anyhow::Context as _;
 use anyhow::Result;
 use mailparse::MailHeaderMap;
 use mailparse::ParsedMail;
-use num_traits::FromPrimitive;
-use num_traits::ToPrimitive;
 use once_cell::sync::Lazy;
 
 use crate::config::Config;
 use crate::context::Context;
 use crate::headerdef::HeaderDef;
-use crate::tools;
 use crate::tools::time;
 use crate::tools::EmailAddress;
 
@@ -35,8 +31,8 @@ pub(crate) async fn handle_authres(
     from: &str,
     message_time: i64,
 ) -> Result<DkimResults> {
-    let from = match EmailAddress::new(from) {
-        Ok(email) => email,
+    let from_domain = match EmailAddress::new(from) {
+        Ok(email) => email.domain,
         Err(e) => {
             // This email is invalid, but don't return an error, we still want to
             // add a stub to the database so that it's not downloaded again
@@ -44,9 +40,9 @@ pub(crate) async fn handle_authres(
         }
     };
 
-    let authres = parse_authres_headers(&mail.get_headers(), &from.domain);
-    update_authservid_candidates(context, &authres, &from).await?;
-    compute_dkim_results(context, authres, &from.domain, message_time).await
+    let authres = parse_authres_headers(&mail.get_headers(), &from_domain);
+    update_authservid_candidates(context, &authres).await?;
+    compute_dkim_results(context, authres, &from_domain, message_time).await
 }
 
 #[derive(Debug)]
@@ -167,16 +163,6 @@ fn parse_one_authres_header(header_value: &str, from_domain: &str) -> DkimResult
     DkimResult::Nothing
 }
 
-#[derive(Debug, FromPrimitive, ToPrimitive, PartialEq, PartialOrd)]
-#[repr(i32)]
-pub(crate) enum AuthservIdCandidatesOrigin {
-    None = 0,
-    /// We just used the authserv-ids from an incoming email.
-    /// They might TODO docs
-    UnknownSender = 10,
-    SameDomainSender = 20,
-}
-
 /// ## About authserv-ids
 ///
 /// After having checked DKIM, our email server adds an Authentication-Results header.
@@ -207,41 +193,19 @@ pub(crate) enum AuthservIdCandidatesOrigin {
 async fn update_authservid_candidates(
     context: &Context,
     authres: &ParsedAuthresHeaders,
-    from: &EmailAddress,
 ) -> Result<()> {
     let mut new_ids: BTreeSet<&str> = authres
         .iter()
         .map(|(authserv_id, _dkim_passed)| authserv_id.as_str())
         .collect();
     if new_ids.is_empty() {
-        // The incoming message doesn't contain any Authentication-Results, maybe it's a
+        // The incoming message doesn't contain any authentication results, maybe it's a
         // self-sent or a mailer-daemon message
         return Ok(());
     }
 
-    let old_origin = get_authservid_condidates_origin(context).await?;
-
-    let self_addr = tools::EmailAddress::new(&context.get_primary_self_addr().await?)?;
-
-    // If we get an email from another account on the same domain as ours, then we can
-    // assume that the Authentication-Results in there were added by our server.
-    // So, the authserv-id in there most probably is the one of our server, i.e. it's
-    // the one we want to remember.
-    // We have the `&& from.local != self_addr.local` because many servers, e.g. GMail,
-    // don't add Authentication-Results for self-sent emails.
-    let new_origin = if from.domain == self_addr.domain && from.local != self_addr.local {
-        AuthservIdCandidatesOrigin::SameDomainSender
-    } else {
-        AuthservIdCandidatesOrigin::UnknownSender
-    };
-
-    if old_origin > new_origin {
-        return Ok(());
-    }
-
     let old_config = context.get_config(Config::AuthservIdCandidates).await?;
     let old_ids = parse_authservid_candidates_config(&old_config);
-
     let intersection: BTreeSet<&str> = old_ids.intersection(&new_ids).copied().collect();
     if !intersection.is_empty() {
         new_ids = intersection;
@@ -259,17 +223,6 @@ async fn update_authservid_candidates(
         // reset our expectation which DKIMs work.
         clear_dkim_works(context).await?
     }
-
-    if old_origin != new_origin {
-        let new_origin_i32 = new_origin.to_i32().context("not a valid i32??")?;
-        context
-            .set_config(
-                Config::AuthservIdCandidatesOrigin,
-                Some(&new_origin_i32.to_string()),
-            )
-            .await?;
-    }
-
     Ok(())
 }
 
@@ -293,33 +246,14 @@ async fn compute_dkim_results(
     let ids_config = context.get_config(Config::AuthservIdCandidates).await?;
     let ids = parse_authservid_candidates_config(&ids_config);
 
-    let origin = get_authservid_condidates_origin(context).await?;
-
-    // Remove all foreign Authentication-Results
+    // Remove all foreign authentication results
     authres.retain(|(authserv_id, _dkim_passed)| ids.contains(authserv_id.as_str()));
 
     if authres.is_empty() {
-        match origin {
-            AuthservIdCandidatesOrigin::None | AuthservIdCandidatesOrigin::UnknownSender => {
-                // If the Authentication-results are empty, then our provider doesn't add them
-                // and an attacker could just add their own Authentication-Results, making us
-                // think that DKIM passed. So, in this case, we can as well assume that DKIM passed.
-                dkim_passed = true;
-            }
-
-            AuthservIdCandidatesOrigin::SameDomainSender => {
-                // We know that usually our provider adds Authentication-Results,
-                // and we are quite certain that we know the correct authserv-id.
-                //
-                // Some providers like buzon.uy, disroot.org, yandex.ru, mailo.com, and
-                // riseup.net only add Authentication-Results if DKIM/SPF passed.
-                // So, probably DKIM failed, and therefore no Authentication-Results
-                // were added. And since we probably know the correct authserv-id,
-                // we can acutally assume that an attacher couldn't just add their own
-                // Authentication-Results.
-                dkim_passed = false;
-            }
-        };
+        // If the authentication results are empty, then our provider doesn't add them
+        // and an attacker could just add their own Authentication-Results, making us
+        // think that DKIM passed. So, in this case, we can as well assume that DKIM passed.
+        dkim_passed = true;
     } else {
         for (_authserv_id, current_dkim_passed) in authres {
             match current_dkim_passed {
@@ -412,15 +346,6 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
         .unwrap_or_default()
 }
 
-async fn get_authservid_condidates_origin(context: &Context) -> Result<AuthservIdCandidatesOrigin> {
-    AuthservIdCandidatesOrigin::from_i32(
-        context
-            .get_config_int(Config::AuthservIdCandidatesOrigin)
-            .await?,
-    )
-    .context("Invalid AuthservIdCandidatesOrigin config")
-}
-
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
@@ -561,60 +486,25 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
     async fn test_update_authservid_candidates() -> Result<()> {
         let t = TestContext::new_alice().await;
 
-        update_authservid_candidates_test(&t, &["mx3.messagingengine.com"], "a@b.c").await;
+        update_authservid_candidates_test(&t, &["mx3.messagingengine.com"]).await;
         let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
         assert_eq!(candidates, "mx3.messagingengine.com");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::UnknownSender);
 
         // "mx4.messagingengine.com" seems to be the new authserv-id, DC should accept it
-        update_authservid_candidates_test(&t, &["mx4.messagingengine.com"], "a@b.c").await;
+        update_authservid_candidates_test(&t, &["mx4.messagingengine.com"]).await;
         let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
         assert_eq!(candidates, "mx4.messagingengine.com");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::UnknownSender);
 
         // A message without any Authentication-Results headers shouldn't remove all
         // candidates since it could be a mailer-daemon message or so
-        update_authservid_candidates_test(&t, &[], "a@b.c").await;
+        update_authservid_candidates_test(&t, &[]).await;
         let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
         assert_eq!(candidates, "mx4.messagingengine.com");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::UnknownSender);
 
-        update_authservid_candidates_test(
-            &t,
-            &["mx4.messagingengine.com", "someotherdomain.com"],
-            "a@b.c",
-        )
-        .await;
+        update_authservid_candidates_test(&t, &["mx4.messagingengine.com", "someotherdomain.com"])
+            .await;
         let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
         assert_eq!(candidates, "mx4.messagingengine.com");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::UnknownSender);
-
-        // We get an email from another example.org user
-        update_authservid_candidates_test(&t, &["mx.example.org"], "friend@example.org").await;
-        let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
-        assert_eq!(candidates, "mx.example.org");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::SameDomainSender);
-
-        // "mx4.messagingengine.com" comes from another domain while "mx.example.org" came
-        // from an example.org address -> the latter is more trustworthy
-        update_authservid_candidates_test(&t, &["mx4.messagingengine.com"], "a@b.c").await;
-        let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
-        assert_eq!(candidates, "mx.example.org");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::SameDomainSender);
-
-        // Another email from an example.org user, seems like example.org changed their
-        // authserv-id
-        update_authservid_candidates_test(&t, &["mx2.example.org"], "friend@example.org").await;
-        let candidates = t.get_config(Config::AuthservIdCandidates).await?.unwrap();
-        assert_eq!(candidates, "mx2.example.org");
-        let origin = get_authservid_condidates_origin(&t).await?;
-        assert_eq!(origin, AuthservIdCandidatesOrigin::SameDomainSender);
 
         Ok(())
     }
@@ -624,18 +514,12 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
     /// update_authservid_candidates() only looks at the keys of its
     /// `authentication_results` parameter. So, this function takes `incoming_ids`
     /// and adds some AuthenticationResults to get the HashMap we need.
-    async fn update_authservid_candidates_test(
-        context: &Context,
-        incoming_ids: &[&str],
-        from_addr: &str,
-    ) {
+    async fn update_authservid_candidates_test(context: &Context, incoming_ids: &[&str]) {
         let v = incoming_ids
             .iter()
             .map(|id| (id.to_string(), DkimResult::Passed))
             .collect();
-        update_authservid_candidates(context, &v, &EmailAddress::new(from_addr).unwrap())
-            .await
-            .unwrap()
+        update_authservid_candidates(context, &v).await.unwrap()
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -667,7 +551,6 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 "outlook.com",
                 "gmx.de",
                 "testrun.org",
-                "buzon.uy",
             ]
             .contains(&self_domain.as_str());
 
@@ -731,17 +614,11 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
 
                 if res.dkim_passed != expected_result {
                     if authres_parsing_works {
-                        println!();
                         println!(
-                            "!!!!!! FAILURE Receiving {:?}, wrong result: !!!!!!",
+                            "!!!!!! FAILURE Receiving {:?}, order {:#?} wrong result: !!!!!!",
                             entry.path(),
-                        );
-                        println!(
-                            "order: {:#?}",
                             dir.iter().map(|e| e.file_name()).collect::<Vec<_>>()
                         );
-                        println!("Info: {:#?}", t.get_info().await?);
-                        println!();
                         test_failed = true;
                     }
                     println!("From {}: {}", from_domain, res.dkim_passed);
@@ -755,7 +632,7 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_handle_authres() {
-        let t = TestContext::new_alice().await;
+        let t = TestContext::new().await;
 
         // Even if the format is wrong and parsing fails, handle_authres() shouldn't
         // return an Err because this would prevent the message from being added

src/blob.rs

@@ -746,7 +746,7 @@ mod tests {
         assert!(file_size(&avatar_blob).await <= 3000);
         assert!(file_size(&avatar_blob).await > 2000);
         tokio::task::block_in_place(move || {
-            let img = image::open(&avatar_blob).unwrap();
+            let img = image::open(avatar_blob).unwrap();
             assert!(img.width() > 130);
             assert_eq!(img.width(), img.height());
         });

src/chat.rs

@@ -1,5 +1,7 @@
 //! # Chat module.
 
+#![allow(missing_docs)]
+
 use std::collections::HashMap;
 use std::convert::{TryFrom, TryInto};
 use std::fmt;
@@ -766,7 +768,7 @@ impl ChatId {
                 paramsv![self],
             )
             .await?;
-        Ok(count as usize)
+        Ok(count)
     }
 
     pub async fn get_fresh_msg_cnt(self, context: &Context) -> Result<usize> {
@@ -780,18 +782,36 @@ impl ChatId {
         // the times are average, no matter if there are fresh messages or not -
         // and have to be multiplied by the number of items shown at once on the chatlist,
         // so savings up to 2 seconds are possible on older devices - newer ones will feel "snappier" :)
-        let count = context
-            .sql
-            .count(
-                "SELECT COUNT(*)
+        let count = if self.is_archived_link() {
+            context
+                .sql
+                .count(
+                    "SELECT COUNT(DISTINCT(m.chat_id))
+                    FROM msgs m
+                    LEFT JOIN chats c ON m.chat_id=c.id
+                    WHERE m.state=10
+                    and m.hidden=0
+                    AND m.chat_id>9
+                    AND c.blocked=0
+                    AND c.archived=1
+                    ",
+                    paramsv![],
+                )
+                .await?
+        } else {
+            context
+                .sql
+                .count(
+                    "SELECT COUNT(*)
                 FROM msgs
                 WHERE state=?
                 AND hidden=0
                 AND chat_id=?;",
-                paramsv![MessageState::InFresh, self],
-            )
-            .await?;
-        Ok(count as usize)
+                    paramsv![MessageState::InFresh, self],
+                )
+                .await?
+        };
+        Ok(count)
     }
 
     pub(crate) async fn get_param(self, context: &Context) -> Result<Params> {
@@ -804,6 +824,19 @@ impl ChatId {
             .unwrap_or_default())
     }
 
+    /// Returns true if the chat is not promoted.
+    pub(crate) async fn is_unpromoted(self, context: &Context) -> Result<bool> {
+        let param = self.get_param(context).await?;
+        let unpromoted = param.get_bool(Param::Unpromoted).unwrap_or_default();
+        Ok(unpromoted)
+    }
+
+    /// Returns true if the chat is promoted.
+    pub(crate) async fn is_promoted(self, context: &Context) -> Result<bool> {
+        let promoted = !self.is_unpromoted(context).await?;
+        Ok(promoted)
+    }
+
     // Returns true if chat is a saved messages chat.
     pub async fn is_self_talk(self, context: &Context) -> Result<bool> {
         Ok(self.get_param(context).await?.exists(Param::Selftalk))
@@ -1096,7 +1129,7 @@ impl Chat {
                         }
                     }
                     Err(err) => {
-                        error!(context, "faild to load contacts for {}: {:?}", chat.id, err);
+                        error!(context, "faild to load contacts for {}: {:#}", chat.id, err);
                     }
                 }
                 chat.name = chat_name;
@@ -1201,6 +1234,10 @@ impl Chat {
             if !image_rel.is_empty() {
                 return Ok(Some(get_abs_path(context, image_rel)));
             }
+        } else if self.id.is_archived_link() {
+            if let Ok(image_rel) = get_archive_icon(context).await {
+                return Ok(Some(get_abs_path(context, image_rel)));
+            }
         } else if self.typ == Chattype::Single {
             let contacts = get_chat_contacts(context, self.id).await?;
             if let Some(contact_id) = contacts.first() {
@@ -1275,7 +1312,7 @@ impl Chat {
     }
 
     pub fn is_unpromoted(&self) -> bool {
-        self.param.get_int(Param::Unpromoted).unwrap_or_default() == 1
+        self.param.get_bool(Param::Unpromoted).unwrap_or_default()
     }
 
     pub fn is_promoted(&self) -> bool {
@@ -1472,7 +1509,7 @@ impl Chat {
                         new_rfc724_mid,
                         self.id,
                         ContactId::SELF,
-                        to_id as i32,
+                        to_id,
                         timestamp,
                         msg.viewtype,
                         msg.state,
@@ -1520,7 +1557,7 @@ impl Chat {
                         new_rfc724_mid,
                         self.id,
                         ContactId::SELF,
-                        to_id as i32,
+                        to_id,
                         timestamp,
                         msg.viewtype,
                         msg.state,
@@ -1693,6 +1730,21 @@ pub(crate) async fn get_broadcast_icon(context: &Context) -> Result<String> {
     Ok(icon)
 }
 
+pub(crate) async fn get_archive_icon(context: &Context) -> Result<String> {
+    if let Some(icon) = context.sql.get_raw_config("icon-archive").await? {
+        return Ok(icon);
+    }
+
+    let icon = include_bytes!("../assets/icon-archive.png");
+    let blob = BlobObject::create(context, "icon-archive.png", icon).await?;
+    let icon = blob.as_name().to_string();
+    context
+        .sql
+        .set_raw_config("icon-archive", Some(&icon))
+        .await?;
+    Ok(icon)
+}
+
 async fn update_special_chat_name(
     context: &Context,
     contact_id: ContactId,
@@ -2094,7 +2146,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<
     let attach_selfavatar = match shall_attach_selfavatar(context, msg.chat_id).await {
         Ok(attach_selfavatar) => attach_selfavatar,
         Err(err) => {
-            warn!(context, "job: cannot get selfavatar-state: {}", err);
+            warn!(context, "job: cannot get selfavatar-state: {:#}", err);
             false
         }
     };
@@ -2156,27 +2208,27 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<
 
     if 0 != rendered_msg.last_added_location_id {
         if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, time()).await {
-            error!(context, "Failed to set kml sent_timestamp: {:?}", err);
+            error!(context, "Failed to set kml sent_timestamp: {:#}", err);
         }
         if !msg.hidden {
             if let Err(err) =
                 location::set_msg_location_id(context, msg.id, rendered_msg.last_added_location_id)
                     .await
             {
-                error!(context, "Failed to set msg_location_id: {:?}", err);
+                error!(context, "Failed to set msg_location_id: {:#}", err);
             }
         }
     }
 
     if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
         if let Err(err) = context.delete_sync_ids(sync_ids).await {
-            error!(context, "Failed to delete sync ids: {:?}", err);
+            error!(context, "Failed to delete sync ids: {:#}", err);
         }
     }
 
     if attach_selfavatar {
         if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
-            error!(context, "Failed to set selfavatar timestamp: {:?}", err);
+            error!(context, "Failed to set selfavatar timestamp: {:#}", err);
         }
     }
 
@@ -2379,31 +2431,63 @@ pub(crate) async fn marknoticed_chat_if_older_than(
 }
 
 /// Marks all messages in the chat as noticed.
+/// If the given chat-id is the archive-link, marks all messages in all archived chats as noticed.
 pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()> {
     // "WHERE" below uses the index `(state, hidden, chat_id)`, see get_fresh_msg_cnt() for reasoning
     // the additional SELECT statement may speed up things as no write-blocking is needed.
-    let exists = context
-        .sql
-        .exists(
-            "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;",
-            paramsv![MessageState::InFresh, chat_id],
-        )
-        .await?;
-    if !exists {
-        return Ok(());
-    }
+    if chat_id.is_archived_link() {
+        let chat_ids_in_archive = context
+            .sql
+            .query_map(
+                "SELECT DISTINCT(m.chat_id) FROM msgs m
+                    LEFT JOIN chats c ON m.chat_id=c.id
+                    WHERE m.state=10 AND m.hidden=0 AND m.chat_id>9 AND c.blocked=0 AND c.archived=1",
+                paramsv![],
+                |row| row.get::<_, ChatId>(0),
+                |ids| ids.collect::<Result<Vec<_>, _>>().map_err(Into::into)
+            )
+            .await?;
+        if chat_ids_in_archive.is_empty() {
+            return Ok(());
+        }
 
-    context
-        .sql
-        .execute(
-            "UPDATE msgs
-            SET state=?
-          WHERE state=?
-            AND hidden=0
-            AND chat_id=?;",
-            paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id],
-        )
-        .await?;
+        context
+            .sql
+            .execute(
+                &format!(
+                    "UPDATE msgs SET state=13 WHERE state=10 AND hidden=0 AND chat_id IN ({});",
+                    sql::repeat_vars(chat_ids_in_archive.len())
+                ),
+                rusqlite::params_from_iter(&chat_ids_in_archive),
+            )
+            .await?;
+        for chat_id_in_archive in chat_ids_in_archive {
+            context.emit_event(EventType::MsgsNoticed(chat_id_in_archive));
+        }
+    } else {
+        let exists = context
+            .sql
+            .exists(
+                "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;",
+                paramsv![MessageState::InFresh, chat_id],
+            )
+            .await?;
+        if !exists {
+            return Ok(());
+        }
+
+        context
+            .sql
+            .execute(
+                "UPDATE msgs
+                SET state=?
+              WHERE state=?
+                AND hidden=0
+                AND chat_id=?;",
+                paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id],
+            )
+            .await?;
+    }
 
     context.emit_event(EventType::MsgsNoticed(chat_id));
 
@@ -3037,7 +3121,11 @@ pub async fn set_chat_name(context: &Context, chat_id: ChatId, new_name: &str) -
                     paramsv![new_name.to_string(), chat_id],
                 )
                 .await?;
-            if chat.is_promoted() && !chat.is_mailing_list() && chat.typ != Chattype::Broadcast {
+            if chat.is_promoted()
+                && !chat.is_mailing_list()
+                && chat.typ != Chattype::Broadcast
+                && improve_single_line_input(&chat.name) != new_name
+            {
                 msg.viewtype = Viewtype::Text;
                 msg.text = Some(
                     stock_str::msg_grp_name(context, &chat.name, &new_name, ContactId::SELF).await,
@@ -3259,7 +3347,7 @@ pub(crate) async fn get_chat_cnt(context: &Context) -> Result<usize> {
                 paramsv![],
             )
             .await?;
-        Ok(count as usize)
+        Ok(count)
     } else {
         Ok(0)
     }
@@ -3543,6 +3631,7 @@ mod tests {
     use crate::constants::{DC_GCL_ARCHIVED_ONLY, DC_GCL_NO_SPECIALS};
     use crate::contact::Contact;
     use crate::receive_imf::receive_imf;
+
     use crate::test_utils::TestContext;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -4403,6 +4492,91 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_archive_fresh_msgs() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        async fn msg_from(t: &TestContext, name: &str, num: u32) -> Result<()> {
+            receive_imf(
+                t,
+                format!(
+                    "From: {}@example.net\n\
+                     To: alice@example.org\n\
+                     Message-ID: <{}@example.org>\n\
+                     Chat-Version: 1.0\n\
+                     Date: Sun, 22 Mar 2022 19:37:57 +0000\n\
+                     \n\
+                     hello\n",
+                    name, num
+                )
+                .as_bytes(),
+                false,
+            )
+            .await?;
+            Ok(())
+        }
+
+        // receive some messages in archived+muted chats
+        msg_from(&t, "bob", 1).await?;
+        let bob_chat_id = t.get_last_msg().await.get_chat_id();
+        bob_chat_id.accept(&t).await?;
+        set_muted(&t, bob_chat_id, MuteDuration::Forever).await?;
+        bob_chat_id
+            .set_visibility(&t, ChatVisibility::Archived)
+            .await?;
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 0);
+
+        msg_from(&t, "bob", 2).await?;
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1);
+
+        msg_from(&t, "bob", 3).await?;
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1);
+
+        msg_from(&t, "claire", 4).await?;
+        let claire_chat_id = t.get_last_msg().await.get_chat_id();
+        claire_chat_id.accept(&t).await?;
+        set_muted(&t, claire_chat_id, MuteDuration::Forever).await?;
+        claire_chat_id
+            .set_visibility(&t, ChatVisibility::Archived)
+            .await?;
+        msg_from(&t, "claire", 5).await?;
+        msg_from(&t, "claire", 6).await?;
+        msg_from(&t, "claire", 7).await?;
+        assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2);
+        assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 3);
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2);
+
+        // mark one of the archived+muted chats as noticed: check that the archive-link counter is changed as well
+        marknoticed_chat(&t, claire_chat_id).await?;
+        assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2);
+        assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0);
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1);
+
+        // receive some more messages
+        msg_from(&t, "claire", 8).await?;
+        assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2);
+        assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 1);
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2);
+        assert_eq!(t.get_fresh_msgs().await?.len(), 0);
+
+        msg_from(&t, "dave", 9).await?;
+        let dave_chat_id = t.get_last_msg().await.get_chat_id();
+        dave_chat_id.accept(&t).await?;
+        assert_eq!(dave_chat_id.get_fresh_msg_cnt(&t).await?, 1);
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2);
+        assert_eq!(t.get_fresh_msgs().await?.len(), 1);
+
+        // mark the archived-link as noticed: check that the real chats are noticed as well
+        marknoticed_chat(&t, DC_CHAT_ID_ARCHIVED_LINK).await?;
+        assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 0);
+        assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0);
+        assert_eq!(dave_chat_id.get_fresh_msg_cnt(&t).await?, 1);
+        assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 0);
+        assert_eq!(t.get_fresh_msgs().await?.len(), 1);
+
+        Ok(())
+    }
+
     async fn get_chats_from_chat_list(ctx: &Context, listflags: usize) -> Vec<ChatId> {
         let chatlist = Chatlist::try_load(ctx, listflags, None, None)
             .await
@@ -5028,7 +5202,7 @@ mod tests {
         assert_eq!(msg.get_filename(), Some(filename.to_string()));
         assert_eq!(msg.get_width(), w);
         assert_eq!(msg.get_height(), h);
-        assert!(msg.get_filebytes(&bob).await > 250);
+        assert!(msg.get_filebytes(&bob).await?.unwrap() > 250);
 
         Ok(())
     }

src/chatlist.rs

@@ -1,5 +1,7 @@
 //! # Chat list module.
 
+#![allow(missing_docs)]
+
 use anyhow::{ensure, Context as _, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
@@ -90,8 +92,6 @@ impl Chatlist {
         let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
         let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
 
-        let mut add_archived_link_item = false;
-
         let process_row = |row: &rusqlite::Row| {
             let chat_id: ChatId = row.get(0)?;
             let msg_id: Option<MsgId> = row.get(1)?;
@@ -121,7 +121,7 @@ impl Chatlist {
         //
         // The query shows messages from blocked contacts in
         // groups. Otherwise it would be hard to follow conversations.
-        let mut ids = if let Some(query_contact_id) = query_contact_id {
+        let ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
                 "SELECT c.id, m.id
@@ -214,7 +214,7 @@ impl Chatlist {
             } else {
                 ChatId::new(0)
             };
-            let ids = context.sql.query_map(
+            let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -234,19 +234,15 @@ impl Chatlist {
                 process_row,
                 process_rows,
             ).await?;
-            if !flag_no_specials {
-                add_archived_link_item = true;
+            if !flag_no_specials && get_archived_cnt(context).await? > 0 {
+                if ids.is_empty() && flag_add_alldone_hint {
+                    ids.push((DC_CHAT_ID_ALLDONE_HINT, None));
+                }
+                ids.insert(0, (DC_CHAT_ID_ARCHIVED_LINK, None));
             }
             ids
         };
 
-        if add_archived_link_item && get_archived_cnt(context).await? > 0 {
-            if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, None));
-            }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, None));
-        }
-
         Ok(Chatlist { ids })
     }
 

src/config.rs

@@ -1,5 +1,7 @@
 //! # Key-value configuration management.
 
+#![allow(missing_docs)]
+
 use anyhow::{ensure, Context as _, Result};
 use strum::{EnumProperty as EnumPropertyTrait, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
@@ -190,10 +192,6 @@ pub enum Config {
     ///
     /// See `crate::authres::update_authservid_candidates`.
     AuthservIdCandidates,
-
-    /// How sure we are that AuthservIdCandidates are the correct authserv-ids of our server.
-    /// This contains a value of [`crate::authres::AuthservIdCandidatesOrigin`].
-    AuthservIdCandidatesOrigin,
 }
 
 impl Context {
@@ -206,7 +204,7 @@ impl Context {
         let value = match key {
             Config::Selfavatar => {
                 let rel_path = self.sql.get_raw_config(key.as_ref()).await?;
-                rel_path.map(|p| get_abs_path(self, &p).to_string_lossy().into_owned())
+                rel_path.map(|p| get_abs_path(self, p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),

src/configure.rs

@@ -579,10 +579,10 @@ async fn try_imap_one_param(
 
     let mut imap = match Imap::new(param, socks5_config.clone(), addr, provider_strict_tls, r) {
         Err(err) => {
-            info!(context, "failure: {}", err);
+            info!(context, "failure: {:#}", err);
             return Err(ConfigurationError {
                 config: inf,
-                msg: err.to_string(),
+                msg: format!("{:#}", err),
             });
         }
         Ok(imap) => imap,
@@ -590,10 +590,10 @@ async fn try_imap_one_param(
 
     match imap.connect(context).await {
         Err(err) => {
-            info!(context, "failure: {}", err);
+            info!(context, "failure: {:#}", err);
             Err(ConfigurationError {
                 config: inf,
-                msg: err.to_string(),
+                msg: format!("{:#}", err),
             })
         }
         Ok(()) => {
@@ -634,7 +634,7 @@ async fn try_smtp_one_param(
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
-            msg: err.to_string(),
+            msg: format!("{:#}", err),
         })
     } else {
         info!(context, "success: {}", inf);

src/configure/auto_mozilla.rs

@@ -62,7 +62,7 @@ fn parse_server<B: BufRead>(
     reader: &mut quick_xml::Reader<B>,
     server_event: &BytesStart,
 ) -> Result<Option<Server>, quick_xml::Error> {
-    let end_tag = String::from_utf8_lossy(server_event.name())
+    let end_tag = String::from_utf8_lossy(server_event.name().as_ref())
         .trim()
         .to_lowercase();
 
@@ -70,12 +70,17 @@ fn parse_server<B: BufRead>(
         .attributes()
         .find(|attr| {
             attr.as_ref()
-                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
+                .map(|a| {
+                    String::from_utf8_lossy(a.key.as_ref())
+                        .trim()
+                        .to_lowercase()
+                        == "type"
+                })
                 .unwrap_or_default()
         })
         .map(|typ| {
             typ.unwrap()
-                .unescape_and_decode_value(reader)
+                .decode_and_unescape_value(reader)
                 .unwrap_or_default()
                 .to_lowercase()
         })
@@ -89,25 +94,23 @@ fn parse_server<B: BufRead>(
     let mut tag_config = MozConfigTag::Undefined;
     let mut buf = Vec::new();
     loop {
-        match reader.read_event(&mut buf)? {
+        match reader.read_event_into(&mut buf)? {
             Event::Start(ref event) => {
-                tag_config = String::from_utf8_lossy(event.name())
+                tag_config = String::from_utf8_lossy(event.name().as_ref())
                     .parse()
                     .unwrap_or_default();
             }
             Event::End(ref event) => {
-                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+                let tag = String::from_utf8_lossy(event.name().as_ref())
+                    .trim()
+                    .to_lowercase();
 
                 if tag == end_tag {
                     break;
                 }
             }
             Event::Text(ref event) => {
-                let val = event
-                    .unescape_and_decode(reader)
-                    .unwrap_or_default()
-                    .trim()
-                    .to_owned();
+                let val = event.unescape().unwrap_or_default().trim().to_owned();
 
                 match tag_config {
                     MozConfigTag::Hostname => hostname = Some(val),
@@ -150,9 +153,11 @@ fn parse_xml_reader<B: BufRead>(
 
     let mut buf = Vec::new();
     loop {
-        match reader.read_event(&mut buf)? {
+        match reader.read_event_into(&mut buf)? {
             Event::Start(ref event) => {
-                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+                let tag = String::from_utf8_lossy(event.name().as_ref())
+                    .trim()
+                    .to_lowercase();
 
                 if tag == "incomingserver" {
                     if let Some(incoming_server) = parse_server(reader, event)? {

src/configure/auto_outlook.rs

@@ -59,12 +59,18 @@ fn parse_protocol<B: BufRead>(
 
     let mut current_tag: Option<String> = None;
     loop {
-        match reader.read_event(&mut buf)? {
+        match reader.read_event_into(&mut buf)? {
             Event::Start(ref event) => {
-                current_tag = Some(String::from_utf8_lossy(event.name()).trim().to_lowercase());
+                current_tag = Some(
+                    String::from_utf8_lossy(event.name().as_ref())
+                        .trim()
+                        .to_lowercase(),
+                );
             }
             Event::End(ref event) => {
-                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+                let tag = String::from_utf8_lossy(event.name().as_ref())
+                    .trim()
+                    .to_lowercase();
                 if tag == "protocol" {
                     break;
                 }
@@ -73,7 +79,7 @@ fn parse_protocol<B: BufRead>(
                 }
             }
             Event::Text(ref e) => {
-                let val = e.unescape_and_decode(reader).unwrap_or_default();
+                let val = e.unescape().unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
@@ -115,9 +121,9 @@ fn parse_redirecturl<B: BufRead>(
     reader: &mut quick_xml::Reader<B>,
 ) -> Result<String, quick_xml::Error> {
     let mut buf = Vec::new();
-    match reader.read_event(&mut buf)? {
+    match reader.read_event_into(&mut buf)? {
         Event::Text(ref e) => {
-            let val = e.unescape_and_decode(reader).unwrap_or_default();
+            let val = e.unescape().unwrap_or_default();
             Ok(val.trim().to_string())
         }
         _ => Ok("".to_string()),
@@ -131,9 +137,11 @@ fn parse_xml_reader<B: BufRead>(
 
     let mut buf = Vec::new();
     loop {
-        match reader.read_event(&mut buf)? {
+        match reader.read_event_into(&mut buf)? {
             Event::Start(ref e) => {
-                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
+                let tag = String::from_utf8_lossy(e.name().as_ref())
+                    .trim()
+                    .to_lowercase();
 
                 if tag == "protocol" {
                     if let Some(protocol) = parse_protocol(reader)? {

src/constants.rs

@@ -1,4 +1,7 @@
 //! # Constants.
+
+#![allow(missing_docs)]
+
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};

src/contact.rs

@@ -46,12 +46,18 @@ const SEEN_RECENTLY_SECONDS: i64 = 600;
 pub struct ContactId(u32);
 
 impl ContactId {
+    /// Undefined contact. Used as a placeholder for trashed messages.
     pub const UNDEFINED: ContactId = ContactId::new(0);
+
     /// The owner of the account.
     ///
     /// The email-address is set by `set_config` using "addr".
     pub const SELF: ContactId = ContactId::new(1);
+
+    /// ID of the contact for info messages.
     pub const INFO: ContactId = ContactId::new(2);
+
+    /// ID of the contact for device messages.
     pub const DEVICE: ContactId = ContactId::new(5);
     const LAST_SPECIAL: ContactId = ContactId::new(9);
 
@@ -175,6 +181,8 @@ pub struct Contact {
 )]
 #[repr(u32)]
 pub enum Origin {
+    /// Unknown origin. Can be used as a minimum origin to specify that the caller does not care
+    /// about origin of the contact.
     Unknown = 0,
 
     /// The contact is a mailing list address, needed to unblock mailing lists
@@ -255,12 +263,13 @@ pub(crate) enum Modifier {
     Created,
 }
 
+/// Verification status of the contact.
 #[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)]
 #[repr(u8)]
 pub enum VerifiedStatus {
     /// Contact is not verified.
     Unverified = 0,
-    // TODO: is this a thing?
+    /// SELF has verified the fingerprint of a contact. Currently unused.
     Verified = 1,
     /// SELF and contact have verified their fingerprints in both directions; in the UI typically checkmarks are shown.
     BidirectVerified = 2,
@@ -273,6 +282,7 @@ impl Default for VerifiedStatus {
 }
 
 impl Contact {
+    /// Loads a contact snapshot from the database.
     pub async fn load_from_db(context: &Context, contact_id: ContactId) -> Result<Self> {
         let mut contact = context
             .sql
@@ -845,6 +855,7 @@ impl Contact {
         Ok(())
     }
 
+    /// Returns number of blocked contacts.
     pub async fn get_blocked_cnt(context: &Context) -> Result<usize> {
         let count = context
             .sql
@@ -853,7 +864,7 @@ impl Contact {
                 paramsv![ContactId::LAST_SPECIAL],
             )
             .await?;
-        Ok(count as usize)
+        Ok(count)
     }
 
     /// Get blocked contacts.
@@ -1136,6 +1147,32 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
+    /// Returns the address that verified the given contact.
+    pub async fn get_verifier_addr(
+        context: &Context,
+        contact_id: &ContactId,
+    ) -> Result<Option<String>> {
+        let contact = Contact::load_from_db(context, *contact_id).await?;
+
+        Ok(Peerstate::from_addr(context, contact.get_addr())
+            .await?
+            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned())))
+    }
+
+    /// Returns the ContactId that verified the given contact.
+    pub async fn get_verifier_id(
+        context: &Context,
+        contact_id: &ContactId,
+    ) -> Result<Option<ContactId>> {
+        let verifier_addr = Contact::get_verifier_addr(context, contact_id).await?;
+        if let Some(addr) = verifier_addr {
+            Ok(Contact::lookup_id_by_addr(context, &addr, Origin::AddressBook).await?)
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// Returns the number of real (i.e. non-special) contacts in the database.
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
             return Ok(0);
@@ -1151,6 +1188,7 @@ impl Contact {
         Ok(count)
     }
 
+    /// Returns true if a contact with this ID exists.
     pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result<bool> {
         if contact_id.is_special() {
             return Ok(false);
@@ -1166,6 +1204,7 @@ impl Contact {
         Ok(exists)
     }
 
+    /// Updates the origin of the contact, but only if new origin is higher than the current one.
     pub async fn scaleup_origin_by_id(
         context: &Context,
         contact_id: ContactId,
@@ -1426,6 +1465,7 @@ fn cat_fingerprint(
     }
 }
 
+/// Compares two email addresses, normalizing them beforehand.
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
     let norm1 = addr_normalize(addr1).to_lowercase();
     let norm2 = addr_normalize(addr2).to_lowercase();
@@ -2298,7 +2338,6 @@ bob@example.net:
 CCCB 5AA9 F6E1 141C 9431
 65F1 DB18 B18C BCF7 0487"
         );
-
         Ok(())
     }
 

src/context.rs

@@ -1,5 +1,7 @@
 //! Context module.
 
+#![allow(missing_docs)]
+
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
@@ -381,7 +383,7 @@ impl Context {
         let mut lock = self.inner.scheduler.write().await;
         if lock.is_none() {
             match Scheduler::start(self.clone()).await {
-                Err(err) => error!(self, "Failed to start IO: {}", err),
+                Err(err) => error!(self, "Failed to start IO: {:#}", err),
                 Ok(scheduler) => *lock = Some(scheduler),
             }
         }
@@ -497,7 +499,7 @@ impl Context {
         match &*s {
             RunningState::Running { cancel_sender } => {
                 if let Err(err) = cancel_sender.send(()).await {
-                    warn!(self, "could not cancel ongoing: {:?}", err);
+                    warn!(self, "could not cancel ongoing: {:#}", err);
                 }
                 info!(self, "Signaling the ongoing process to stop ASAP.",);
                 *s = RunningState::ShallStop;
@@ -526,10 +528,10 @@ impl Context {
         let l2 = LoginParam::load_configured_params(self).await?;
         let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
         let displayname = self.get_config(Config::Displayname).await?;
-        let chats = get_chat_cnt(self).await? as usize;
-        let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
-        let request_msgs = message::get_request_msg_cnt(self).await as usize;
-        let contacts = Contact::get_real_cnt(self).await? as usize;
+        let chats = get_chat_cnt(self).await?;
+        let unblocked_msgs = message::get_unblocked_msg_cnt(self).await;
+        let request_msgs = message::get_request_msg_cnt(self).await;
+        let contacts = Contact::get_real_cnt(self).await?;
         let is_configured = self.get_config_int(Config::Configured).await?;
         let socks5_enabled = self.get_config_int(Config::Socks5Enabled).await?;
         let dbversion = self
@@ -705,12 +707,6 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
-        res.insert(
-            "authserv_id_candidates_origin",
-            self.get_config(Config::AuthservIdCandidatesOrigin)
-                .await?
-                .unwrap_or_default(),
-        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));

src/decrypt.rs

@@ -2,10 +2,10 @@
 
 use std::collections::HashSet;
 
-use anyhow::{Context as _, Result};
+use anyhow::Result;
 use mailparse::ParsedMail;
 
-use crate::aheader::{Aheader, EncryptPreference};
+use crate::aheader::Aheader;
 use crate::authres::handle_authres;
 use crate::authres::{self, DkimResults};
 use crate::contact::addr_cmp;
@@ -17,43 +17,32 @@ use crate::log::LogExt;
 use crate::peerstate::Peerstate;
 use crate::pgp;
 
-/// Tries to decrypt a message, but only if it is structured as an
-/// Autocrypt message.
+/// Tries to decrypt a message, but only if it is structured as an Autocrypt message.
 ///
-/// Returns decrypted body and a set of valid signature fingerprints
-/// if successful.
+/// If successful and the message is encrypted, returns decrypted body and a set of valid
+/// signature fingerprints.
 ///
-/// If the message is wrongly signed, this will still return the decrypted
-/// message but the HashSet will be empty.
-pub async fn try_decrypt(
+/// If the message is wrongly signed, HashSet will be empty.
+pub fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
-    decryption_info: &DecryptionInfo,
+    private_keyring: &Keyring<SignedSecretKey>,
+    public_keyring_for_validate: &Keyring<SignedPublicKey>,
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    // Possibly perform decryption
-    let public_keyring_for_validate = keyring_from_peerstate(&decryption_info.peerstate);
-
     let encrypted_data_part = match get_autocrypt_mime(mail)
         .or_else(|| get_mixed_up_mime(mail))
         .or_else(|| get_attachment_mime(mail))
     {
-        None => {
-            // not an autocrypt mime message, abort and ignore
-            return Ok(None);
-        }
+        None => return Ok(None),
         Some(res) => res,
     };
     info!(context, "Detected Autocrypt-mime message");
-    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context)
-        .await
-        .context("failed to get own keyring")?;
 
     decrypt_part(
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
     )
-    .await
 }
 
 pub(crate) async fn prepare_decryption(
@@ -61,7 +50,6 @@ pub(crate) async fn prepare_decryption(
     mail: &ParsedMail<'_>,
     from: &str,
     message_time: i64,
-    is_thunderbird: bool,
 ) -> Result<DecryptionInfo> {
     if mail.headers.get_header(HeaderDef::ListPost).is_some() {
         if mail.headers.get_header(HeaderDef::Autocrypt).is_some() {
@@ -84,16 +72,10 @@ pub(crate) async fn prepare_decryption(
         });
     }
 
-    let mut autocrypt_header = Aheader::from_headers(from, &mail.headers)
+    let autocrypt_header = Aheader::from_headers(from, &mail.headers)
         .ok_or_log_msg(context, "Failed to parse Autocrypt header")
         .flatten();
 
-    if is_thunderbird {
-        if let Some(autocrypt_header) = &mut autocrypt_header {
-            autocrypt_header.prefer_encrypt = EncryptPreference::Mutual;
-        }
-    }
-
     let dkim_results = handle_authres(context, mail, from, message_time).await?;
 
     let peerstate = get_autocrypt_peerstate(
@@ -211,30 +193,17 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-async fn decrypt_part(
+fn decrypt_part(
     mail: &ParsedMail<'_>,
-    private_keyring: Keyring<SignedSecretKey>,
-    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    private_keyring: &Keyring<SignedSecretKey>,
+    public_keyring_for_validate: &Keyring<SignedPublicKey>,
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
     let data = mail.get_body_raw()?;
 
     if has_decrypted_pgp_armor(&data) {
         let (plain, ret_valid_signatures) =
-            pgp::pk_decrypt(data, private_keyring, &public_keyring_for_validate).await?;
-
-        // Check for detached signatures.
-        // If decrypted part is a multipart/signed, then there is a detached signature.
-        let decrypted_part = mailparse::parse_mail(&plain)?;
-        if let Some((content, valid_detached_signatures)) =
-            validate_detached_signature(&decrypted_part, &public_keyring_for_validate)?
-        {
-            return Ok(Some((content, valid_detached_signatures)));
-        } else {
-            // If the message was wrongly or not signed, still return the plain text.
-            // The caller has to check if the signatures set is empty then.
-
-            return Ok(Some((plain, ret_valid_signatures)));
-        }
+            pgp::pk_decrypt(data, private_keyring, public_keyring_for_validate)?;
+        return Ok(Some((plain, ret_valid_signatures)));
     }
 
     Ok(None)
@@ -256,32 +225,35 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
 
 /// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
 ///
-/// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
+/// Returns the signed part and the set of key
 /// fingerprints for which there is a valid signature.
-fn validate_detached_signature(
-    mail: &ParsedMail<'_>,
+///
+/// Returns None if the message is not Multipart/Signed or doesn't contain necessary parts.
+pub(crate) fn validate_detached_signature<'a, 'b>(
+    mail: &'a ParsedMail<'b>,
     public_keyring_for_validate: &Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+) -> Option<(&'a ParsedMail<'b>, HashSet<Fingerprint>)> {
     if mail.ctype.mimetype != "multipart/signed" {
-        return Ok(None);
+        return None;
     }
 
     if let [first_part, second_part] = &mail.subparts[..] {
         // First part is the content, second part is the signature.
         let content = first_part.raw_bytes;
-        let signature = second_part.get_body_raw()?;
-        let ret_valid_signatures =
-            pgp::pk_validate(content, &signature, public_keyring_for_validate)?;
-
-        Ok(Some((content.to_vec(), ret_valid_signatures)))
+        let ret_valid_signatures = match second_part.get_body_raw() {
+            Ok(signature) => pgp::pk_validate(content, &signature, public_keyring_for_validate)
+                .unwrap_or_default(),
+            Err(_) => Default::default(),
+        };
+        Some((first_part, ret_valid_signatures))
     } else {
-        Ok(None)
+        None
     }
 }
 
-fn keyring_from_peerstate(peerstate: &Option<Peerstate>) -> Keyring<SignedPublicKey> {
+pub(crate) fn keyring_from_peerstate(peerstate: Option<&Peerstate>) -> Keyring<SignedPublicKey> {
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
-    if let Some(ref peerstate) = *peerstate {
+    if let Some(peerstate) = peerstate {
         if let Some(key) = &peerstate.public_key {
             public_keyring_for_validate.add(key.clone());
         } else if let Some(key) = &peerstate.gossip_key {

src/dehtml.rs

@@ -20,7 +20,7 @@ struct Dehtml {
     /// increased at each `<div>` and decreased at each `</div>`. This way we know when the quote ends.
     /// If this is > `0`, then we are inside a `<div name="quote">`
     divs_since_quote_div: u32,
-    /// Everything between <div name="quote"> and <div name="quoted-content"> is usually metadata
+    /// Everything between `<div name="quote">` and `<div name="quoted-content">` is usually metadata
     /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
     divs_since_quoted_content_div: u32,
     /// All-Inkl just puts the quote into `<blockquote> </blockquote>`. This count is
@@ -42,7 +42,7 @@ impl Dehtml {
     }
     fn get_add_text(&self) -> AddText {
         if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
-            AddText::No // Everything between <div name="quoted"> and <div name="quoted_content"> is metadata which we don't want
+            AddText::No // Everything between `<div name="quoted">` and `<div name="quoted_content">` is metadata which we don't want
         } else {
             self.add_text
         }
@@ -88,18 +88,30 @@ fn dehtml_quick_xml(buf: &str) -> String {
     let mut buf = Vec::new();
 
     loop {
-        match reader.read_event(&mut buf) {
+        match reader.read_event_into(&mut buf) {
             Ok(quick_xml::events::Event::Start(ref e)) => {
                 dehtml_starttag_cb(e, &mut dehtml, &reader)
             }
             Ok(quick_xml::events::Event::End(ref e)) => dehtml_endtag_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::Text(ref e)) => dehtml_text_cb(e, &mut dehtml),
-            Ok(quick_xml::events::Event::CData(e)) => dehtml_text_cb(&e.escape(), &mut dehtml),
+            Ok(quick_xml::events::Event::CData(e)) => match e.escape() {
+                Ok(e) => dehtml_text_cb(&e, &mut dehtml),
+                Err(e) => {
+                    eprintln!(
+                        "CDATA escape error at position {}: {:?}",
+                        reader.buffer_position(),
+                        e,
+                    );
+                }
+            },
             Ok(quick_xml::events::Event::Empty(ref e)) => {
                 // Handle empty tags as a start tag immediately followed by end tag.
                 // For example, `<p/>` is treated as `<p></p>`.
                 dehtml_starttag_cb(e, &mut dehtml, &reader);
-                dehtml_endtag_cb(&BytesEnd::borrowed(e.name()), &mut dehtml);
+                dehtml_endtag_cb(
+                    &BytesEnd::new(String::from_utf8_lossy(e.name().as_ref())),
+                    &mut dehtml,
+                );
             }
             Err(e) => {
                 eprintln!(
@@ -121,7 +133,7 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
     if dehtml.get_add_text() == AddText::YesPreserveLineEnds
         || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
-        let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
+        let last_added = escaper::decode_html_buf_sloppy(event as &[_]).unwrap_or_default();
 
         if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
@@ -135,7 +147,9 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
 }
 
 fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+    let tag = String::from_utf8_lossy(event.name().as_ref())
+        .trim()
+        .to_lowercase();
 
     match tag.as_str() {
         "p" | "table" | "td" | "style" | "script" | "title" | "pre" => {
@@ -176,7 +190,9 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     dehtml: &mut Dehtml,
     reader: &quick_xml::Reader<B>,
 ) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+    let tag = String::from_utf8_lossy(event.name().as_ref())
+        .trim()
+        .to_lowercase();
 
     match tag.as_str() {
         "p" | "table" | "td" => {
@@ -206,10 +222,15 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             if let Some(href) = event
                 .html_attributes()
                 .filter_map(|attr| attr.ok())
-                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
+                .find(|attr| {
+                    String::from_utf8_lossy(attr.key.as_ref())
+                        .trim()
+                        .to_lowercase()
+                        == "href"
+                })
             {
                 let href = href
-                    .unescape_and_decode_value(reader)
+                    .decode_and_unescape_value(reader)
                     .unwrap_or_default()
                     .to_lowercase();
 
@@ -258,7 +279,7 @@ fn maybe_push_tag(
 fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
     event.attributes().any(|r| {
         r.map(|a| {
-            a.unescape_and_decode_value(reader)
+            a.decode_and_unescape_value(reader)
                 .map(|v| v == name)
                 .unwrap_or(false)
         })

src/download.rs

@@ -31,6 +31,7 @@ const MIN_DOWNLOAD_LIMIT: u32 = 32768;
 /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
 pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
 
+/// Download state of the message.
 #[derive(
     Debug,
     Display,
@@ -47,9 +48,16 @@ pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
 )]
 #[repr(u32)]
 pub enum DownloadState {
+    /// Message is fully downloaded.
     Done = 0,
+
+    /// Message is partially downloaded and can be fully downloaded at request.
     Available = 10,
+
+    /// Failed to fully download the message.
     Failure = 20,
+
+    /// Full download of the message is in progress.
     InProgress = 1000,
 }
 
@@ -124,7 +132,7 @@ impl Job {
     /// Called in response to `Action::DownloadMsg`.
     pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
-            warn!(context, "download: could not connect: {:?}", err);
+            warn!(context, "download: could not connect: {:#}", err);
             return Status::RetryNow;
         }
 

src/e2ee.rs

@@ -297,6 +297,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
             fingerprint_changed: false,
+            verifier: None,
         };
         vec![(Some(peerstate), addr)]
     }

src/ephemeral.rs

@@ -62,6 +62,8 @@
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
+#![allow(missing_docs)]
+
 use std::convert::{TryFrom, TryInto};
 use std::num::ParseIntError;
 use std::str::FromStr;
@@ -203,14 +205,17 @@ impl ChatId {
             return Ok(());
         }
         self.inner_set_ephemeral_timer(context, timer).await?;
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
-        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
-        if let Err(err) = send_msg(context, self, &mut msg).await {
-            error!(
-                context,
-                "Failed to send a message about ephemeral message timer change: {:?}", err
-            );
+
+        if self.is_promoted(context).await? {
+            let mut msg = Message::new(Viewtype::Text);
+            msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
+            msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+            if let Err(err) = send_msg(context, self, &mut msg).await {
+                error!(
+                    context,
+                    "Failed to send a message about ephemeral message timer change: {:?}", err
+                );
+            }
         }
         Ok(())
     }
@@ -628,7 +633,7 @@ mod tests {
     use crate::test_utils::TestContext;
     use crate::tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::{
-        chat::{self, Chat, ChatItem},
+        chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus},
         tools::IsNoneOrEmpty,
     };
 
@@ -793,6 +798,42 @@ mod tests {
         Ok(())
     }
 
+    /// Test that enabling ephemeral timer in unpromoted group does not send a message.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_ephemeral_unpromoted() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let chat_id =
+            create_group_chat(&alice, ProtectionStatus::Unprotected, "Group name").await?;
+
+        // Group is unpromoted, the timer can be changed without sending a message.
+        assert!(chat_id.is_unpromoted(&alice).await?);
+        chat_id
+            .set_ephemeral_timer(&alice, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
+        assert!(sent.is_none());
+        assert_eq!(
+            chat_id.get_ephemeral_timer(&alice).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Promote the group.
+        send_text_msg(&alice, chat_id, "hi!".to_string()).await?;
+        assert!(chat_id.is_promoted(&alice).await?);
+        let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
+        assert!(sent.is_some());
+
+        chat_id
+            .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
+            .await?;
+        let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
+        assert!(sent.is_some());
+        assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
+
+        Ok(())
+    }
+
     /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_ephemeral_enable_lost() -> Result<()> {

src/events.rs

@@ -1,5 +1,7 @@
 //! # Events specification.
 
+#![allow(missing_docs)]
+
 use std::path::PathBuf;
 
 use async_channel::{self as channel, Receiver, Sender, TrySendError};

src/fuzzing.rs

@@ -0,0 +1,12 @@
+//! # Fuzzing module.
+//!
+//! This module exposes private APIs for fuzzing.
+
+/// Fuzzing target for simplify().
+///
+/// Calls simplify() and panics if simplify() panics.
+/// Does not return any vaule to avoid exposing internal crate types.
+#[cfg(fuzzing)]
+pub fn simplify(input: String, is_chat_message: bool) {
+    crate::simplify::simplify(input, is_chat_message);
+}

src/headerdef.rs

@@ -1,5 +1,7 @@
 //! # List of email headers.
 
+#![allow(missing_docs)]
+
 use mailparse::{MailHeader, MailHeaderMap};
 
 #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, IntoStaticStr)]

src/html.rs

@@ -234,7 +234,7 @@ impl HtmlMsgParser {
 /// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
 fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
     let data = mail.get_body_raw()?;
-    let data = base64::encode(&data);
+    let data = base64::encode(data);
     Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
 }
 
@@ -250,7 +250,7 @@ impl MsgId {
         if !rawmime.is_empty() {
             match HtmlMsgParser::from_bytes(context, &rawmime).await {
                 Err(err) => {
-                    warn!(context, "get_html: parser error: {}", err);
+                    warn!(context, "get_html: parser error: {:#}", err);
                     Ok(None)
                 }
                 Ok(parser) => Ok(Some(parser.html)),

src/imap.rs

@@ -297,6 +297,7 @@ impl Imap {
 
         let oauth2 = self.config.lp.oauth2;
 
+        info!(context, "Connecting to IMAP server");
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
         {
@@ -304,22 +305,23 @@ impl Imap {
             let imap_server: &str = config.lp.server.as_ref();
             let imap_port = config.lp.port;
 
-            let connection = if let Some(socks5_config) = &config.socks5_config {
-                Client::connect_insecure_socks5((imap_server, imap_port), socks5_config.clone())
+            if let Some(socks5_config) = &config.socks5_config {
+                if config.lp.security == Socket::Starttls {
+                    Client::connect_starttls_socks5(
+                        imap_server,
+                        imap_port,
+                        socks5_config.clone(),
+                        config.strict_tls,
+                    )
                     .await
+                } else {
+                    Client::connect_insecure_socks5((imap_server, imap_port), socks5_config.clone())
+                        .await
+                }
+            } else if config.lp.security == Socket::Starttls {
+                Client::connect_starttls(imap_server, imap_port, config.strict_tls).await
             } else {
                 Client::connect_insecure((imap_server, imap_port)).await
-            };
-
-            match connection {
-                Ok(client) => {
-                    if config.lp.security == Socket::Starttls {
-                        client.secure(imap_server, config.strict_tls).await
-                    } else {
-                        Ok(client)
-                    }
-                }
-                Err(err) => Err(err),
             }
         } else {
             let config = &self.config;
@@ -328,8 +330,8 @@ impl Imap {
 
             if let Some(socks5_config) = &config.socks5_config {
                 Client::connect_secure_socks5(
-                    (imap_server, imap_port),
                     imap_server,
+                    imap_port,
                     config.strict_tls,
                     socks5_config.clone(),
                 )
@@ -345,6 +347,7 @@ impl Imap {
         let imap_pw: &str = config.lp.password.as_ref();
 
         let login_res = if oauth2 {
+            info!(context, "Logging into IMAP server with OAuth 2");
             let addr: &str = config.addr.as_ref();
 
             let token = get_oauth2_access_token(context, addr, imap_pw, true)
@@ -356,6 +359,7 @@ impl Imap {
             };
             client.authenticate("XOAUTH2", auth).await
         } else {
+            info!(context, "Logging into IMAP server with LOGIN");
             client.login(imap_user, imap_pw).await
         };
 
@@ -371,6 +375,7 @@ impl Imap {
                     "IMAP-LOGIN as {}",
                     self.config.lp.user
                 )));
+                info!(context, "Successfully logged into IMAP server");
                 Ok(())
             }
 
@@ -378,7 +383,7 @@ impl Imap {
                 let imap_user = self.config.lp.user.to_owned();
                 let message = stock_str::cannot_login(context, &imap_user).await;
 
-                warn!(context, "{} ({})", message, err);
+                warn!(context, "{} ({:#})", message, err);
 
                 let lock = context.wrong_pw_warning_mutex.lock().await;
                 if self.login_failed_once
@@ -386,7 +391,7 @@ impl Imap {
                     && context.get_config_bool(Config::NotifyAboutWrongPw).await?
                 {
                     if let Err(e) = context.set_config(Config::NotifyAboutWrongPw, None).await {
-                        warn!(context, "{}", e);
+                        warn!(context, "{:#}", e);
                     }
                     drop(lock);
 
@@ -396,13 +401,13 @@ impl Imap {
                         chat::add_device_msg_with_importance(context, None, Some(&mut msg), true)
                             .await
                     {
-                        warn!(context, "{}", e);
+                        warn!(context, "{:#}", e);
                     }
                 } else {
                     self.login_failed_once = true;
                 }
 
-                Err(format_err!("{}\n\n{}", message, err))
+                Err(format_err!("{}\n\n{:#}", message, err))
             }
         }
     }
@@ -549,7 +554,10 @@ impl Imap {
         folder: &str,
     ) -> Result<bool> {
         let session = self.session.as_mut().context("no session")?;
-        let newly_selected = session.select_or_create_folder(context, folder).await?;
+        let newly_selected = session
+            .select_or_create_folder(context, folder)
+            .await
+            .with_context(|| format!("failed to select or create folder {}", folder))?;
         let mailbox = session
             .selected_mailbox
             .as_mut()
@@ -559,8 +567,12 @@ impl Imap {
             .uid_validity
             .with_context(|| format!("No UIDVALIDITY for folder {}", folder))?;
 
-        let old_uid_validity = get_uidvalidity(context, folder).await?;
-        let old_uid_next = get_uid_next(context, folder).await?;
+        let old_uid_validity = get_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID validity for folder {}", folder))?;
+        let old_uid_next = get_uid_next(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID NEXT for folder {}", folder))?;
 
         if new_uid_validity == old_uid_validity {
             let new_emails = if newly_selected == NewlySelected::No {
@@ -672,7 +684,10 @@ impl Imap {
             return Ok(false);
         }
 
-        let new_emails = self.select_with_uidvalidity(context, folder).await?;
+        let new_emails = self
+            .select_with_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to select folder {}", folder))?;
 
         if !new_emails && !fetch_existing_msgs {
             info!(context, "No new emails in folder {}", folder);
@@ -832,9 +847,15 @@ impl Imap {
         }
         self.prepare(context).await.context("could not connect")?;
 
-        add_all_recipients_as_contacts(context, self, Config::ConfiguredSentboxFolder).await;
-        add_all_recipients_as_contacts(context, self, Config::ConfiguredMvboxFolder).await;
-        add_all_recipients_as_contacts(context, self, Config::ConfiguredInboxFolder).await;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredSentboxFolder)
+            .await
+            .context("failed to get recipients from the sentbox")?;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredMvboxFolder)
+            .await
+            .context("failed to ge recipients from the movebox")?;
+        add_all_recipients_as_contacts(context, self, Config::ConfiguredInboxFolder)
+            .await
+            .context("failed to get recipients from the inbox")?;
 
         if context.get_config_bool(Config::FetchExistingMsgs).await? {
             for config in &[
@@ -843,17 +864,18 @@ impl Imap {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = context.get_config(*config).await? {
+                    info!(
+                        context,
+                        "Fetching existing messages from folder \"{}\"", folder
+                    );
                     self.fetch_new_messages(context, &folder, false, true)
                         .await
-                        .context("could not fetch messages")?;
+                        .context("could not fetch existing messages")?;
                 }
             }
         }
 
         info!(context, "Done fetching existing messages.");
-        context
-            .set_config_bool(Config::FetchedExistingMsgs, true)
-            .await?;
         Ok(())
     }
 }
@@ -1202,8 +1224,10 @@ impl Imap {
     /// Prefetch all messages greater than or equal to `uid_next`. Returns a list of fetch results
     /// in the order of ascending delivery time to the server (INTERNALDATE).
     async fn prefetch(&mut self, uid_next: u32) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
-        let session = self.session.as_mut();
-        let session = session.context("fetch_after(): IMAP No Connection established")?;
+        let session = self
+            .session
+            .as_mut()
+            .context("no IMAP connection established")?;
 
         // fetch messages with larger UID than the last one seen
         let set = format!("{}:*", uid_next);
@@ -1228,7 +1252,6 @@ impl Imap {
                 }
             }
         }
-        drop(list);
 
         Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
     }
@@ -2301,49 +2324,58 @@ impl std::fmt::Display for UidRange {
         }
     }
 }
-async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
-    let mailbox = if let Ok(Some(m)) = context.get_config(folder).await {
+async fn add_all_recipients_as_contacts(
+    context: &Context,
+    imap: &mut Imap,
+    folder: Config,
+) -> Result<()> {
+    let mailbox = if let Some(m) = context.get_config(folder).await? {
         m
     } else {
-        return;
+        info!(
+            context,
+            "Folder {} is not configured, skipping fetching contacts from it.", folder
+        );
+        return Ok(());
     };
-    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
-        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-        warn!(context, "Could not select {}: {:#}", mailbox, e);
-        return;
-    }
-    match imap.get_all_recipients(context).await {
-        Ok(contacts) => {
-            let mut any_modified = false;
-            for contact in contacts {
-                let display_name_normalized = contact
-                    .display_name
-                    .as_ref()
-                    .map(|s| normalize_name(s))
-                    .unwrap_or_default();
+    imap.select_with_uidvalidity(context, &mailbox)
+        .await
+        .with_context(|| format!("could not select {}", mailbox))?;
 
-                match Contact::add_or_lookup(
-                    context,
-                    &display_name_normalized,
-                    &contact.addr,
-                    Origin::OutgoingTo,
-                )
-                .await
-                {
-                    Ok((_, modified)) => {
-                        if modified != Modifier::None {
-                            any_modified = true;
-                        }
-                    }
-                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+    let contacts = imap
+        .get_all_recipients(context)
+        .await
+        .context("could not get recipients")?;
+
+    let mut any_modified = false;
+    for contact in contacts {
+        let display_name_normalized = contact
+            .display_name
+            .as_ref()
+            .map(|s| normalize_name(s))
+            .unwrap_or_default();
+
+        match Contact::add_or_lookup(
+            context,
+            &display_name_normalized,
+            &contact.addr,
+            Origin::OutgoingTo,
+        )
+        .await
+        {
+            Ok((_, modified)) => {
+                if modified != Modifier::None {
+                    any_modified = true;
                 }
             }
-            if any_modified {
-                context.emit_event(EventType::ContactsChanged(None));
-            }
+            Err(e) => warn!(context, "Could not add recipient: {}", e),
         }
-        Err(e) => warn!(context, "Could not add recipients: {}", e),
-    };
+    }
+    if any_modified {
+        context.emit_event(EventType::ContactsChanged(None));
+    }
+
+    Ok(())
 }
 
 #[cfg(test)]

src/imap/client.rs

@@ -8,13 +8,13 @@ use anyhow::{Context as _, Result};
 use async_imap::Client as ImapClient;
 use async_imap::Session as ImapSession;
 
-use tokio::net::{self, TcpStream};
-use tokio::time::timeout;
-use tokio_io_timeout::TimeoutStream;
+use tokio::io::BufWriter;
+use tokio::net::ToSocketAddrs;
 
 use super::capabilities::Capabilities;
 use super::session::Session;
 use crate::login_param::build_tls;
+use crate::net::connect_tcp;
 use crate::socks::Socks5Config;
 
 use super::session::SessionStream;
@@ -24,7 +24,6 @@ pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(30);
 
 #[derive(Debug)]
 pub(crate) struct Client {
-    is_secure: bool,
     inner: ImapClient<Box<dyn SessionStream>>,
 }
 
@@ -93,108 +92,131 @@ impl Client {
     }
 
     pub async fn connect_secure(hostname: &str, port: u16, strict_tls: bool) -> Result<Self> {
-        let tcp_stream = timeout(IMAP_TIMEOUT, TcpStream::connect((hostname, port))).await??;
-        let mut timeout_stream = TimeoutStream::new(tcp_stream);
-        timeout_stream.set_write_timeout(Some(IMAP_TIMEOUT));
-        timeout_stream.set_read_timeout(Some(IMAP_TIMEOUT));
-        let timeout_stream = Box::pin(timeout_stream);
-
+        let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?;
         let tls = build_tls(strict_tls);
-        let tls_stream: Box<dyn SessionStream> =
-            Box::new(tls.connect(hostname, timeout_stream).await?);
-        let mut client = ImapClient::new(tls_stream);
+        let tls_stream = tls.connect(hostname, tcp_stream).await?;
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
+        let mut client = ImapClient::new(session_stream);
 
         let _greeting = client
             .read_response()
             .await
-            .context("failed to read greeting")?;
+            .context("failed to read greeting")??;
 
-        Ok(Client {
-            is_secure: true,
-            inner: client,
-        })
+        Ok(Client { inner: client })
     }
 
-    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> Result<Self> {
-        let tcp_stream = timeout(IMAP_TIMEOUT, TcpStream::connect(addr)).await??;
-        let mut timeout_stream = TimeoutStream::new(tcp_stream);
-        timeout_stream.set_write_timeout(Some(IMAP_TIMEOUT));
-        timeout_stream.set_read_timeout(Some(IMAP_TIMEOUT));
-        let timeout_stream = Box::pin(timeout_stream);
-        let stream: Box<dyn SessionStream> = Box::new(timeout_stream);
-
-        let mut client = ImapClient::new(stream);
+    pub async fn connect_insecure(addr: impl ToSocketAddrs) -> Result<Self> {
+        let tcp_stream = connect_tcp(addr, IMAP_TIMEOUT).await?;
+        let buffered_stream = BufWriter::new(tcp_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
+        let mut client = ImapClient::new(session_stream);
         let _greeting = client
             .read_response()
             .await
-            .context("failed to read greeting")?;
+            .context("failed to read greeting")??;
 
-        Ok(Client {
-            is_secure: false,
-            inner: client,
-        })
+        Ok(Client { inner: client })
+    }
+
+    pub async fn connect_starttls(hostname: &str, port: u16, strict_tls: bool) -> Result<Self> {
+        let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?;
+
+        // Run STARTTLS command and convert the client back into a stream.
+        let session_stream: Box<dyn SessionStream> = Box::new(tcp_stream);
+        let mut client = ImapClient::new(session_stream);
+        let _greeting = client
+            .read_response()
+            .await
+            .context("failed to read greeting")??;
+        client
+            .run_command_and_check_ok("STARTTLS", None)
+            .await
+            .context("STARTTLS command failed")?;
+        let tcp_stream = client.into_inner();
+
+        let tls = build_tls(strict_tls);
+        let tls_stream = tls
+            .connect(hostname, tcp_stream)
+            .await
+            .context("STARTTLS upgrade failed")?;
+
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
+        let client = ImapClient::new(session_stream);
+
+        Ok(Client { inner: client })
     }
 
     pub async fn connect_secure_socks5(
-        target_addr: impl net::ToSocketAddrs,
         domain: &str,
+        port: u16,
         strict_tls: bool,
         socks5_config: Socks5Config,
     ) -> Result<Self> {
-        let socks5_stream: Box<dyn SessionStream> =
-            Box::new(socks5_config.connect(target_addr, IMAP_TIMEOUT).await?);
-
+        let socks5_stream = socks5_config.connect((domain, port), IMAP_TIMEOUT).await?;
         let tls = build_tls(strict_tls);
-        let tls_stream: Box<dyn SessionStream> =
-            Box::new(tls.connect(domain, socks5_stream).await?);
-        let mut client = ImapClient::new(tls_stream);
-
+        let tls_stream = tls.connect(domain, socks5_stream).await?;
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
+        let mut client = ImapClient::new(session_stream);
         let _greeting = client
             .read_response()
             .await
-            .context("failed to read greeting")?;
+            .context("failed to read greeting")??;
 
-        Ok(Client {
-            is_secure: true,
-            inner: client,
-        })
+        Ok(Client { inner: client })
     }
 
     pub async fn connect_insecure_socks5(
-        target_addr: impl net::ToSocketAddrs,
+        target_addr: impl ToSocketAddrs,
         socks5_config: Socks5Config,
     ) -> Result<Self> {
-        let socks5_stream: Box<dyn SessionStream> =
-            Box::new(socks5_config.connect(target_addr, IMAP_TIMEOUT).await?);
-
-        let mut client = ImapClient::new(socks5_stream);
+        let socks5_stream = socks5_config.connect(target_addr, IMAP_TIMEOUT).await?;
+        let buffered_stream = BufWriter::new(socks5_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
+        let mut client = ImapClient::new(session_stream);
         let _greeting = client
             .read_response()
             .await
-            .context("failed to read greeting")?;
+            .context("failed to read greeting")??;
 
-        Ok(Client {
-            is_secure: false,
-            inner: client,
-        })
+        Ok(Client { inner: client })
     }
 
-    pub async fn secure(self, domain: &str, strict_tls: bool) -> Result<Self> {
-        if self.is_secure {
-            Ok(self)
-        } else {
-            let Client { mut inner, .. } = self;
-            let tls = build_tls(strict_tls);
-            inner.run_command_and_check_ok("STARTTLS", None).await?;
+    pub async fn connect_starttls_socks5(
+        hostname: &str,
+        port: u16,
+        socks5_config: Socks5Config,
+        strict_tls: bool,
+    ) -> Result<Self> {
+        let socks5_stream = socks5_config
+            .connect((hostname, port), IMAP_TIMEOUT)
+            .await?;
 
-            let stream = inner.into_inner();
-            let ssl_stream = tls.connect(domain, stream).await?;
-            let boxed: Box<dyn SessionStream> = Box::new(ssl_stream);
+        // Run STARTTLS command and convert the client back into a stream.
+        let session_stream: Box<dyn SessionStream> = Box::new(socks5_stream);
+        let mut client = ImapClient::new(session_stream);
+        let _greeting = client
+            .read_response()
+            .await
+            .context("failed to read greeting")??;
+        client
+            .run_command_and_check_ok("STARTTLS", None)
+            .await
+            .context("STARTTLS command failed")?;
+        let socks5_stream = client.into_inner();
 
-            Ok(Client {
-                is_secure: true,
-                inner: ImapClient::new(boxed),
-            })
-        }
+        let tls = build_tls(strict_tls);
+        let tls_stream = tls
+            .connect(hostname, socks5_stream)
+            .await
+            .context("STARTTLS upgrade failed")?;
+        let buffered_stream = BufWriter::new(tls_stream);
+        let session_stream: Box<dyn SessionStream> = Box::new(buffered_stream);
+        let client = ImapClient::new(session_stream);
+
+        Ok(Client { inner: client })
     }
 }

src/imap/select_folder.rs

@@ -109,13 +109,14 @@ impl ImapSession {
             Ok(newly_selected) => Ok(newly_selected),
             Err(err) => match err {
                 Error::NoFolder(..) => {
+                    info!(context, "Failed to select folder {} because it does not exist, trying to create it.", folder);
                     self.create(folder).await.with_context(|| {
                         format!("Couldn't select folder ('{}'), then create() failed", err)
                     })?;
 
-                    Ok(self.select_folder(context, Some(folder)).await?)
+                    Ok(self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {}", folder))?)
                 }
-                _ => Err(err.into()),
+                _ => Err(err).with_context(|| format!("failed to select folder {} with error other than NO, not trying to create it", folder)),
             },
         }
     }

src/imap/session.rs

@@ -6,6 +6,7 @@ use async_imap::types::Mailbox;
 use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
 use fast_socks5::client::Socks5Stream;
+use tokio::io::BufWriter;
 use tokio::net::TcpStream;
 use tokio_io_timeout::TimeoutStream;
 
@@ -33,12 +34,17 @@ pub(crate) trait SessionStream:
     fn set_read_timeout(&mut self, timeout: Option<Duration>);
 }
 
-impl SessionStream for TlsStream<Box<dyn SessionStream>> {
+impl SessionStream for Box<dyn SessionStream> {
+    fn set_read_timeout(&mut self, timeout: Option<Duration>) {
+        self.as_mut().set_read_timeout(timeout);
+    }
+}
+impl<T: SessionStream> SessionStream for TlsStream<T> {
     fn set_read_timeout(&mut self, timeout: Option<Duration>) {
         self.get_mut().set_read_timeout(timeout);
     }
 }
-impl SessionStream for TlsStream<Pin<Box<TimeoutStream<TcpStream>>>> {
+impl<T: SessionStream> SessionStream for BufWriter<T> {
     fn set_read_timeout(&mut self, timeout: Option<Duration>) {
         self.get_mut().set_read_timeout(timeout);
     }
@@ -48,7 +54,7 @@ impl SessionStream for Pin<Box<TimeoutStream<TcpStream>>> {
         self.as_mut().set_read_timeout_pinned(timeout);
     }
 }
-impl SessionStream for Socks5Stream<Pin<Box<TimeoutStream<TcpStream>>>> {
+impl<T: SessionStream> SessionStream for Socks5Stream<T> {
     fn set_read_timeout(&mut self, timeout: Option<Duration>) {
         self.get_socket_mut().set_read_timeout(timeout)
     }

src/imex.rs

@@ -1,5 +1,7 @@
 //! # Import/export module.
 
+#![allow(missing_docs)]
+
 use std::any::Any;
 use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
@@ -348,7 +350,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
 fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
-        if ('0'..='9').contains(&c) {
+        if c.is_ascii_digit() {
             out.push(c);
             if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
                 out += "-"

src/job.rs

@@ -2,6 +2,9 @@
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
+
+#![allow(missing_docs)]
+
 use std::fmt;
 
 use anyhow::{Context as _, Result};
@@ -154,7 +157,7 @@ impl Job {
     /// Synchronizes UIDs for all folders.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
-            warn!(context, "could not connect: {:?}", err);
+            warn!(context, "could not connect: {:#}", err);
             return Status::RetryLater;
         }
 
@@ -238,12 +241,12 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                 info!(
                     context,
                     "job #{} not succeeded on try #{}, retry in {} seconds.",
-                    job.job_id as u32,
+                    job.job_id,
                     tries,
                     time_offset
                 );
                 job.save(context).await.unwrap_or_else(|err| {
-                    error!(context, "failed to save job: {}", err);
+                    error!(context, "failed to save job: {:#}", err);
                 });
             } else {
                 info!(
@@ -251,7 +254,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
                     "remove job {} as it exhausted {} retries", job, JOB_RETRIES
                 );
                 job.delete(context).await.unwrap_or_else(|err| {
-                    error!(context, "failed to delete job: {}", err);
+                    error!(context, "failed to delete job: {:#}", err);
                 });
             }
         }
@@ -266,7 +269,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
             }
 
             job.delete(context).await.unwrap_or_else(|err| {
-                error!(context, "failed to delete job: {}", err);
+                error!(context, "failed to delete job: {:#}", err);
             });
         }
     }
@@ -400,7 +403,7 @@ LIMIT 1;
             Ok(job) => return Ok(job),
             Err(err) => {
                 // Remove invalid job from the DB
-                info!(context, "cleaning up job, because of {}", err);
+                info!(context, "cleaning up job, because of {:#}", err);
 
                 // TODO: improve by only doing a single query
                 let id = context

src/key.rs

@@ -1,5 +1,7 @@
 //! Cryptographic key module.
 
+#![allow(missing_docs)]
+
 use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
@@ -28,17 +30,13 @@ pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
 pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
-    type KeyType: Serialize + Deserializable + KeyTrait + Clone;
-
     /// Create a key from some bytes.
-    fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
-        Ok(<Self::KeyType as Deserializable>::from_bytes(Cursor::new(
-            bytes,
-        ))?)
+    fn from_slice(bytes: &[u8]) -> Result<Self> {
+        Ok(<Self as Deserializable>::from_bytes(Cursor::new(bytes))?)
     }
 
     /// Create a key from a base64 string.
-    fn from_base64(data: &str) -> Result<Self::KeyType> {
+    fn from_base64(data: &str) -> Result<Self> {
         // strip newlines and other whitespace
         let cleaned: String = data.split_whitespace().collect();
         let bytes = base64::decode(cleaned.as_bytes())?;
@@ -49,15 +47,15 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     ///
     /// Returns the key and a map of any headers which might have been set in
     /// the ASCII-armored representation.
-    fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
+    fn from_asc(data: &str) -> Result<(Self, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::KeyType::from_armor_single(Cursor::new(bytes)).context("rPGP error")
+        Self::from_armor_single(Cursor::new(bytes)).context("rPGP error")
     }
 
     /// Load the users' default key from the database.
     fn load_self<'a>(
         context: &'a Context,
-    ) -> Pin<Box<dyn Future<Output = Result<Self::KeyType>> + 'a + Send>>;
+    ) -> Pin<Box<dyn Future<Output = Result<Self>> + 'a + Send>>;
 
     /// Serialise the key as bytes.
     fn to_bytes(&self) -> Vec<u8> {
@@ -72,7 +70,7 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 
     /// Serialise the key to a base64 string.
     fn to_base64(&self) -> String {
-        base64::encode(&DcKey::to_bytes(self))
+        base64::encode(DcKey::to_bytes(self))
     }
 
     /// Serialise the key to ASCII-armored representation.
@@ -90,11 +88,9 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 }
 
 impl DcKey for SignedPublicKey {
-    type KeyType = SignedPublicKey;
-
     fn load_self<'a>(
         context: &'a Context,
-    ) -> Pin<Box<dyn Future<Output = Result<Self::KeyType>> + 'a + Send>> {
+    ) -> Pin<Box<dyn Future<Output = Result<Self>> + 'a + Send>> {
         Box::pin(async move {
             let addr = context.get_primary_self_addr().await?;
             match context
@@ -141,11 +137,9 @@ impl DcKey for SignedPublicKey {
 }
 
 impl DcKey for SignedSecretKey {
-    type KeyType = SignedSecretKey;
-
     fn load_self<'a>(
         context: &'a Context,
-    ) -> Pin<Box<dyn Future<Output = Result<Self::KeyType>> + 'a + Send>> {
+    ) -> Pin<Box<dyn Future<Output = Result<Self>> + 'a + Send>> {
         Box::pin(async move {
             match context
                 .sql
@@ -385,7 +379,7 @@ impl std::str::FromStr for Fingerprint {
         let hex_repr: String = input
             .to_uppercase()
             .chars()
-            .filter(|&c| ('0'..='9').contains(&c) || ('A'..='F').contains(&c))
+            .filter(|&c| c.is_ascii_hexdigit())
             .collect();
         let v: Vec<u8> = hex::decode(&hex_repr)?;
         ensure!(v.len() == 20, "wrong fingerprint length: {}", hex_repr);

src/keyring.rs

@@ -19,7 +19,7 @@ where
 
 impl<T> Keyring<T>
 where
-    T: DcKey<KeyType = T>,
+    T: DcKey,
 {
     /// New empty keyring.
     pub fn new() -> Keyring<T> {

src/lib.rs

@@ -6,6 +6,7 @@
     unused,
     clippy::correctness,
     missing_debug_implementations,
+    missing_docs,
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
@@ -14,15 +15,13 @@
     clippy::unused_async
 )]
 #![allow(
+    clippy::uninlined_format_args,
     clippy::match_bool,
     clippy::mixed_read_write_in_expression,
     clippy::bool_assert_comparison,
     clippy::manual_split_once,
     clippy::format_push_string,
-    clippy::bool_to_int_with_if,
-    // This lint can be re-enabled once we don't target
-    // Rust 1.56 anymore:
-    clippy::collapsible_str_replace
+    clippy::bool_to_int_with_if
 )]
 
 #[macro_use]
@@ -34,6 +33,7 @@ extern crate rusqlite;
 #[macro_use]
 extern crate strum_macros;
 
+#[allow(missing_docs)]
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
 impl<T: rusqlite::ToSql + Send + Sync> ToSql for T {}
@@ -71,7 +71,6 @@ pub mod imex;
 mod scheduler;
 #[macro_use]
 mod job;
-mod format_flowed;
 pub mod key;
 mod keyring;
 pub mod location;
@@ -101,6 +100,7 @@ mod dehtml;
 mod authres;
 mod color;
 pub mod html;
+mod net;
 pub mod plaintext;
 mod ratelimit;
 pub mod summary;
@@ -118,3 +118,6 @@ pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 mod test_utils;
 #[cfg(test)]
 mod tests;
+
+#[cfg(fuzzing)]
+pub mod fuzzing;

src/location.rs

@@ -1,4 +1,5 @@
 //! Location handling.
+
 use std::convert::TryFrom;
 use std::time::Duration;
 
@@ -17,32 +18,63 @@ use crate::mimeparser::SystemMessage;
 use crate::stock_str;
 use crate::tools::{duration_to_str, time};
 
-/// Location record
+/// Location record.
 #[derive(Debug, Clone, Default)]
 pub struct Location {
+    /// Row ID of the location.
     pub location_id: u32,
+
+    /// Location latitude.
     pub latitude: f64,
+
+    /// Location longitude.
     pub longitude: f64,
+
+    /// Nonstandard `accuracy` attribute of the `coordinates` tag.
     pub accuracy: f64,
+
+    /// Location timestamp in seconds.
     pub timestamp: i64,
+
+    /// Contact ID.
     pub contact_id: ContactId,
+
+    /// Message ID.
     pub msg_id: u32,
+
+    /// Chat ID.
     pub chat_id: ChatId,
+
+    /// A marker string, such as an emoji, to be displayed on top of the location.
     pub marker: Option<String>,
+
+    /// Whether location is independent, i.e. not part of the path.
     pub independent: u32,
 }
 
 impl Location {
+    /// Creates a new empty location.
     pub fn new() -> Self {
         Default::default()
     }
 }
 
+/// KML document.
+///
+/// See <https://www.ogc.org/standards/kml/> for the standard and
+/// <https://developers.google.com/kml> for documentation.
 #[derive(Debug, Clone, Default)]
 pub struct Kml {
+    /// Nonstandard `addr` attribute of the `Document` tag storing the user email address.
     pub addr: Option<String>,
+
+    /// Placemarks.
     pub locations: Vec<Location>,
+
+    /// Currently parsed XML tag.
     tag: KmlTag,
+
+    /// Currently parsed placemark.
     pub curr: Location,
 }
 
@@ -59,10 +91,12 @@ bitflags! {
 }
 
 impl Kml {
+    /// Creates a new empty KML document.
     pub fn new() -> Self {
         Default::default()
     }
 
+    /// Parses a KML document.
     pub fn parse(to_parse: &[u8]) -> Result<Self> {
         ensure!(to_parse.len() <= 1024 * 1024, "kml-file is too large");
 
@@ -75,7 +109,7 @@ impl Kml {
         let mut buf = Vec::new();
 
         loop {
-            match reader.read_event(&mut buf).with_context(|| {
+            match reader.read_event_into(&mut buf).with_context(|| {
                 format!(
                     "location parsing error at position {}",
                     reader.buffer_position()
@@ -83,7 +117,7 @@ impl Kml {
             })? {
                 quick_xml::events::Event::Start(ref e) => kml.starttag_cb(e, &reader),
                 quick_xml::events::Event::End(ref e) => kml.endtag_cb(e),
-                quick_xml::events::Event::Text(ref e) => kml.text_cb(e, &reader),
+                quick_xml::events::Event::Text(ref e) => kml.text_cb(e),
                 quick_xml::events::Event::Eof => break,
                 _ => (),
             }
@@ -93,15 +127,11 @@ impl Kml {
         Ok(kml)
     }
 
-    fn text_cb<B: std::io::BufRead>(&mut self, event: &BytesText, reader: &quick_xml::Reader<B>) {
+    fn text_cb(&mut self, event: &BytesText) {
         if self.tag.contains(KmlTag::WHEN) || self.tag.contains(KmlTag::COORDINATES) {
-            let val = event.unescape_and_decode(reader).unwrap_or_default();
+            let val = event.unescape().unwrap_or_default();
 
-            let val = val
-                .replace('\n', "")
-                .replace('\r', "")
-                .replace('\t', "")
-                .replace(' ', "");
+            let val = val.replace(['\n', '\r', '\t', ' '], "");
 
             if self.tag.contains(KmlTag::WHEN) && val.len() >= 19 {
                 // YYYY-MM-DDTHH:MM:SSZ
@@ -128,7 +158,9 @@ impl Kml {
     }
 
     fn endtag_cb(&mut self, event: &BytesEnd) {
-        let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+        let tag = String::from_utf8_lossy(event.name().as_ref())
+            .trim()
+            .to_lowercase();
 
         if tag == "placemark" {
             if self.tag.contains(KmlTag::PLACEMARK)
@@ -148,14 +180,20 @@ impl Kml {
         event: &BytesStart,
         reader: &quick_xml::Reader<B>,
     ) {
-        let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+        let tag = String::from_utf8_lossy(event.name().as_ref())
+            .trim()
+            .to_lowercase();
         if tag == "document" {
-            if let Some(addr) = event
-                .attributes()
-                .filter_map(|a| a.ok())
-                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "addr")
-            {
-                self.addr = addr.unescape_and_decode_value(reader).ok();
+            if let Some(addr) = event.attributes().filter_map(|a| a.ok()).find(|attr| {
+                String::from_utf8_lossy(attr.key.as_ref())
+                    .trim()
+                    .to_lowercase()
+                    == "addr"
+            }) {
+                self.addr = addr
+                    .decode_and_unescape_value(reader)
+                    .ok()
+                    .map(|a| a.into_owned());
             }
         } else if tag == "placemark" {
             self.tag = KmlTag::PLACEMARK;
@@ -173,12 +211,17 @@ impl Kml {
             self.tag = KmlTag::PLACEMARK | KmlTag::POINT | KmlTag::COORDINATES;
             if let Some(acc) = event.attributes().find(|attr| {
                 attr.as_ref()
-                    .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "accuracy")
+                    .map(|a| {
+                        String::from_utf8_lossy(a.key.as_ref())
+                            .trim()
+                            .to_lowercase()
+                            == "accuracy"
+                    })
                     .unwrap_or_default()
             }) {
                 let v = acc
                     .unwrap()
-                    .unescape_and_decode_value(reader)
+                    .decode_and_unescape_value(reader)
                     .unwrap_or_default();
 
                 self.curr.accuracy = v.trim().parse().unwrap_or_default();
@@ -260,6 +303,7 @@ pub async fn is_sending_locations_to_chat(
     Ok(exists)
 }
 
+/// Sets current location of the user device.
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
     if latitude == 0.0 && longitude == 0.0 {
         return true;
@@ -293,7 +337,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                         ContactId::SELF,
                     ]
             ).await {
-                warn!(context, "failed to store location {:?}", err);
+                warn!(context, "failed to store location {:#}", err);
             } else {
                 info!(context, "stored location for chat {}", chat_id);
                 continue_streaming = true;
@@ -307,6 +351,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
     continue_streaming
 }
 
+/// Searches for locations in the given time range, optionally filtering by chat and contact IDs.
 pub async fn get_range(
     context: &Context,
     chat_id: Option<ChatId>,
@@ -397,6 +442,7 @@ pub async fn delete_all(context: &Context) -> Result<()> {
     Ok(())
 }
 
+/// Returns `location.kml` contents.
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
@@ -482,6 +528,7 @@ fn get_kml_timestamp(utc: i64) -> String {
         .to_string()
 }
 
+/// Returns a KML document containing a single location with the given timestamp and coordinates.
 pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String {
     format!(
         "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
@@ -499,6 +546,7 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
     )
 }
 
+/// Sets the timestamp of the last time location was sent in the chat.
 pub async fn set_kml_sent_timestamp(
     context: &Context,
     chat_id: ChatId,
@@ -514,6 +562,7 @@ pub async fn set_kml_sent_timestamp(
     Ok(())
 }
 
+/// Sets the location of the message.
 pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id: u32) -> Result<()> {
     context
         .sql
@@ -589,7 +638,7 @@ pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receive
     loop {
         let next_event = match maybe_send_locations(context).await {
             Err(err) => {
-                warn!(context, "maybe_send_locations failed: {}", err);
+                warn!(context, "maybe_send_locations failed: {:#}", err);
                 Some(60) // Retry one minute later.
             }
             Ok(next_event) => next_event,

src/log.rs

@@ -1,5 +1,7 @@
 //! # Logging.
 
+#![allow(missing_docs)]
+
 use crate::context::Context;
 
 #[macro_export]

src/message.rs

@@ -1,5 +1,7 @@
 //! # Messages and their identifiers.
 
+#![allow(missing_docs)]
+
 use std::collections::BTreeSet;
 use std::path::{Path, PathBuf};
 
@@ -492,11 +494,12 @@ impl Message {
             .map(|name| name.to_string_lossy().to_string())
     }
 
-    pub async fn get_filebytes(&self, context: &Context) -> u64 {
-        match self.param.get_path(Param::File, context) {
-            Ok(Some(path)) => get_filebytes(context, &path).await,
-            Ok(None) => 0,
-            Err(_) => 0,
+    /// Returns the size of the file in bytes, if applicable.
+    pub async fn get_filebytes(&self, context: &Context) -> Result<Option<u64>> {
+        if let Some(path) = self.param.get_path(Param::File, context)? {
+            Ok(Some(get_filebytes(context, &path).await?))
+        } else {
+            Ok(None)
         }
     }
 
@@ -577,14 +580,14 @@ impl Message {
 
     pub fn has_deviating_timestamp(&self) -> bool {
         let cnv_to_local = gm2local_offset();
-        let sort_timestamp = self.get_sort_timestamp() as i64 + cnv_to_local;
-        let send_timestamp = self.get_timestamp() as i64 + cnv_to_local;
+        let sort_timestamp = self.get_sort_timestamp() + cnv_to_local;
+        let send_timestamp = self.get_timestamp() + cnv_to_local;
 
         sort_timestamp / 86400 != send_timestamp / 86400
     }
 
     pub fn is_sent(&self) -> bool {
-        self.state as i32 >= MessageState::OutDelivered as i32
+        self.state >= MessageState::OutDelivered
     }
 
     pub fn is_forwarded(&self) -> bool {
@@ -897,6 +900,8 @@ impl Message {
     Copy,
     PartialEq,
     Eq,
+    PartialOrd,
+    Ord,
     FromPrimitive,
     ToPrimitive,
     ToSql,
@@ -1098,7 +1103,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     }
 
     if let Some(path) = msg.get_file(context) {
-        let bytes = get_filebytes(context, &path).await;
+        let bytes = get_filebytes(context, &path).await?;
         ret += &format!("\nFile: {}, {}, bytes\n", path.display(), bytes);
     }
 
@@ -1166,6 +1171,7 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "3gp" => (Viewtype::Video, "video/3gpp"),
         "aac" => (Viewtype::Audio, "audio/aac"),
         "avi" => (Viewtype::Video, "video/x-msvideo"),
+        "avif" => (Viewtype::File, "image/avif"), // supported since Android 12 / iOS 16
         "doc" => (Viewtype::File, "application/msword"),
         "docx" => (
             Viewtype::File,
@@ -1174,6 +1180,8 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "epub" => (Viewtype::File, "application/epub+zip"),
         "flac" => (Viewtype::Audio, "audio/flac"),
         "gif" => (Viewtype::Gif, "image/gif"),
+        "heic" => (Viewtype::File, "image/heic"), // supported since Android 10 / iOS 11
+        "heif" => (Viewtype::File, "image/heif"), // supported since Android 10 / iOS 11
         "html" => (Viewtype::File, "text/html"),
         "htm" => (Viewtype::File, "text/html"),
         "ico" => (Viewtype::File, "image/vnd.microsoft.icon"),
@@ -1198,10 +1206,15 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "oga" => (Viewtype::Audio, "audio/ogg"),
         "ogg" => (Viewtype::Audio, "audio/ogg"),
         "ogv" => (Viewtype::File, "video/ogg"),
-        "opus" => (Viewtype::File, "audio/ogg"), // not supported eg. on Android 4
+        "opus" => (Viewtype::File, "audio/ogg"), // supported since Android 10
         "otf" => (Viewtype::File, "font/otf"),
         "pdf" => (Viewtype::File, "application/pdf"),
         "png" => (Viewtype::Image, "image/png"),
+        "ppt" => (Viewtype::File, "application/vnd.ms-powerpoint"),
+        "pptx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        ),
         "rar" => (Viewtype::File, "application/vnd.rar"),
         "rtf" => (Viewtype::File, "application/rtf"),
         "spx" => (Viewtype::File, "audio/ogg"), // Ogg Speex Profile
@@ -1210,6 +1223,7 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "tiff" => (Viewtype::File, "image/tiff"),
         "tif" => (Viewtype::File, "image/tiff"),
         "ttf" => (Viewtype::File, "font/ttf"),
+        "txt" => (Viewtype::File, "text/plain"),
         "vcard" => (Viewtype::File, "text/vcard"),
         "vcf" => (Viewtype::File, "text/vcard"),
         "wav" => (Viewtype::File, "audio/wav"),
@@ -1219,11 +1233,12 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
         "xdc" => (Viewtype::Webxdc, "application/webxdc+zip"),
         "xhtml" => (Viewtype::File, "application/xhtml+xml"),
+        "xls" => (Viewtype::File, "application/vnd.ms-excel"),
         "xlsx" => (
             Viewtype::File,
             "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
         ),
-        "xml" => (Viewtype::File, "application/vnd.ms-excel"),
+        "xml" => (Viewtype::File, "application/xml"),
         "zip" => (Viewtype::File, "application/zip"),
         _ => {
             return None;
@@ -1675,7 +1690,7 @@ pub async fn get_unblocked_msg_cnt(context: &Context) -> usize {
     {
         Ok(res) => res,
         Err(err) => {
-            error!(context, "get_unblocked_msg_cnt() failed. {}", err);
+            error!(context, "get_unblocked_msg_cnt() failed. {:#}", err);
             0
         }
     }
@@ -1695,7 +1710,7 @@ pub async fn get_request_msg_cnt(context: &Context) -> usize {
     {
         Ok(res) => res,
         Err(err) => {
-            error!(context, "get_request_msg_cnt() failed. {}", err);
+            error!(context, "get_request_msg_cnt() failed. {:#}", err);
             0
         }
     }
@@ -1875,7 +1890,7 @@ mod tests {
     use crate::chatlist::Chatlist;
     use crate::receive_imf::receive_imf;
     use crate::test_utils as test;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{TestContext, TestContextManager};
 
     use super::*;
 
@@ -2338,4 +2353,57 @@ mod tests {
         );
         assert_eq!(Viewtype::Webxdc, Viewtype::from_i32(80).unwrap());
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_send_quotes() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        let sent = alice.send_text(chat.id, "> First quote").await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some("> First quote"));
+        assert!(received.quoted_text().is_none());
+        assert!(received.quoted_message(&bob).await?.is_none());
+
+        let sent = alice.send_text(chat.id, "> Second quote").await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some("> Second quote"));
+        assert!(received.quoted_text().is_none());
+        assert!(received.quoted_message(&bob).await?.is_none());
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_format_flowed_round_trip() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+        let chat = alice.create_chat(&bob).await;
+
+        let text = "  Foo bar";
+        let sent = alice.send_text(chat.id, text).await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some(text));
+
+        let text = "Foo                         bar                                                             baz";
+        let sent = alice.send_text(chat.id, text).await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some(text));
+
+        let text = "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A";
+        let sent = alice.send_text(chat.id, text).await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some(text));
+
+        let python_program = "\
+def hello():
+    return 'Hello, world!'";
+        let sent = alice.send_text(chat.id, python_program).await;
+        let received = bob.recv_msg(&sent).await;
+        assert_eq!(received.text.as_deref(), Some(python_program));
+
+        Ok(())
+    }
 }

src/mimefactory.rs

@@ -4,6 +4,7 @@ use std::convert::TryInto;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use chrono::TimeZone;
+use format_flowed::{format_flowed, format_flowed_quote};
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 use tokio::fs;
 
@@ -15,7 +16,6 @@ use crate::contact::Contact;
 use crate::context::{get_version_str, Context};
 use crate::e2ee::EncryptHelper;
 use crate::ephemeral::Timer as EphemeralTimer;
-use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::html::new_html_mimepart;
 use crate::location;
 use crate::message::{self, Message, MsgId, Viewtype};
@@ -799,6 +799,7 @@ impl<'a> MimeFactory<'a> {
         })
     }
 
+    /// Returns MIME part with a `message.kml` attachment.
     fn get_message_kml_part(&self) -> Option<PartBuilder> {
         let latitude = self.msg.param.get_float(Param::SetLatitude)?;
         let longitude = self.msg.param.get_float(Param::SetLongitude)?;
@@ -818,6 +819,7 @@ impl<'a> MimeFactory<'a> {
         Some(part)
     }
 
+    /// Returns MIME part with a `location.kml` attachment.
     async fn get_location_kml_part(&mut self, context: &Context) -> Result<PartBuilder> {
         let (kml_content, last_added_location_id) =
             location::get_kml(context, self.msg.chat_id).await?;
@@ -1430,7 +1432,7 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool
 async fn is_file_size_okay(context: &Context, msg: &Message) -> Result<bool> {
     match msg.param.get_path(Param::File, context)? {
         Some(path) => {
-            let bytes = get_filebytes(context, &path).await;
+            let bytes = get_filebytes(context, &path).await?;
             Ok(bytes <= UPPER_LIMIT_FILE_SIZE)
         }
         None => Ok(false),

src/mimeparser.rs

@@ -1,26 +1,33 @@
 //! # MIME message parsing module.
 
+#![allow(missing_docs)]
+
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::pin::Pin;
+use std::str;
 
 use anyhow::{bail, Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
+use format_flowed::unformat_flowed;
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 use once_cell::sync::Lazy;
 
-use crate::aheader::Aheader;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::blob::BlobObject;
 use crate::constants::{DC_DESIRED_TEXT_LINES, DC_DESIRED_TEXT_LINE_LEN};
 use crate::contact::{addr_cmp, addr_normalize, ContactId};
 use crate::context::Context;
-use crate::decrypt::{prepare_decryption, try_decrypt};
+use crate::decrypt::{
+    keyring_from_peerstate, prepare_decryption, try_decrypt, validate_detached_signature,
+    DecryptionInfo,
+};
 use crate::dehtml::dehtml;
 use crate::events::EventType;
-use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::key::Fingerprint;
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
+use crate::keyring::Keyring;
 use crate::message::{self, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
@@ -52,6 +59,7 @@ pub struct MimeMessage {
     pub from_is_signed: bool,
     pub list_post: Option<String>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
+    pub decryption_info: DecryptionInfo,
     pub decrypting_failed: bool,
 
     /// Set of valid signature fingerprints if a message is an
@@ -60,7 +68,6 @@ pub struct MimeMessage {
     /// If a message is not encrypted or the signature is not valid,
     /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
-
     /// The set of mail recipient addresses for which gossip headers were applied, regardless of
     /// whether they modified any peerstates.
     pub gossiped_addr: HashSet<String>,
@@ -216,16 +223,12 @@ impl MimeMessage {
         headers.remove("secure-join-fingerprint");
         headers.remove("chat-verified");
 
-        let is_thunderbird = headers
-            .get("user-agent")
-            .map_or(false, |user_agent| user_agent.contains("Thunderbird"));
-        if is_thunderbird {
-            info!(context, "Detected Thunderbird");
-        }
-
         let from = from.context("No from in message")?;
+        let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context)
+            .await
+            .context("failed to get own keyring")?;
         let mut decryption_info =
-            prepare_decryption(context, &mail, &from.addr, message_time, is_thunderbird).await?;
+            prepare_decryption(context, &mail, &from.addr, message_time).await?;
 
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
@@ -234,92 +237,100 @@ impl MimeMessage {
         hop_info += "\n\n";
         hop_info += &decryption_info.dkim_results.to_string();
 
-        // `signatures` is non-empty exactly if the message was encrypted and correctly signed.
-        let (mail, signatures, warn_empty_signature) =
-            match try_decrypt(context, &mail, &decryption_info).await {
+        let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref());
+        let (mail, mut signatures, encrypted) =
+            match try_decrypt(context, &mail, &private_keyring, &public_keyring) {
                 Ok(Some((raw, signatures))) => {
-                    // Encrypted, but maybe unsigned message. Only if
-                    // `signatures` set is non-empty, it is a valid
-                    // autocrypt message.
-
                     mail_raw = raw;
                     let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
                     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
                         info!(context, "decrypted message mime-body:");
                         println!("{}", String::from_utf8_lossy(&mail_raw));
                     }
-
-                    // Handle any gossip headers if the mail was encrypted.  See section
-                    // "3.6 Key Gossip" of <https://autocrypt.org/autocrypt-spec-1.1.0.pdf>
-                    // but only if the mail was correctly signed:
-                    if !signatures.is_empty() {
-                        let gossip_headers =
-                            decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
-                        gossiped_addr =
-                            update_gossip_peerstates(context, message_time, &mail, gossip_headers)
-                                .await?;
-                    }
-
-                    // let known protected headers from the decrypted
-                    // part override the unencrypted top-level
-
-                    // Signature was checked for original From, so we
-                    // do not allow overriding it.
-                    let mut signed_from = None;
-
-                    // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
-                    // See <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
-                    headers.remove("subject");
-
-                    MimeMessage::merge_headers(
-                        context,
-                        &mut headers,
-                        &mut recipients,
-                        &mut signed_from,
-                        &mut list_post,
-                        &mut chat_disposition_notification_to,
-                        &decrypted_mail.headers,
-                    );
-                    if let Some(signed_from) = signed_from {
-                        if addr_cmp(&signed_from.addr, &from.addr) {
-                            from_is_signed = true;
-                        } else {
-                            // There is a From: header in the encrypted &
-                            // signed part, but it doesn't match the outer one.
-                            // This _might_ be because the sender's mail server
-                            // replaced the sending address, e.g. in a mailing list.
-                            // Or it's because someone is doing some replay attack
-                            // - OTOH, I can't come up with an attack scenario
-                            // where this would be useful.
-                            warn!(
-                                context,
-                                "From header in signed part does't match the outer one"
-                            );
-                        }
-                    }
-
                     (Ok(decrypted_mail), signatures, true)
                 }
-                Ok(None) => {
-                    // Message was not encrypted.
-                    // If it is not a read receipt, degrade encryption.
-                    if let Some(peerstate) = &mut decryption_info.peerstate {
-                        if message_time > peerstate.last_seen_autocrypt
-                            && mail.ctype.mimetype != "multipart/report"
-                        // Disallowing keychanges is disabled for now:
-                        // && decryption_info.dkim_results.allow_keychange
-                        {
-                            peerstate.degrade_encryption(message_time);
-                            peerstate.save_to_db(&context.sql).await?;
-                        }
-                    }
-                    (Ok(mail), HashSet::new(), false)
-                }
+                Ok(None) => (Ok(mail), HashSet::new(), false),
                 Err(err) => {
-                    warn!(context, "decryption failed: {}", err);
-                    (Err(err), HashSet::new(), true)
+                    warn!(context, "decryption failed: {:#}", err);
+                    (Err(err), HashSet::new(), false)
                 }
             };
+        let mail = mail.as_ref().map(|mail| {
+            let (content, signatures_detached) = validate_detached_signature(mail, &public_keyring)
+                .unwrap_or((mail, Default::default()));
+            signatures.extend(signatures_detached);
+            content
+        });
+        if let (Ok(mail), true) = (mail, encrypted) {
+            // Handle any gossip headers if the mail was encrypted.  See section
+            // "3.6 Key Gossip" of <https://autocrypt.org/autocrypt-spec-1.1.0.pdf>
+            // but only if the mail was correctly signed:
+            if !signatures.is_empty() {
+                let gossip_headers = mail.headers.get_all_values("Autocrypt-Gossip");
+                gossiped_addr = update_gossip_peerstates(
+                    context,
+                    message_time,
+                    &from.addr,
+                    &recipients,
+                    gossip_headers,
+                )
+                .await?;
+            }
+
+            // let known protected headers from the decrypted
+            // part override the unencrypted top-level
+
+            // Signature was checked for original From, so we
+            // do not allow overriding it.
+            let mut signed_from = None;
+
+            // We do not want to allow unencrypted subject in encrypted emails because the
+            // user might falsely think that the subject is safe.
+            // See <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
+            headers.remove("subject");
+
+            MimeMessage::merge_headers(
+                context,
+                &mut headers,
+                &mut recipients,
+                &mut signed_from,
+                &mut list_post,
+                &mut chat_disposition_notification_to,
+                &mail.headers,
+            );
+            if let Some(signed_from) = signed_from {
+                if addr_cmp(&signed_from.addr, &from.addr) {
+                    from_is_signed = true;
+                } else {
+                    // There is a From: header in the encrypted &
+                    // signed part, but it doesn't match the outer one.
+                    // This _might_ be because the sender's mail server
+                    // replaced the sending address, e.g. in a mailing list.
+                    // Or it's because someone is doing some replay attack
+                    // - OTOH, I can't come up with an attack scenario
+                    // where this would be useful.
+                    warn!(
+                        context,
+                        "From header in signed part does't match the outer one",
+                    );
+                }
+            }
+        }
+        if signatures.is_empty() {
+            // If it is not a read receipt, degrade encryption.
+            if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) {
+                if message_time > peerstate.last_seen_autocrypt
+                    && mail.ctype.mimetype != "multipart/report"
+                // Disallowing keychanges is disabled for now:
+                // && decryption_info.dkim_results.allow_keychange
+                {
+                    peerstate.degrade_encryption(message_time);
+                }
+            }
+        }
+        if !encrypted {
+            signatures.clear();
+        }
 
         let mut parser = MimeMessage {
             parts: Vec::new(),
@@ -329,6 +340,7 @@ impl MimeMessage {
             from,
             from_is_signed,
             chat_disposition_notification_to,
+            decryption_info,
             decrypting_failed: mail.is_err(),
 
             // only non-empty if it was a valid autocrypt message
@@ -358,7 +370,7 @@ impl MimeMessage {
             }
             None => match mail {
                 Ok(mail) => {
-                    parser.parse_mime_recursive(context, &mail, false).await?;
+                    parser.parse_mime_recursive(context, mail, false).await?;
                 }
                 Err(err) => {
                     let msg_body = stock_str::cant_decrypt_msg_body(context).await;
@@ -368,7 +380,7 @@ impl MimeMessage {
                         typ: Viewtype::Text,
                         msg_raw: Some(txt.clone()),
                         msg: txt,
-                        error: Some(format!("Decrypting failed: {}", err)),
+                        error: Some(format!("Decrypting failed: {:#}", err)),
                         ..Default::default()
                     };
                     parser.parts.push(part);
@@ -387,21 +399,23 @@ impl MimeMessage {
         //         part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string());
         //     }
         // }
-        if warn_empty_signature && parser.signatures.is_empty() {
-            for part in parser.parts.iter_mut() {
-                part.error = Some("No valid signature".to_string());
-            }
-        }
 
         if parser.is_mime_modified {
             parser.decoded_data = mail_raw;
         }
 
-        crate::peerstate::maybe_do_aeap_transition(context, &mut decryption_info, &parser).await?;
-        if let Some(peerstate) = decryption_info.peerstate {
+        crate::peerstate::maybe_do_aeap_transition(context, &mut parser).await?;
+        if let Some(peerstate) = &parser.decryption_info.peerstate {
             peerstate
                 .handle_fingerprint_change(context, message_time)
                 .await?;
+            // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
+            // of an error. Otherwise we don't save peerstate until get here to reduce the number of
+            // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
+            // successfully.
+            if peerstate.prefer_encrypt != EncryptPreference::Mutual {
+                peerstate.save_to_db(&context.sql).await?;
+            }
         }
 
         Ok(parser)
@@ -666,7 +680,7 @@ impl MimeMessage {
                     Err(err) => {
                         warn!(
                             context,
-                            "Could not save decoded avatar to blob file: {}", err
+                            "Could not save decoded avatar to blob file: {:#}", err
                         );
                         None
                     }
@@ -957,7 +971,7 @@ impl MimeMessage {
                     &filename,
                     is_related,
                 )
-                .await;
+                .await?;
             }
             None => {
                 match mime_type.type_() {
@@ -973,7 +987,7 @@ impl MimeMessage {
                         let decoded_data = match mail.get_body() {
                             Ok(decoded_data) => decoded_data,
                             Err(err) => {
-                                warn!(context, "Invalid body parsed {:?}", err);
+                                warn!(context, "Invalid body parsed {:#}", err);
                                 // Note that it's not always an error - might be no data
                                 return Ok(false);
                             }
@@ -993,7 +1007,7 @@ impl MimeMessage {
                         let decoded_data = match mail.get_body() {
                             Ok(decoded_data) => decoded_data,
                             Err(err) => {
-                                warn!(context, "Invalid body parsed {:?}", err);
+                                warn!(context, "Invalid body parsed {:#}", err);
                                 // Note that it's not always an error - might be no data
                                 return Ok(false);
                             }
@@ -1100,9 +1114,18 @@ impl MimeMessage {
         decoded_data: &[u8],
         filename: &str,
         is_related: bool,
-    ) {
+    ) -> Result<()> {
         if decoded_data.is_empty() {
-            return;
+            return Ok(());
+        }
+        if let Some(peerstate) = &mut self.decryption_info.peerstate {
+            if peerstate.prefer_encrypt != EncryptPreference::Mutual
+                && mime_type.type_() == mime::APPLICATION
+                && mime_type.subtype().as_str() == "pgp-keys"
+                && Self::try_set_peer_key_from_file_part(context, peerstate, decoded_data).await?
+            {
+                return Ok(());
+            }
         }
         let msg_type = if context
             .is_webxdc_file(filename, decoded_data)
@@ -1116,7 +1139,7 @@ impl MimeMessage {
             if filename.starts_with("location") || filename.starts_with("message") {
                 let parsed = location::Kml::parse(decoded_data)
                     .map_err(|err| {
-                        warn!(context, "failed to parse kml part: {}", err);
+                        warn!(context, "failed to parse kml part: {:#}", err);
                     })
                     .ok();
                 if filename.starts_with("location") {
@@ -1124,7 +1147,7 @@ impl MimeMessage {
                 } else {
                     self.message_kml = parsed;
                 }
-                return;
+                return Ok(());
             }
             msg_type
         } else if filename == "multi-device-sync.json" {
@@ -1134,16 +1157,16 @@ impl MimeMessage {
             self.sync_items = context
                 .parse_sync_items(serialized)
                 .map_err(|err| {
-                    warn!(context, "failed to parse sync data: {}", err);
+                    warn!(context, "failed to parse sync data: {:#}", err);
                 })
                 .ok();
-            return;
+            return Ok(());
         } else if filename == "status-update.json" {
             let serialized = String::from_utf8_lossy(decoded_data)
                 .parse()
                 .unwrap_or_default();
             self.webxdc_status_update = Some(serialized);
-            return;
+            return Ok(());
         } else {
             msg_type
         };
@@ -1156,9 +1179,9 @@ impl MimeMessage {
             Err(err) => {
                 error!(
                     context,
-                    "Could not add blob for mime part {}, error {}", filename, err
+                    "Could not add blob for mime part {}, error {:#}", filename, err
                 );
-                return;
+                return Ok(());
             }
         };
         info!(context, "added blobfile: {:?}", blob.as_name());
@@ -1181,6 +1204,66 @@ impl MimeMessage {
         part.is_related = is_related;
 
         self.do_add_single_part(part);
+        Ok(())
+    }
+
+    /// Returns whether a key from the attachment was set as peer's pubkey.
+    async fn try_set_peer_key_from_file_part(
+        context: &Context,
+        peerstate: &mut Peerstate,
+        decoded_data: &[u8],
+    ) -> Result<bool> {
+        let key = match str::from_utf8(decoded_data) {
+            Err(err) => {
+                warn!(context, "PGP key attachment is not a UTF-8 file: {}", err);
+                return Ok(false);
+            }
+            Ok(key) => key,
+        };
+        let key = match SignedPublicKey::from_asc(key) {
+            Err(err) => {
+                warn!(
+                    context,
+                    "PGP key attachment is not an ASCII-armored file: {:#}", err
+                );
+                return Ok(false);
+            }
+            Ok((key, _)) => key,
+        };
+        if let Err(err) = key.verify() {
+            warn!(context, "attached PGP key verification failed: {}", err);
+            return Ok(false);
+        }
+        if !key.details.users.iter().any(|user| {
+            user.id
+                .id()
+                .ends_with(&(String::from("<") + &peerstate.addr + ">"))
+        }) {
+            return Ok(false);
+        }
+        if let Some(curr_key) = &peerstate.public_key {
+            if key != *curr_key && peerstate.prefer_encrypt != EncryptPreference::Reset {
+                // We don't want to break the existing Autocrypt setup. Yes, it's unlikely that a
+                // user have an Autocrypt-capable MUA and also attaches a key, but if that's the
+                // case, let 'em first disable Autocrypt and then change the key by attaching it.
+                warn!(
+                    context,
+                    "not using attached PGP key for peer '{}' because another one is already set \
+                    with prefer-encrypt={}",
+                    peerstate.addr,
+                    peerstate.prefer_encrypt,
+                );
+                return Ok(false);
+            }
+        }
+        peerstate.public_key = Some(key);
+        info!(
+            context,
+            "using attached PGP key for peer '{}' with prefer-encrypt=mutual", peerstate.addr,
+        );
+        peerstate.prefer_encrypt = EncryptPreference::Mutual;
+        peerstate.save_to_db(&context.sql).await?;
+        Ok(true)
     }
 
     fn do_add_single_part(&mut self, mut part: Part) {
@@ -1552,12 +1635,15 @@ impl MimeMessage {
 }
 
 /// Parses `Autocrypt-Gossip` headers from the email and applies them to peerstates.
+/// Params:
+/// from: The address which sent the message currently beeing parsed
 ///
 /// Returns the set of mail recipient addresses for which valid gossip headers were found.
 async fn update_gossip_peerstates(
     context: &Context,
     message_time: i64,
-    mail: &mailparse::ParsedMail<'_>,
+    from: &str,
+    recipients: &[SingleInfo],
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
@@ -1572,9 +1658,9 @@ async fn update_gossip_peerstates(
             }
         };
 
-        if !get_recipients(&mail.headers)
+        if !recipients
             .iter()
-            .any(|info| info.addr == header.addr.to_lowercase())
+            .any(|info| addr_cmp(&info.addr, &header.addr))
         {
             warn!(
                 context,
@@ -1582,6 +1668,14 @@ async fn update_gossip_peerstates(
             );
             continue;
         }
+        if addr_cmp(from, &header.addr) {
+            // Non-standard, but anyway we can't update the cached peerstate here.
+            warn!(
+                context,
+                "Ignoring gossiped \"{}\" as it equals the From address", &header.addr,
+            );
+            continue;
+        }
 
         let peerstate;
         if let Some(mut p) = Peerstate::from_addr(context, &header.addr).await? {
@@ -1936,7 +2030,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mimeparser_crash() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1948,7 +2042,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_rfc724_mid_exists() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/mail_with_message_id.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -1962,7 +2056,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_rfc724_mid_not_exists() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -2168,7 +2262,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_parent_timestamp() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = b"From: foo@example.org\n\
                     Content-Type: text/plain\n\
                     Chat-Version: 1.0\n\
@@ -2201,7 +2295,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mimeparser_with_context() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = b"From: hello@example.org\n\
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
@@ -2253,7 +2347,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mimeparser_with_avatars() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
         let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
@@ -2294,7 +2388,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mimeparser_with_videochat() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
 
         let raw = include_bytes!("../test-data/message/videochat_invitation.eml");
         let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
@@ -2316,7 +2410,7 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mimeparser_message_kml() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = b"Chat-Version: 1.0\n\
 From: foo <foo@example.org>\n\
 To: bar <bar@example.org>\n\
@@ -2361,7 +2455,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_mdn() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -2411,7 +2505,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
     /// multipart MIME messages.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_multiple_mdns() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -2487,7 +2581,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_mdn_with_additional_message_ids() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
 Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
 Chat-Version: 1.0\n\
@@ -2542,7 +2636,7 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_parse_inline_attachment() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
 From: sender@example.com
 To: receiver@example.com
@@ -2582,7 +2676,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_hide_html_without_content() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
 From: sender@example.com
 To: receiver@example.com
@@ -2631,7 +2725,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn parse_inline_image() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br#"Message-ID: <foobar@example.org>
 From: foo <foo@example.org>
 Subject: example
@@ -2677,7 +2771,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn parse_thunderbird_html_embedded_image() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br#"To: Alice <alice@example.org>
 From: Bob <bob@example.org>
 Subject: Test subject
@@ -2750,7 +2844,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
     // Outlook specifies filename in the "name" attribute of Content-Type
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn parse_outlook_html_embedded_image() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br##"From: Anonymous <anonymous@example.org>
 To: Anonymous <anonymous@example.org>
 Subject: Delta Chat is great stuff!
@@ -2889,7 +2983,7 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn parse_format_flowed_quote() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 Subject: Re: swipe-to-reply
 MIME-Version: 1.0
@@ -2925,7 +3019,7 @@ Reply
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn parse_quote_without_reply() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 Subject: Re: swipe-to-reply
 MIME-Version: 1.0
@@ -2957,7 +3051,7 @@ From: alice <alice@example.org>
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn parse_quote_top_posting() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 Subject: Re: top posting
 MIME-Version: 1.0
@@ -2988,7 +3082,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_attachment_quote() {
-        let context = TestContext::new().await;
+        let context = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/quote_attach.eml");
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
             .await
@@ -3006,7 +3100,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_quote_div() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/gmx-quote.eml");
         let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap();
         assert_eq!(mimeparser.parts[0].msg, "YIPPEEEEEE\n\nMulti-line");
@@ -3016,7 +3110,7 @@ On 2020-10-25, Bob wrote:
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_allinkl_blockquote() {
         // all-inkl.com puts quotes into `<blockquote> </blockquote>`.
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/allinkl-quote.eml");
         let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap();
         assert!(mimeparser.parts[0].msg.starts_with("It's 1.0."));
@@ -3051,7 +3145,7 @@ On 2020-10-25, Bob wrote:
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
         assert_eq!(msg.chat_blocked, Blocked::Request);
         assert_eq!(msg.state, MessageState::InFresh);
-        assert_eq!(msg.get_filebytes(&t).await, 2115);
+        assert_eq!(msg.get_filebytes(&t).await.unwrap().unwrap(), 2115);
         assert!(msg.get_file(&t).is_some());
         assert_eq!(msg.get_filename().unwrap(), "avatar64x64.png");
         assert_eq!(msg.get_width(), 64);
@@ -3061,7 +3155,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mime_modified_plain() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(!mimeparser.is_mime_modified);
@@ -3073,7 +3167,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mime_modified_alt_plain_html() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(mimeparser.is_mime_modified);
@@ -3085,7 +3179,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mime_modified_alt_plain() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(!mimeparser.is_mime_modified);
@@ -3100,7 +3194,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mime_modified_alt_html() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(mimeparser.is_mime_modified);
@@ -3112,7 +3206,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mime_modified_html() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let raw = include_bytes!("../test-data/message/text_html.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(mimeparser.is_mime_modified);
@@ -3124,7 +3218,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mime_modified_large_plain() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
 
         static REPEAT_TXT: &str = "this text with 42 chars is just repeated.\n";
         static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_DESIRED_TEXT_LEN
@@ -3145,7 +3239,7 @@ On 2020-10-25, Bob wrote:
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_x_microsoft_original_message_id() {
-        let t = TestContext::new().await;
+        let t = TestContext::new_alice().await;
         let message = MimeMessage::from_bytes(&t, b"Date: Wed, 17 Feb 2021 15:45:15 +0000\n\
                 Chat-Version: 1.0\n\
                 Message-ID: <DBAPR03MB1180CE51A1BFE265BD018D4790869@DBAPR03MB6691.eurprd03.prod.outlook.com>\n\

src/net.rs

@@ -0,0 +1,33 @@
+///! # Common network utilities.
+use std::pin::Pin;
+use std::time::Duration;
+
+use anyhow::{Context as _, Result};
+use tokio::net::{TcpStream, ToSocketAddrs};
+use tokio::time::timeout;
+use tokio_io_timeout::TimeoutStream;
+
+/// Returns a TCP connection stream with read/write timeouts set
+/// and Nagle's algorithm disabled with `TCP_NODELAY`.
+///
+/// `TCP_NODELAY` ensures writing to the stream always results in immediate sending of the packet
+/// to the network, which is important to reduce the latency of interactive protocols such as IMAP.
+pub(crate) async fn connect_tcp(
+    addr: impl ToSocketAddrs,
+    timeout_val: Duration,
+) -> Result<Pin<Box<TimeoutStream<TcpStream>>>> {
+    let tcp_stream = timeout(timeout_val, TcpStream::connect(addr))
+        .await
+        .context("connection timeout")?
+        .context("connection failure")?;
+
+    // Disable Nagle's algorithm.
+    tcp_stream.set_nodelay(true)?;
+
+    let mut timeout_stream = TimeoutStream::new(tcp_stream);
+    timeout_stream.set_write_timeout(Some(timeout_val));
+    timeout_stream.set_read_timeout(Some(timeout_val));
+    let pinned_stream = Box::pin(timeout_stream);
+
+    Ok(pinned_stream)
+}

src/oauth2.rs

@@ -1,5 +1,7 @@
 //! OAuth 2 module.
 
+#![allow(missing_docs)]
+
 use std::collections::HashMap;
 
 use anyhow::Result;

src/param.rs

@@ -131,7 +131,7 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
-    /// For Chats: On sending a new message we set the subject to "Re: <last subject>".
+    /// For Chats: On sending a new message we set the subject to `Re: <last subject>`.
     /// Usually we just use the subject of the parent message, but if the parent message
     /// is deleted, we use the LastSubject of the chat.
     LastSubject = b't',

src/peerstate.rs

@@ -1,5 +1,7 @@
 //! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module.
 
+#![allow(missing_docs)]
+
 use std::collections::HashSet;
 use std::fmt;
 
@@ -9,7 +11,6 @@ use crate::chatlist::Chatlist;
 use crate::constants::Chattype;
 use crate::contact::{addr_cmp, Contact, Origin};
 use crate::context::Context;
-use crate::decrypt::DecryptionInfo;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::message::Message;
@@ -47,6 +48,8 @@ pub struct Peerstate {
     pub verified_key: Option<SignedPublicKey>,
     pub verified_key_fingerprint: Option<Fingerprint>,
     pub fingerprint_changed: bool,
+    /// The address that verified this contact
+    pub verifier: Option<String>,
 }
 
 impl PartialEq for Peerstate {
@@ -102,9 +105,11 @@ impl Peerstate {
             verified_key: None,
             verified_key_fingerprint: None,
             fingerprint_changed: false,
+            verifier: None,
         }
     }
 
+    /// Create peerstate from gossip
     pub fn from_gossip(gossip_header: &Aheader, message_time: i64) -> Self {
         Peerstate {
             addr: gossip_header.addr.clone(),
@@ -118,7 +123,6 @@ impl Peerstate {
             // learn encryption preferences of other members immediately and don't send unencrypted
             // messages to a group where everyone prefers encryption.
             prefer_encrypt: gossip_header.prefer_encrypt,
-
             public_key: None,
             public_key_fingerprint: None,
             gossip_key: Some(gossip_header.public_key.clone()),
@@ -127,13 +131,14 @@ impl Peerstate {
             verified_key: None,
             verified_key_fingerprint: None,
             fingerprint_changed: false,
+            verifier: None,
         }
     }
 
     pub async fn from_addr(context: &Context, addr: &str) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint \
+                     verified_key, verified_key_fingerprint, verifier \
                      FROM acpeerstates \
                      WHERE addr=? COLLATE NOCASE LIMIT 1;";
         Self::from_stmt(context, query, paramsv![addr]).await
@@ -145,7 +150,7 @@ impl Peerstate {
     ) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint \
+                     verified_key, verified_key_fingerprint, verifier \
                      FROM acpeerstates  \
                      WHERE public_key_fingerprint=? \
                      OR gossip_key_fingerprint=? \
@@ -161,7 +166,7 @@ impl Peerstate {
     ) -> Result<Option<Peerstate>> {
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
-                     verified_key, verified_key_fingerprint \
+                     verified_key, verified_key_fingerprint, verifier \
                      FROM acpeerstates  \
                      WHERE verified_key_fingerprint=? \
                      OR addr=? COLLATE NOCASE \
@@ -218,6 +223,7 @@ impl Peerstate {
                         .transpose()
                         .unwrap_or_default(),
                     fingerprint_changed: false,
+                    verifier: row.get("verifier")?,
                 };
 
                 Ok(res)
@@ -357,11 +363,19 @@ impl Peerstate {
         }
     }
 
+    /// Set this peerstate to verified
+    /// Make sure to call `self.save_to_db` to save these changes
+    /// Params:
+    /// verifier:
+    ///   The address which verifies the given contact
+    ///   If we are verifying the contact, use that contacts address
+    /// Returns whether the value of the key has changed
     pub fn set_verified(
         &mut self,
         which_key: PeerstateKeyType,
         fingerprint: &Fingerprint,
         verified: PeerstateVerifiedStatus,
+        verifier: String,
     ) -> bool {
         if verified == PeerstateVerifiedStatus::BidirectVerified {
             match which_key {
@@ -371,6 +385,7 @@ impl Peerstate {
                     {
                         self.verified_key = self.public_key.clone();
                         self.verified_key_fingerprint = self.public_key_fingerprint.clone();
+                        self.verifier = Some(verifier);
                         true
                     } else {
                         false
@@ -382,6 +397,7 @@ impl Peerstate {
                     {
                         self.verified_key = self.gossip_key.clone();
                         self.verified_key_fingerprint = self.gossip_key_fingerprint.clone();
+                        self.verifier = Some(verifier);
                         true
                     } else {
                         false
@@ -406,8 +422,9 @@ impl Peerstate {
                 gossip_key_fingerprint,
                 verified_key,
                 verified_key_fingerprint,
-                addr)
-                VALUES (?,?,?,?,?,?,?,?,?,?,?)
+                addr,
+                verifier)
+                VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
                 ON CONFLICT (addr)
                 DO UPDATE SET
                   last_seen = excluded.last_seen,
@@ -419,7 +436,8 @@ impl Peerstate {
                   public_key_fingerprint = excluded.public_key_fingerprint,
                   gossip_key_fingerprint = excluded.gossip_key_fingerprint,
                   verified_key = excluded.verified_key,
-                  verified_key_fingerprint = excluded.verified_key_fingerprint",
+                  verified_key_fingerprint = excluded.verified_key_fingerprint,
+                  verifier = excluded.verifier",
             paramsv![
                 self.last_seen,
                 self.last_seen_autocrypt,
@@ -432,6 +450,7 @@ impl Peerstate {
                 self.verified_key.as_ref().map(|k| k.to_bytes()),
                 self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()),
                 self.addr,
+                self.verifier,
             ],
         )
         .await?;
@@ -446,6 +465,11 @@ impl Peerstate {
         }
     }
 
+    /// Returns the address that verified the contact
+    pub fn get_verifier(&self) -> Option<&str> {
+        self.verifier.as_deref()
+    }
+
     /// Add an info message to all the chats with this contact, informing about
     /// a [`PeerstateChange`].
     ///
@@ -565,10 +589,10 @@ impl Peerstate {
 /// In `drafts/aeap_mvp.md` there is a "big picture" overview over AEAP.
 pub async fn maybe_do_aeap_transition(
     context: &Context,
-    info: &mut DecryptionInfo,
-    mime_parser: &crate::mimeparser::MimeMessage,
+    mime_parser: &mut crate::mimeparser::MimeMessage,
 ) -> Result<()> {
-    if let Some(peerstate) = &mut info.peerstate {
+    let info = &mime_parser.decryption_info;
+    if let Some(peerstate) = &info.peerstate {
         // If the from addr is different from the peerstate address we know,
         // we may want to do an AEAP transition.
         if !addr_cmp(&peerstate.addr, &mime_parser.from.addr)
@@ -588,6 +612,8 @@ pub async fn maybe_do_aeap_transition(
                 && mime_parser.from_is_signed
                 && info.message_time > peerstate.last_seen
         {
+            let info = &mut mime_parser.decryption_info;
+            let peerstate = info.peerstate.as_mut().context("no peerstate??")?;
             // Add info messages to chats with this (verified) contact
             //
             peerstate
@@ -669,6 +695,7 @@ mod tests {
             verified_key: Some(pub_key.clone()),
             verified_key_fingerprint: Some(pub_key.fingerprint()),
             fingerprint_changed: false,
+            verifier: None,
         };
 
         assert!(
@@ -708,6 +735,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             fingerprint_changed: false,
+            verifier: None,
         };
 
         assert!(
@@ -740,6 +768,7 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             fingerprint_changed: false,
+            verifier: None,
         };
 
         assert!(
@@ -802,8 +831,8 @@ mod tests {
             verified_key: None,
             verified_key_fingerprint: None,
             fingerprint_changed: false,
+            verifier: None,
         };
-        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::NoPreference);
 
         peerstate.apply_header(&header, 100);
         assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);

src/pgp.rs

@@ -1,5 +1,7 @@
 //! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp).
 
+#![allow(missing_docs)]
+
 use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
@@ -263,23 +265,20 @@ pub async fn pk_encrypt(
 /// of all keys from the `public_keys_for_validation` keyring that
 /// have valid signatures there.
 #[allow(clippy::implicit_hasher)]
-pub async fn pk_decrypt(
+pub fn pk_decrypt(
     ctext: Vec<u8>,
-    private_keys_for_decryption: Keyring<SignedSecretKey>,
+    private_keys_for_decryption: &Keyring<SignedSecretKey>,
     public_keys_for_validation: &Keyring<SignedPublicKey>,
 ) -> Result<(Vec<u8>, HashSet<Fingerprint>)> {
     let mut ret_signature_fingerprints: HashSet<Fingerprint> = Default::default();
 
-    let msgs = tokio::task::spawn_blocking(move || {
-        let cursor = Cursor::new(ctext);
-        let (msg, _) = Message::from_armor_single(cursor)?;
+    let cursor = Cursor::new(ctext);
+    let (msg, _) = Message::from_armor_single(cursor)?;
 
-        let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.keys().iter().collect();
+    let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption.keys().iter().collect();
 
-        let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
-        decryptor.collect::<pgp::errors::Result<Vec<_>>>()
-    })
-    .await??;
+    let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
+    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
 
     if let Some(msg) = msgs.into_iter().next() {
         // get_content() will decompress the message if needed,
@@ -512,10 +511,9 @@ mod tests {
         sig_check_keyring.add(KEYS.alice_public.clone());
         let (plain, valid_signatures) = pk_decrypt(
             ctext_signed().await.as_bytes().to_vec(),
-            decrypt_keyring,
+            &decrypt_keyring,
             &sig_check_keyring,
         )
-        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -527,10 +525,9 @@ mod tests {
         sig_check_keyring.add(KEYS.alice_public.clone());
         let (plain, valid_signatures) = pk_decrypt(
             ctext_signed().await.as_bytes().to_vec(),
-            decrypt_keyring,
+            &decrypt_keyring,
             &sig_check_keyring,
         )
-        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -543,10 +540,9 @@ mod tests {
         let empty_keyring = Keyring::new();
         let (plain, valid_signatures) = pk_decrypt(
             ctext_signed().await.as_bytes().to_vec(),
-            keyring,
+            &keyring,
             &empty_keyring,
         )
-        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
@@ -561,10 +557,9 @@ mod tests {
         sig_check_keyring.add(KEYS.bob_public.clone());
         let (plain, valid_signatures) = pk_decrypt(
             ctext_signed().await.as_bytes().to_vec(),
-            decrypt_keyring,
+            &decrypt_keyring,
             &sig_check_keyring,
         )
-        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);
@@ -577,10 +572,9 @@ mod tests {
         let sig_check_keyring = Keyring::new();
         let (plain, valid_signatures) = pk_decrypt(
             ctext_unsigned().await.as_bytes().to_vec(),
-            decrypt_keyring,
+            &decrypt_keyring,
             &sig_check_keyring,
         )
-        .await
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 0);

src/plaintext.rs

@@ -1,5 +1,7 @@
 //! Handle plain text together with some attributes.
 
+#![allow(missing_docs)]
+
 use crate::simplify::split_lines;
 use once_cell::sync::Lazy;