feat(sql): truncate WAL on stop_io()

2026-07-09 02:45:02 +03:00 · 2023-12-05 00:46:43 +00:00
31 changed files with 302 additions and 525 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
    name: Lint Rust
    runs-on: ubuntu-latest
    env:
-      RUSTUP_TOOLCHAIN: 1.74.1
+      RUSTUP_TOOLCHAIN: 1.74.0
    steps:
      - uses: actions/checkout@v3
      - name: Install rustfmt and clippy
@@ -76,11 +76,11 @@ jobs:
      matrix:
        include:
          - os: ubuntu-latest
-            rust: 1.74.1
+            rust: 1.74.0
          - os: windows-latest
-            rust: 1.74.1
+            rust: 1.74.0
          - os: macos-latest
-            rust: 1.74.1
+            rust: 1.74.0

          # Minimum Supported Rust Version = 1.70.0
          - os: ubuntu-latest
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,38 +1,5 @@
 # Changelog

-## [1.132.1] - 2023-12-12
-
-### Features / Changes
-
- Add "From:" to protected headers for signed-only messages.
- Sync user actions for ad-hoc groups across devices ([#5065](https://github.com/deltachat/deltachat-core-rust/pull/5065)).
-
-### Fixes
-
- Add padlock to empty part if the whole message is empty.
- Renew IDLE timeout on keepalives and reduce it to 5 minutes.
- connectivity: Return false from `all_work_done()` immediately after connecting (iOS notification fix).
-
-### API-Changes
-
- deltachat-jsonrpc-client: add `Account.{import,export}_self_keys`.
-
-### CI
-
- Update to Rust 1.74.1.
-
-## [1.132.0] - 2023-12-06
-
-### Features / Changes
-
- Increase TCP timeouts from 30 to 60 seconds.
-
-### Fixes
-
- Don't sort message creating a protected group over a protection message ([#4963](https://github.com/deltachat/deltachat-core-rust/pull/4963)).
- Do not lock accounts.toml on iOS.
- Protect groups even if some members are not verified and add `test_securejoin_after_contact_resetup` regression test.
-
 ## [1.131.9] - 2023-12-02

 ### API-Changes
@@ -3355,5 +3322,3 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.131.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.6...v1.131.7
 [1.131.8]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.7...v1.131.8
 [1.131.9]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.8...v1.131.9
-[1.132.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.9...v1.132.0
-[1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -224,9 +224,9 @@ dependencies = [

 [[package]]
 name = "async-imap"
-version = "0.9.5"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9d69fc1499878158750f644c4eb46aff55bb9d32d77e3dc4aecf8308d5c3ba6"
+checksum = "d736a74edf6c327b53dd9c932eae834253470ac5f0c55770e7e133bcbf986362"
 dependencies = [
 "async-channel 2.1.1",
 "base64 0.21.5",
@@ -315,9 +315,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

 [[package]]
 name = "axum"
-version = "0.7.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "202651474fe73c62d9e0a56c6133f7a0ff1dc1c8cf7a5b03381af2a26553ac9d"
+checksum = "810a80b128d70e6ed2bdf3fe8ed72c0ae56f5f5948d01c2753282dd92a84fce8"
 dependencies = [
 "async-trait",
 "axum-core",
@@ -351,9 +351,9 @@ dependencies = [

 [[package]]
 name = "axum-core"
-version = "0.4.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77cb22c689c44d4c07b0ab44ebc25d69d8ae601a2f28fb8d672d344178fa17aa"
+checksum = "de0ddc355eab88f4955090a823715df47acf0b7660aab7a69ad5ce6301ee3b73"
 dependencies = [
 "async-trait",
 "bytes",
@@ -711,18 +711,18 @@ dependencies = [

 [[package]]
 name = "clap"
-version = "4.4.11"
+version = "4.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2"
+checksum = "41fffed7514f420abec6d183b1d3acfd9099c79c3a10a06ade4f8203f1411272"
 dependencies = [
 "clap_builder",
 ]

 [[package]]
 name = "clap_builder"
-version = "4.4.11"
+version = "4.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb"
+checksum = "63361bae7eef3771745f02d8d892bec2fee5f6e34af316ba556e7f97a7069ff1"
 dependencies = [
 "anstyle",
 "clap_lex",
@@ -759,9 +759,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"

 [[package]]
 name = "concurrent-queue"
-version = "2.4.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363"
+checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400"
 dependencies = [
 "crossbeam-utils",
 ]
@@ -1091,7 +1091,7 @@ dependencies = [

 [[package]]
 name = "deltachat"
-version = "1.132.1"
+version = "1.131.9"
 dependencies = [
 "ansi_term",
 "anyhow",
@@ -1169,7 +1169,7 @@ dependencies = [

 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.132.1"
+version = "1.131.9"
 dependencies = [
 "anyhow",
 "async-channel 2.1.1",
@@ -1193,7 +1193,7 @@ dependencies = [

 [[package]]
 name = "deltachat-repl"
-version = "1.132.1"
+version = "1.131.9"
 dependencies = [
 "ansi_term",
 "anyhow",
@@ -1208,7 +1208,7 @@ dependencies = [

 [[package]]
 name = "deltachat-rpc-server"
-version = "1.132.1"
+version = "1.131.9"
 dependencies = [
 "anyhow",
 "deltachat",
@@ -1233,7 +1233,7 @@ dependencies = [

 [[package]]
 name = "deltachat_ffi"
-version = "1.132.1"
+version = "1.131.9"
 dependencies = [
 "anyhow",
 "deltachat",
@@ -1300,9 +1300,9 @@ dependencies = [

 [[package]]
 name = "deranged"
-version = "0.3.10"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
+checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
 dependencies = [
 "powerfmt",
 ]
@@ -1921,14 +1921,14 @@ checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7"

 [[package]]
 name = "filetime"
-version = "0.2.23"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
+checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0"
 dependencies = [
 "cfg-if",
 "libc",
- "redox_syscall 0.4.1",
- "windows-sys 0.52.0",
+ "redox_syscall 0.3.5",
+ "windows-sys 0.48.0",
 ]

 [[package]]
@@ -2038,13 +2038,14 @@ checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"

 [[package]]
 name = "futures-lite"
-version = "2.1.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aeee267a1883f7ebef3700f262d2d54de95dfaf38189015a74fdc4e0c7ad8143"
+checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb"
 dependencies = [
 "fastrand",
 "futures-core",
 "futures-io",
+ "memchr",
 "parking",
 "pin-project-lite",
 ]
@@ -2913,9 +2914,9 @@ dependencies = [

 [[package]]
 name = "mio"
-version = "0.8.10"
+version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
+checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0"
 dependencies = [
 "libc",
 "wasi 0.11.0+wasi-snapshot-preview1",
@@ -3183,9 +3184,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

 [[package]]
 name = "openssl"
-version = "0.10.61"
+version = "0.10.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45"
+checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800"
 dependencies = [
 "bitflags 2.4.1",
 "cfg-if",
@@ -3224,9 +3225,9 @@ dependencies = [

 [[package]]
 name = "openssl-sys"
-version = "0.9.97"
+version = "0.9.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b"
+checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f"
 dependencies = [
 "cc",
 "libc",
@@ -3554,9 +3555,9 @@ dependencies = [

 [[package]]
 name = "portable-atomic"
-version = "1.6.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
+checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b"

 [[package]]
 name = "postcard"
@@ -4060,9 +4061,9 @@ dependencies = [

 [[package]]
 name = "ring"
-version = "0.17.7"
+version = "0.17.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74"
+checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866"
 dependencies = [
 "cc",
 "getrandom 0.2.11",
@@ -4191,7 +4192,7 @@ version = "0.21.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9"
 dependencies = [
- "ring 0.17.7",
+ "ring 0.17.6",
 "rustls-webpki",
 "sct",
 ]
@@ -4223,7 +4224,7 @@ version = "0.101.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
 dependencies = [
- "ring 0.17.7",
+ "ring 0.17.6",
 "untrusted 0.9.0",
 ]

@@ -4332,7 +4333,7 @@ version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
 dependencies = [
- "ring 0.17.7",
+ "ring 0.17.6",
 "untrusted 0.9.0",
 ]

@@ -5751,9 +5752,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"

 [[package]]
 name = "winnow"
-version = "0.5.25"
+version = "0.5.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7e87b8dfbe3baffbe687eef2e164e32286eff31a5ee16463ce03d991643ec94"
+checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b"
 dependencies = [
 "memchr",
 ]
@@ -5847,9 +5848,9 @@ dependencies = [

 [[package]]
 name = "yerpc_derive"
-version = "0.5.3"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e510aa045bc7be964b982c68f001933fce4fbe609bb98de60068fa8cefe6308"
+checksum = "f321bb5f728fb066af06c5a994e4375f1f8b054ee6d650766f0bd68dfa4faefe"
 dependencies = [
 "convert_case 0.5.0",
 "darling 0.14.4",
@@ -5860,18 +5861,18 @@ dependencies = [

 [[package]]
 name = "zerocopy"
-version = "0.7.29"
+version = "0.7.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e"
+checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e"
 dependencies = [
 "zerocopy-derive",
 ]

 [[package]]
 name = "zerocopy-derive"
-version = "0.7.29"
+version = "0.7.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2"
+checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b"
 dependencies = [
 "proc-macro2",
 "quote",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.132.1"
+version = "1.131.9"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.70"
@@ -40,7 +40,7 @@ ratelimit = { path = "./deltachat-ratelimit" }

 anyhow = "1"
 async-channel = "2.0.0"
-async-imap = { version = "0.9.5", default-features = false, features = ["runtime-tokio"] }
+async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
--- a/deltachat-ffi/Cargo.toml
+++ b/deltachat-ffi/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.132.1"
+version = "1.131.9"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
--- a/deltachat-jsonrpc/Cargo.toml
+++ b/deltachat-jsonrpc/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.132.1"
+version = "1.131.9"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
--- a/deltachat-jsonrpc/typescript/package.json
+++ b/deltachat-jsonrpc/typescript/package.json
@@ -53,5 +53,5 @@
  },
  "type": "module",
  "types": "dist/deltachat.d.ts",
-  "version": "1.132.1"
+  "version": "1.131.9"
 }
--- a/deltachat-repl/Cargo.toml
+++ b/deltachat-repl/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.132.1"
+version = "1.131.9"
 license = "MPL-2.0"
 edition = "2021"

--- a/deltachat-rpc-client/src/deltachat_rpc_client/account.py
+++ b/deltachat-rpc-client/src/deltachat_rpc_client/account.py
@@ -300,13 +300,3 @@ class Account:
    def import_backup(self, path, passphrase: str = "") -> None:
        """Import backup."""
        self._rpc.import_backup(self.id, str(path), passphrase)
-
-    def export_self_keys(self, path) -> None:
-        """Export keys."""
-        passphrase = ""  # Setting passphrase is currently not supported.
-        self._rpc.export_self_keys(self.id, str(path), passphrase)
-
-    def import_self_keys(self, path) -> None:
-        """Import keys."""
-        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
-        self._rpc.import_self_keys(self.id, str(path), passphrase)
--- a/deltachat-rpc-client/tests/test_securejoin.py
+++ b/deltachat-rpc-client/tests/test_securejoin.py
@@ -477,76 +477,3 @@ def test_gossip_verification(acfactory) -> None:
    # Securejoin propagates verification.
    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
    assert carol_contact_alice_snapshot.is_verified
-
-
-def test_securejoin_after_contact_resetup(acfactory) -> None:
-    """
-    Regression test for a bug that prevented joining verified group with a QR code
-    if the group is already created and contains
-    a contact with inconsistent (Autocrypt and verified keys exist but don't match) key state.
-    """
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    # ac3 creates protected group with ac1.
-    ac3_chat = ac3.create_group("Verified group", protect=True)
-
-    # ac1 joins ac3 group.
-    ac3_qr_code, _svg = ac3_chat.get_qr_code()
-    ac1.secure_join(ac3_qr_code)
-    ac1.wait_for_securejoin_joiner_success()
-
-    # ac1 waits for member added message and creates a QR code.
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
-
-    # ac2 verifies ac1
-    qr_code, _svg = ac1.get_qr_code()
-    ac2.secure_join(qr_code)
-    ac2.wait_for_securejoin_joiner_success()
-
-    # ac1 is verified for ac2.
-    ac2_contact_ac1 = ac2.create_contact(ac1.get_config("addr"), "")
-    assert ac2_contact_ac1.get_snapshot().is_verified
-
-    # ac1 resetups the account.
-    ac1 = acfactory.resetup_account(ac1)
-
-    # ac1 sends a message to ac2.
-    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
-    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
-    ac1_chat_ac2.send_text("Hello!")
-
-    # ac2 receives a message.
-    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Hello!"
-
-    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
-    assert not ac2_contact_ac1.get_snapshot().is_verified
-
-    # ac1 goes offline.
-    ac1.remove()
-
-    # Scanning a QR code results in creating an unprotected group with an inviter.
-    # In this case inviter is ac1 which has an inconsistent key state.
-    # Normally inviter becomes verified as a result of Securejoin protocol
-    # and then the group chat becomes verified when "Member added" is received,
-    # but in this case ac1 is offline and this Securejoin process will never finish.
-    logging.info("ac2 scans ac1 QR code, this is not expected to finish")
-    ac2.secure_join(ac1_qr_code)
-
-    logging.info("ac2 scans ac3 QR code")
-    ac2.secure_join(ac3_qr_code)
-
-    logging.info("ac2 waits for joiner success")
-    ac2.wait_for_securejoin_joiner_success()
-
-    # Wait for member added.
-    logging.info("ac2 waits for member added message")
-    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.is_info
-    ac2_chat = snapshot.chat
-    assert ac2_chat.get_basic_snapshot().is_protected
-    assert len(ac2_chat.get_contacts()) == 3
-
-    # ac1 is still "not verified" for ac2 due to inconsistent state.
-    assert not ac2_contact_ac1.get_snapshot().is_verified
--- a/deltachat-rpc-client/tests/test_something.py
+++ b/deltachat-rpc-client/tests/test_something.py
@@ -331,7 +331,7 @@ def test_wait_next_messages(acfactory) -> None:
        next_messages_task = executor.submit(bot.wait_next_messages)

        bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
        alice_chat_bot = alice_contact_bot.create_chat()
        alice_chat_bot.send_text("Hello!")

@@ -341,7 +341,7 @@ def test_wait_next_messages(acfactory) -> None:
        assert snapshot.text == "Hello!"


-def test_import_export_backup(acfactory, tmp_path) -> None:
+def test_import_export(acfactory, tmp_path) -> None:
    alice = acfactory.new_configured_account()
    alice.export_backup(tmp_path)

@@ -352,31 +352,6 @@ def test_import_export_backup(acfactory, tmp_path) -> None:
    assert alice2.manager.get_system_info()


-def test_import_export_keys(acfactory, tmp_path) -> None:
-    alice, bob = acfactory.get_online_accounts(2)
-
-    bob_addr = bob.get_config("addr")
-    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
-    alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_text("Hello Bob!")
-
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Hello Bob!"
-
-    # Alice resetups account, but keeps the key.
-    alice_keys_path = tmp_path / "alice_keys"
-    alice_keys_path.mkdir()
-    alice.export_self_keys(alice_keys_path)
-    alice = acfactory.resetup_account(alice)
-    alice.import_self_keys(alice_keys_path)
-
-    snapshot.chat.accept()
-    snapshot.chat.send_text("Hello Alice!")
-    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Hello Alice!"
-    assert snapshot.show_padlock
-
-
 def test_openrpc_command_line() -> None:
    """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
    out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
--- a/deltachat-rpc-server/Cargo.toml
+++ b/deltachat-rpc-server/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.132.1"
+version = "1.131.9"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
--- a/package.json
+++ b/package.json
@@ -56,5 +56,5 @@
    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
  },
  "types": "node/dist/index.d.ts",
-  "version": "1.132.1"
+  "version": "1.131.9"
 }
--- a/python/tests/test_1_online.py
+++ b/python/tests/test_1_online.py
@@ -1979,32 +1979,6 @@ def test_connectivity(acfactory, lp):
    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)


-def test_all_work_done(acfactory, lp):
-    """
-    Tests that calling start_io() immediately followed by maybe_network()
-    and then waiting for all_work_done() reliably fetches the messages
-    delivered while account was offline.
-    In other words, connectivity should not change to a state
-    where all_work_done() returns true until IMAP connection goes idle.
-    """
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    ac1.stop_io()
-    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
-
-    ac1.direct_imap.select_config_folder("inbox")
-    with ac1.direct_imap.idle() as idle1:
-        ac2.create_chat(ac1).send_text("Hi")
-        idle1.wait_for_new_message()
-
-    ac1.start_io()
-    ac1.maybe_network()
-    ac1._evtracker.wait_for_all_work_done()
-    msgs = ac1.create_chat(ac2).get_messages()
-    assert len(msgs) == 1
-    assert msgs[0].text == "Hi"
-
-
 def test_fetch_deleted_msg(acfactory, lp):
    """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
    hundreds of times, because uid_next was not updated.
--- a/release-date.in
+++ b/release-date.in
@@ -1 +1 @@
-2023-12-12
+2023-12-02
--- a/src/accounts.rs
+++ b/src/accounts.rs
@@ -424,13 +424,11 @@ impl Config {
    /// Takes a mutable reference because the saved file is a part of the `Config` state. This
    /// protects from parallel calls resulting to a wrong file contents.
    async fn sync(&mut self) -> Result<()> {
-        #[cfg(not(target_os = "ios"))]
        ensure!(!self
            .lock_task
            .as_ref()
            .context("Config is read-only")?
            .is_finished());
-
        let tmp_path = self.file.with_extension("toml.tmp");
        let mut file = fs::File::create(&tmp_path)
            .await
--- a/src/chat.rs
+++ b/src/chat.rs
@@ -209,30 +209,6 @@ impl ChatId {
        self == DC_CHAT_ID_ALLDONE_HINT
    }

-    /// Returns [`ChatId`] of a chat that `msg` belongs to.
-    ///
-    /// Checks that `msg` is assigned to the right chat.
-    pub(crate) fn lookup_by_message(msg: &Message) -> Option<Self> {
-        if msg.chat_id == DC_CHAT_ID_TRASH {
-            return None;
-        }
-        if msg.download_state != DownloadState::Done
-            // TODO (2023-09-12): Added for backward compatibility with versions that did not have
-            // `DownloadState::Undecipherable`. Remove eventually with the comment in
-            // `MimeMessage::from_bytes()`.
-            || msg
-                .error
-                .as_ref()
-                .filter(|e| e.starts_with("Decrypting failed:"))
-                .is_some()
-        {
-            // If `msg` is not fully downloaded or undecipherable, it may have been assigned to the
-            // wrong chat (they often get assigned to the 1:1 chat with the sender).
-            return None;
-        }
-        Some(msg.chat_id)
-    }
-
    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id`
    /// if it exists and is not blocked.
    ///
@@ -351,7 +327,7 @@ impl ChatId {

        info!(
            context,
-            "Created group/mailinglist '{}' grpid={} as {}, blocked={}, protected={create_protected}.",
+            "Created group/mailinglist '{}' grpid={} as {}, blocked={}.",
            &grpname,
            grpid,
            chat_id,
@@ -398,7 +374,6 @@ impl ChatId {

    pub(crate) async fn block_ex(self, context: &Context, sync: sync::Sync) -> Result<()> {
        let chat = Chat::load_from_db(context, self).await?;
-        let mut delete = false;

        match chat.typ {
            Chattype::Broadcast => {
@@ -417,7 +392,7 @@ impl ChatId {
            }
            Chattype::Group => {
                info!(context, "Can't block groups yet, deleting the chat.");
-                delete = true;
+                self.delete(context).await?;
            }
            Chattype::Mailinglist => {
                if self.set_blocked(context, Blocked::Yes).await? {
@@ -433,9 +408,6 @@ impl ChatId {
                .log_err(context)
                .ok();
        }
-        if delete {
-            self.delete(context).await?;
-        }
        Ok(())
    }

@@ -530,7 +502,15 @@ impl ChatId {

        match protect {
            ProtectionStatus::Protected => match chat.typ {
-                Chattype::Single | Chattype::Group | Chattype::Broadcast => {}
+                Chattype::Single | Chattype::Group | Chattype::Broadcast => {
+                    let contact_ids = get_chat_contacts(context, self).await?;
+                    for contact_id in contact_ids {
+                        let contact = Contact::get_by_id(context, contact_id).await?;
+                        if !contact.is_verified(context).await? {
+                            bail!("{} is not verified.", contact.get_display_name());
+                        }
+                    }
+                }
                Chattype::Mailinglist => bail!("Cannot protect mailing lists"),
            },
            ProtectionStatus::Unprotected | ProtectionStatus::ProtectionBroken => {}
@@ -1152,46 +1132,47 @@ impl ChatId {
        Ok(self.get_param(context).await?.exists(Param::Devicetalk))
    }

-    async fn parent_query<T, F>(
-        self,
-        context: &Context,
-        fields: &str,
-        state_out_min: MessageState,
-        f: F,
-    ) -> Result<Option<T>>
+    async fn parent_query<T, F>(self, context: &Context, fields: &str, f: F) -> Result<Option<T>>
    where
        F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
        T: Send + 'static,
    {
        let sql = &context.sql;
+        // Do not reply to not fully downloaded messages. Such a message could be a group chat
+        // message that we assigned to 1:1 chat.
        let query = format!(
            "SELECT {fields} \
-             FROM msgs \
-             WHERE chat_id=? \
-             AND ((state BETWEEN {} AND {}) OR (state >= {})) \
-             AND NOT hidden \
-             AND download_state={} \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden AND download_state={} \
             ORDER BY timestamp DESC, id DESC \
             LIMIT 1;",
-            MessageState::InFresh as u32,
-            MessageState::InSeen as u32,
-            state_out_min as u32,
-            // Do not reply to not fully downloaded messages. Such a message could be a group chat
-            // message that we assigned to 1:1 chat.
-            DownloadState::Done as u32,
+             DownloadState::Done as u32,
        );
-        sql.query_row_optional(&query, (self,), f).await
+        let row = sql
+            .query_row_optional(
+                &query,
+                (
+                    self,
+                    MessageState::OutPreparing,
+                    MessageState::OutDraft,
+                    // We don't filter `OutPending` and `OutFailed` messages because the new message
+                    // for which `parent_query()` is done may assume that it will be received in a
+                    // context affected by those messages, e.g. they could add new members to a
+                    // group and the new message will contain them in "To:". Anyway recipients must
+                    // be prepared to orphaned references.
+                ),
+                f,
+            )
+            .await?;
+        Ok(row)
    }

    async fn get_parent_mime_headers(
        self,
        context: &Context,
-        state_out_min: MessageState,
    ) -> Result<Option<(String, String, String)>> {
        self.parent_query(
            context,
            "rfc724_mid, mime_in_reply_to, mime_references",
-            state_out_min,
            |row: &rusqlite::Row| {
                let rfc724_mid: String = row.get(0)?;
                let mime_in_reply_to: String = row.get(1)?;
@@ -1768,15 +1749,7 @@ impl Chat {
        // we do not set In-Reply-To/References in this case.
        if !self.is_self_talk() {
            if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                // We don't filter `OutPending` and `OutFailed` messages because the new message for
-                // which `parent_query()` is done may assume that it will be received in a context
-                // affected by those messages, e.g. they could add new members to a group and the
-                // new message will contain them in "To:". Anyway recipients must be prepared to
-                // orphaned references.
-                self
-                    .id
-                    .get_parent_mime_headers(context, MessageState::OutPending)
-                    .await?
+                self.id.get_parent_mime_headers(context).await?
            {
                // "In-Reply-To:" is not changed if it is set manually.
                // This does not affect "References:" header, it will contain "default parent" (the
@@ -1994,26 +1967,10 @@ impl Chat {
                Ok(r)
            }
            Chattype::Broadcast | Chattype::Group | Chattype::Mailinglist => {
-                if !self.grpid.is_empty() {
-                    return Ok(Some(SyncId::Grpid(self.grpid.clone())));
-                }
-
-                let Some((parent_rfc724_mid, parent_in_reply_to, _)) = self
-                    .id
-                    .get_parent_mime_headers(context, MessageState::OutDelivered)
-                    .await?
-                else {
-                    warn!(
-                        context,
-                        "Chat::get_sync_id({}): No good message identifying the chat found.",
-                        self.id
-                    );
+                if self.grpid.is_empty() {
                    return Ok(None);
-                };
-                Ok(Some(SyncId::Msgids(vec![
-                    parent_in_reply_to,
-                    parent_rfc724_mid,
-                ])))
+                }
+                Ok(Some(SyncId::Grpid(self.grpid.clone())))
            }
        }
    }
@@ -4293,8 +4250,8 @@ async fn set_contacts_by_addrs(context: &Context, id: ChatId, addrs: &[String])
 pub(crate) enum SyncId {
    ContactAddr(String),
    Grpid(String),
-    /// "Message-ID"-s, from oldest to latest. Used for ad-hoc groups.
-    Msgids(Vec<String>),
+    // NOTE: Ad-hoc groups lack an identifier that can be used across devices so
+    // block/mute/etc. actions on them are not synchronized to other devices.
 }

 /// An action synchronised to other devices.
@@ -4317,9 +4274,12 @@ impl Context {
    pub(crate) async fn sync_alter_chat(&self, id: &SyncId, action: &SyncAction) -> Result<()> {
        let chat_id = match id {
            SyncId::ContactAddr(addr) => {
-                let contact_id = Contact::lookup_id_by_addr_ex(self, addr, Origin::Unknown, None)
-                    .await?
-                    .with_context(|| format!("No contact for addr '{addr}'"))?;
+                let Some(contact_id) =
+                    Contact::lookup_id_by_addr_ex(self, addr, Origin::Unknown, None).await?
+                else {
+                    warn!(self, "sync_alter_chat: No contact for addr '{addr}'.");
+                    return Ok(());
+                };
                match action {
                    SyncAction::Block => {
                        return contact::set_blocked(self, Nosync, contact_id, true).await
@@ -4329,26 +4289,22 @@ impl Context {
                    }
                    _ => (),
                }
-                ChatId::lookup_by_contact(self, contact_id)
-                    .await?
-                    .with_context(|| format!("No chat for addr '{addr}'"))?
+                let Some(chat_id) = ChatId::lookup_by_contact(self, contact_id).await? else {
+                    warn!(self, "sync_alter_chat: No chat for addr '{addr}'.");
+                    return Ok(());
+                };
+                chat_id
            }
            SyncId::Grpid(grpid) => {
                if let SyncAction::CreateBroadcast(name) = action {
                    create_broadcast_list_ex(self, Nosync, grpid.clone(), name.clone()).await?;
                    return Ok(());
                }
-                get_chat_id_by_grpid(self, grpid)
-                    .await?
-                    .with_context(|| format!("No chat for grpid '{grpid}'"))?
-                    .0
-            }
-            SyncId::Msgids(msgids) => {
-                let msg = message::get_latest_by_rfc724_mids(self, msgids)
-                    .await?
-                    .with_context(|| format!("No message found for Message-IDs {msgids:?}"))?;
-                ChatId::lookup_by_message(&msg)
-                    .with_context(|| format!("No chat found for Message-IDs {msgids:?}"))?
+                let Some((chat_id, ..)) = get_chat_id_by_grpid(self, grpid).await? else {
+                    warn!(self, "sync_alter_chat: No chat for grpid '{grpid}'.");
+                    return Ok(());
+                };
+                chat_id
            }
        };
        match action {
@@ -6993,51 +6949,6 @@ mod tests {
        Ok(())
    }

-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_sync_adhoc_grp() -> Result<()> {
-        let alice0 = &TestContext::new_alice().await;
-        let alice1 = &TestContext::new_alice().await;
-        for a in [alice0, alice1] {
-            a.set_config_bool(Config::SyncMsgs, true).await?;
-        }
-
-        let mut chat_ids = Vec::new();
-        for a in [alice0, alice1] {
-            let msg = receive_imf(
-                a,
-                b"Subject: =?utf-8?q?Message_from_alice=40example=2Eorg?=\r\n\
-                    From: alice@example.org\r\n\
-                    To: <bob@example.net>, <fiona@example.org> \r\n\
-                    Date: Mon, 2 Dec 2023 16:59:39 +0000\r\n\
-                    Message-ID: <Mr.alices_original_mail@example.org>\r\n\
-                    Chat-Version: 1.0\r\n\
-                    \r\n\
-                    hi\r\n",
-                false,
-            )
-            .await?
-            .unwrap();
-            chat_ids.push(msg.chat_id);
-        }
-        let chat1 = Chat::load_from_db(alice1, chat_ids[1]).await?;
-        assert_eq!(chat1.typ, Chattype::Group);
-        assert!(chat1.grpid.is_empty());
-
-        // Test synchronisation on chat blocking because it causes chat deletion currently and thus
-        // requires generating a sync message in advance.
-        chat_ids[0].block(alice0).await?;
-        sync(alice0, alice1).await;
-        assert!(Chat::load_from_db(alice1, chat_ids[1]).await.is_err());
-        assert!(
-            !alice1
-                .sql
-                .exists("SELECT COUNT(*) FROM chats WHERE id=?", (chat_ids[1],))
-                .await?
-        );
-
-        Ok(())
-    }
-
    /// Tests syncing of chat visibility on a self-chat. This way we test:
    /// - Self-chat synchronisation.
    /// - That sync messages don't unarchive the self-chat.
--- a/src/contact.rs
+++ b/src/contact.rs
@@ -772,7 +772,7 @@ impl Contact {

                sth_modified = Modifier::Created;
                row_id = u32::try_from(transaction.last_insert_rowid())?;
-                info!(context, "Added contact id={row_id} addr={addr}.");
+                info!(context, "added contact id={} addr={}", row_id, &addr);
            }
            Ok(row_id)
        }).await?;
--- a/src/context.rs
+++ b/src/context.rs
@@ -423,6 +423,9 @@ impl Context {
    /// Stops the IO scheduler.
    pub async fn stop_io(&self) {
        self.scheduler.stop(self).await;
+        if let Err(err) = self.sql.checkpoint(self).await {
+            error!(self, "Failed to checkpoint the database: {err:#}.");
+        }
    }

    /// Restarts the IO scheduler if it was running before
--- a/src/decrypt.rs
+++ b/src/decrypt.rs
@@ -23,14 +23,32 @@ use crate::pgp;
 ///
 /// If the message is wrongly signed, HashSet will be empty.
 pub fn try_decrypt(
+    context: &Context,
    mail: &ParsedMail<'_>,
    private_keyring: &[SignedSecretKey],
    public_keyring_for_validate: &[SignedPublicKey],
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let encrypted_data_part = match get_autocrypt_mime(mail)
-        .or_else(|| get_mixed_up_mime(mail))
-        .or_else(|| get_attachment_mime(mail))
-    {
+    let encrypted_data_part = match {
+        let mime = get_autocrypt_mime(mail);
+        if mime.is_some() {
+            info!(context, "Detected Autocrypt-mime message.");
+        }
+        mime
+    }
+    .or_else(|| {
+        let mime = get_mixed_up_mime(mail);
+        if mime.is_some() {
+            info!(context, "Detected mixed-up mime message.");
+        }
+        mime
+    })
+    .or_else(|| {
+        let mime = get_attachment_mime(mail);
+        if mime.is_some() {
+            info!(context, "Detected attached Autocrypt-mime message.");
+        }
+        mime
+    }) {
        None => return Ok(None),
        Some(res) => res,
    };
--- a/src/e2ee.rs
+++ b/src/e2ee.rs
@@ -62,19 +62,21 @@ impl EncryptHelper {
        for (peerstate, addr) in peerstates {
            match peerstate {
                Some(peerstate) => {
-                    let prefer_encrypt = peerstate.prefer_encrypt;
-                    info!(context, "Peerstate for {addr:?} is {prefer_encrypt}.");
+                    info!(
+                        context,
+                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
+                    );
                    match peerstate.prefer_encrypt {
                        EncryptPreference::NoPreference | EncryptPreference::Reset => {}
                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
                    };
                }
                None => {
-                    let msg = format!("Peerstate for {addr:?} missing, cannot encrypt");
+                    let msg = format!("peerstate for {addr:?} missing, cannot encrypt");
                    if e2ee_guaranteed {
-                        return Err(format_err!("{msg}"));
+                        return Err(format_err!("{}", msg));
                    } else {
-                        info!(context, "{msg}.");
+                        info!(context, "{}", msg);
                        return Ok(false);
                    }
                }
@@ -169,10 +171,11 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::chat;
    use crate::key::DcKey;
    use crate::message::{Message, Viewtype};
    use crate::param::Param;
-    use crate::test_utils::{bob_keypair, TestContext, TestContextManager};
+    use crate::test_utils::{bob_keypair, TestContext};

    mod ensure_secret_key_exists {
        use super::*;
@@ -216,35 +219,37 @@ Sent with my Delta Chat Messenger: https://delta.chat";

    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
    async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;

        let chat_alice = alice.create_chat(&bob).await.id;
        let chat_bob = bob.create_chat(&alice).await.id;

        // Alice sends unencrypted message to Bob
        let mut msg = Message::new(Viewtype::Text);
-        let sent = alice.send_msg(chat_alice, &mut msg).await;
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;

        // Bob receives unencrypted message from Alice
-        let msg = bob.recv_msg(&sent).await;
-        assert!(!msg.get_showpadlock());
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());

+        // Parsing a message is enough to update peerstate
        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
            .await?
            .expect("no peerstate found in the database");
        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);

-        // Bob sends empty encrypted message to Alice
+        // Bob sends encrypted message to Alice
        let mut msg = Message::new(Viewtype::Text);
-        let sent = bob.send_msg(chat_bob, &mut msg).await;
+        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        let sent = bob.pop_sent_msg().await;

-        // Alice receives an empty encrypted message from Bob.
-        // This is also a regression test for previously existing bug
-        // that resulted in no padlock on encrypted empty messages.
-        let msg = alice.recv_msg(&sent).await;
-        assert!(msg.get_showpadlock());
+        // Alice receives encrypted message from Bob
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());

        let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
            .await?
@@ -256,10 +261,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
        // Alice sends encrypted message without Autocrypt header.
        let mut msg = Message::new(Viewtype::Text);
        msg.param.set_int(Param::SkipAutocrypt, 1);
-        let sent = alice.send_msg(chat_alice, &mut msg).await;
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;

-        let msg = bob.recv_msg(&sent).await;
-        assert!(msg.get_showpadlock());
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
            .await?
            .expect("no peerstate found in the database");
@@ -268,10 +275,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
        // Alice sends plaintext message with Autocrypt header.
        let mut msg = Message::new(Viewtype::Text);
        msg.force_plaintext();
-        let sent = alice.send_msg(chat_alice, &mut msg).await;
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;

-        let msg = bob.recv_msg(&sent).await;
-        assert!(!msg.get_showpadlock());
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
            .await?
            .expect("no peerstate found in the database");
@@ -281,10 +290,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
        let mut msg = Message::new(Viewtype::Text);
        msg.force_plaintext();
        msg.param.set_int(Param::SkipAutocrypt, 1);
-        let sent = alice.send_msg(chat_alice, &mut msg).await;
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;

-        let msg = bob.recv_msg(&sent).await;
-        assert!(!msg.get_showpadlock());
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
            .await?
            .expect("no peerstate found in the database");
--- a/src/imap/idle.rs
+++ b/src/imap/idle.rs
@@ -12,13 +12,7 @@ use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
 use crate::log::LogExt;

-/// Timeout after which IDLE is finished
-/// if there are no responses from the server.
-///
-/// If `* OK Still here` keepalives are sent more frequently
-/// than this duration, timeout should never be triggered.
-/// For example, Dovecot sends keepalives every 2 minutes by default.
-const IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
+const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);

 impl Session {
    pub async fn idle(
--- a/src/message.rs
+++ b/src/message.rs
@@ -1842,24 +1842,6 @@ pub(crate) async fn rfc724_mid_exists_and(
    Ok(res)
 }

-/// Given a list of Message-IDs, returns the latest message found in the database.
-///
-/// Only messages that are not in the trash chat are considered.
-pub(crate) async fn get_latest_by_rfc724_mids(
-    context: &Context,
-    mids: &[String],
-) -> Result<Option<Message>> {
-    for id in mids.iter().rev() {
-        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
-            let msg = Message::load_from_db(context, msg_id).await?;
-            if msg.chat_id != DC_CHAT_ID_TRASH {
-                return Ok(Some(msg));
-            }
-        }
-    }
-    Ok(None)
-}
-
 /// How a message is primarily displayed.
 #[derive(
    Debug,
--- a/src/mimefactory.rs
+++ b/src/mimefactory.rs
@@ -1,6 +1,5 @@
 //! # MIME message production.

-use std::collections::HashSet;
 use std::convert::TryInto;

 use anyhow::{bail, ensure, Context as _, Result};
@@ -518,7 +517,6 @@ impl<'a> MimeFactory<'a> {
        // <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.1.1>.
        let from_header = Header::new_with_value("From".into(), vec![from]).unwrap();
        headers.unprotected.push(from_header.clone());
-        headers.protected.push(from_header);

        if let Some(sender_displayname) = &self.sender_displayname {
            let sender =
@@ -706,6 +704,8 @@ impl<'a> MimeFactory<'a> {
            )
        };
        let outer_message = if is_encrypted {
+            headers.protected.push(from_header);
+
            // Store protected headers in the inner message.
            let message = headers
                .protected
@@ -800,18 +800,12 @@ impl<'a> MimeFactory<'a> {
            // Store protected headers in the outer message.
            let message = headers
                .protected
-                .iter()
-                .fold(message, |message, header| message.header(header.clone()));
+                .into_iter()
+                .fold(message, |message, header| message.header(header));

            if self.should_skip_autocrypt()
                || !context.get_config_bool(Config::SignUnencrypted).await?
            {
-                let protected: HashSet<Header> = HashSet::from_iter(headers.protected.into_iter());
-                for h in headers.unprotected.split_off(0) {
-                    if !protected.contains(&h) {
-                        headers.unprotected.push(h);
-                    }
-                }
                message
            } else {
                let message = message.header(get_content_type_directives_header());
@@ -994,7 +988,8 @@ impl<'a> MimeFactory<'a> {
                    {
                        info!(
                            context,
-                            "Sending secure-join message {:?}.", "vg-member-added",
+                            "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>",
+                            "vg-member-added",
                        );
                        headers.protected.push(Header::new(
                            "Secure-Join".to_string(),
@@ -1076,7 +1071,10 @@ impl<'a> MimeFactory<'a> {
                let msg = &self.msg;
                let step = msg.param.get(Param::Arg).unwrap_or_default();
                if !step.is_empty() {
-                    info!(context, "Sending secure-join message {step:?}.");
+                    info!(
+                        context,
+                        "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>", step,
+                    );
                    headers
                        .protected
                        .push(Header::new("Secure-Join".into(), step.into()));
@@ -2222,7 +2220,6 @@ mod tests {

        let part = payload.next().unwrap();
        assert_eq!(part.match_indices("multipart/signed").count(), 1);
-        assert_eq!(part.match_indices("From:").count(), 1);
        assert_eq!(part.match_indices("Subject:").count(), 0);
        assert_eq!(part.match_indices("Autocrypt:").count(), 1);
        assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
@@ -2233,14 +2230,12 @@ mod tests {
                .count(),
            1
        );
-        assert_eq!(part.match_indices("From:").count(), 1);
        assert_eq!(part.match_indices("Subject:").count(), 1);
        assert_eq!(part.match_indices("Autocrypt:").count(), 0);
        assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);

        let part = payload.next().unwrap();
        assert_eq!(part.match_indices("text/plain").count(), 1);
-        assert_eq!(part.match_indices("From:").count(), 0);
        assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1);
        assert_eq!(part.match_indices("Subject:").count(), 0);

@@ -2267,14 +2262,12 @@ mod tests {

        let part = payload.next().unwrap();
        assert_eq!(part.match_indices("multipart/signed").count(), 1);
-        assert_eq!(part.match_indices("From:").count(), 1);
        assert_eq!(part.match_indices("Subject:").count(), 0);
        assert_eq!(part.match_indices("Autocrypt:").count(), 1);
        assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);

        let part = payload.next().unwrap();
        assert_eq!(part.match_indices("text/plain").count(), 1);
-        assert_eq!(part.match_indices("From:").count(), 1);
        assert_eq!(part.match_indices("Subject:").count(), 1);
        assert_eq!(part.match_indices("Autocrypt:").count(), 0);
        assert_eq!(part.match_indices("multipart/mixed").count(), 0);
@@ -2283,7 +2276,6 @@ mod tests {
        let body = payload.next().unwrap();
        assert_eq!(body.match_indices("this is the text!").count(), 1);
        assert_eq!(body.match_indices("text/plain").count(), 0);
-        assert_eq!(body.match_indices("From:").count(), 0);
        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
        assert_eq!(body.match_indices("Subject:").count(), 0);

--- a/src/mimeparser.rs
+++ b/src/mimeparser.rs
@@ -278,7 +278,7 @@ impl MimeMessage {

        let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref());
        let (mail, mut signatures, encrypted) = match tokio::task::block_in_place(|| {
-            try_decrypt(&mail, &private_keyring, &public_keyring)
+            try_decrypt(context, &mail, &private_keyring, &public_keyring)
        }) {
            Ok(Some((raw, signatures))) => {
                mail_raw = raw;
@@ -461,6 +461,20 @@ impl MimeMessage {
            parser.decoded_data = mail_raw;
        }

+        crate::peerstate::maybe_do_aeap_transition(context, &mut parser).await?;
+        if let Some(peerstate) = &parser.decryption_info.peerstate {
+            peerstate
+                .handle_fingerprint_change(context, message_time)
+                .await?;
+            // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
+            // of an error. Otherwise we don't save peerstate until get here to reduce the number of
+            // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
+            // successfully.
+            if peerstate.prefer_encrypt != EncryptPreference::Mutual {
+                peerstate.save_to_db(&context.sql).await?;
+            }
+        }
+
        Ok(parser)
    }

@@ -680,7 +694,7 @@ impl MimeMessage {
                }
            }

-            self.do_add_single_part(part);
+            self.parts.push(part);
        }

        if self.headers.contains_key("auto-submitted") {
--- a/src/receive_imf.rs
+++ b/src/receive_imf.rs
@@ -10,7 +10,6 @@ use num_traits::FromPrimitive;
 use once_cell::sync::Lazy;
 use regex::Regex;

-use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, ShowEmails, DC_CHAT_ID_TRASH};
@@ -140,6 +139,8 @@ pub(crate) async fn receive_imf_inner(
    is_partial_download: Option<u32>,
    fetching_existing_messages: bool,
 ) -> Result<Option<ReceivedMsg>> {
+    info!(context, "Receiving message, seen={seen}...");
+
    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
        info!(
            context,
@@ -172,30 +173,7 @@ pub(crate) async fn receive_imf_inner(
        Ok(mime_parser) => mime_parser,
    };

-    let rcvd_timestamp = smeared_time(context);
-
-    // Sender timestamp is allowed to be a bit in the future due to
-    // unsynchronized clocks, but not too much.
-    let sent_timestamp = mime_parser
-        .get_header(HeaderDef::Date)
-        .and_then(|value| mailparse::dateparse(value).ok())
-        .map_or(rcvd_timestamp, |value| min(value, rcvd_timestamp + 60));
-
-    crate::peerstate::maybe_do_aeap_transition(context, &mut mime_parser).await?;
-    if let Some(peerstate) = &mime_parser.decryption_info.peerstate {
-        peerstate
-            .handle_fingerprint_change(context, sent_timestamp)
-            .await?;
-        // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
-        // of an error. Otherwise we don't save peerstate until get here to reduce the number of
-        // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
-        // successfully.
-        if peerstate.prefer_encrypt != EncryptPreference::Mutual {
-            peerstate.save_to_db(&context.sql).await?;
-        }
-    }
-
-    info!(context, "Receiving message {rfc724_mid:?}, seen={seen}...");
+    info!(context, "Received message has Message-Id: {rfc724_mid}");

    // check, if the mail is already in our database.
    // make sure, this check is done eg. before securejoin-processing.
@@ -256,10 +234,21 @@ pub(crate) async fn receive_imf_inner(
    )
    .await?;

+    let rcvd_timestamp = smeared_time(context);
+
+    // Sender timestamp is allowed to be a bit in the future due to
+    // unsynchronized clocks, but not too much.
+    let sent_timestamp = mime_parser
+        .get_header(HeaderDef::Date)
+        .and_then(|value| mailparse::dateparse(value).ok())
+        .map_or(rcvd_timestamp, |value| min(value, rcvd_timestamp + 60));
+
    update_verified_keys(context, &mut mime_parser, from_id).await?;

    let received_msg;
-    if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
+    if let Some(securejoin_step) = mime_parser.get_header(HeaderDef::SecureJoin) {
+        info!(context, "Received securejoin step {securejoin_step}.");
+
        let res;
        if incoming {
            res = handle_securejoin_handshake(context, &mime_parser, from_id)
@@ -343,7 +332,7 @@ pub(crate) async fn receive_imf_inner(
    {
        info!(
            context,
-            "Received message contains Autocrypt-Gossip for all members of {chat_id}, updating timestamp."
+            "Received message contains Autocrypt-Gossip for all members, updating timestamp."
        );
        if chat_id.get_gossiped_timestamp(context).await? < sent_timestamp {
            chat_id
@@ -1544,10 +1533,27 @@ async fn lookup_chat_by_reply(
    let Some(parent) = parent else {
        return Ok(None);
    };
-    let Some(parent_chat_id) = ChatId::lookup_by_message(parent) else {
+
+    let parent_chat = Chat::load_from_db(context, parent.chat_id).await?;
+
+    if parent.download_state != DownloadState::Done
+        // TODO (2023-09-12): Added for backward compatibility with versions that did not have
+        // `DownloadState::Undecipherable`. Remove eventually with the comment in
+        // `MimeMessage::from_bytes()`.
+        || parent
+            .error
+            .as_ref()
+            .filter(|e| e.starts_with("Decrypting failed:"))
+            .is_some()
+    {
+        // If the parent msg is not fully downloaded or undecipherable, it may have been
+        // assigned to the wrong chat (they often get assigned to the 1:1 chat with the sender).
        return Ok(None);
-    };
-    let parent_chat = Chat::load_from_db(context, parent_chat_id).await?;
+    }
+
+    if parent_chat.id == DC_CHAT_ID_TRASH {
+        return Ok(None);
+    }

    // If this was a private message just to self, it was probably a private reply.
    // It should not go into the group then, but into the private chat.
@@ -2269,6 +2275,11 @@ async fn create_adhoc_group(
    timestamp: i64,
 ) -> Result<Option<ChatId>> {
    if mime_parser.is_mailinglist_message() {
+        info!(
+            context,
+            "Not creating ad-hoc group for mailing list message."
+        );
+
        return Ok(None);
    }

@@ -2289,6 +2300,7 @@ async fn create_adhoc_group(
    }

    if member_ids.len() < 3 {
+        info!(context, "Not creating ad-hoc group: too few contacts.");
        return Ok(None);
    }

@@ -2308,11 +2320,6 @@ async fn create_adhoc_group(
        timestamp,
    )
    .await?;
-
-    info!(
-        context,
-        "Created ad-hoc group id={new_chat_id}, name={grpname:?}."
-    );
    chat::add_to_chat_contacts_table(context, new_chat_id, member_ids).await?;

    context.emit_event(EventType::ChatModified(new_chat_id));
@@ -2541,7 +2548,20 @@ async fn get_previous_message(
 ///
 /// Only messages that are not in the trash chat are considered.
 async fn get_rfc724_mid_in_list(context: &Context, mid_list: &str) -> Result<Option<Message>> {
-    message::get_latest_by_rfc724_mids(context, &parse_message_ids(mid_list)).await
+    if mid_list.is_empty() {
+        return Ok(None);
+    }
+
+    for id in parse_message_ids(mid_list).iter().rev() {
+        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
+            let msg = Message::load_from_db(context, msg_id).await?;
+            if msg.chat_id != DC_CHAT_ID_TRASH {
+                return Ok(Some(msg));
+            }
+        }
+    }
+
+    Ok(None)
 }

 /// Returns the last message referenced from References: header found in the database.
--- a/src/scheduler.rs
+++ b/src/scheduler.rs
@@ -590,7 +590,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
        .log_err(ctx)
        .ok();

-    connection.connectivity.set_idle(ctx).await;
+    connection.connectivity.set_connected(ctx).await;

    ctx.emit_event(EventType::ImapInboxIdle);
    let Some(session) = connection.session.take() else {
@@ -727,7 +727,7 @@ async fn smtp_loop(
            // Fake Idle
            info!(ctx, "smtp fake idle - started");
            match &connection.last_send_error {
-                None => connection.connectivity.set_idle(&ctx).await,
+                None => connection.connectivity.set_connected(&ctx).await,
                Some(err) => connection.connectivity.set_err(&ctx, err).await,
            }

--- a/src/scheduler/connectivity.rs
+++ b/src/scheduler/connectivity.rs
@@ -33,18 +33,9 @@ enum DetailedConnectivity {
    #[default]
    Uninitialized,
    Connecting,
-
-    /// Connection is just established, but there may be work to do.
-    Connected,
-
-    /// There is actual work to do, e.g. there are messages in SMTP queue
-    /// or we detected a message that should be downloaded.
    Working,
-
    InterruptingIdle,
-
-    /// Connection is established and is idle.
-    Idle,
+    Connected,

    /// The folder was configured not to be watched or configured_*_folder is not set
    NotConfigured,
@@ -63,8 +54,6 @@ impl DetailedConnectivity {
            // Just don't return a connectivity, probably the folder is configured not to be
            // watched or there is e.g. no "Sent" folder, so we are not interested in it
            DetailedConnectivity::NotConfigured => None,
-
-            DetailedConnectivity::Idle => Some(Connectivity::Connected),
        }
    }

@@ -76,8 +65,7 @@ impl DetailedConnectivity {
            DetailedConnectivity::Connecting => "<span class=\"yellow dot\"></span>".to_string(),
            DetailedConnectivity::Working
            | DetailedConnectivity::InterruptingIdle
-            | DetailedConnectivity::Connected
-            | DetailedConnectivity::Idle => "<span class=\"green dot\"></span>".to_string(),
+            | DetailedConnectivity::Connected => "<span class=\"green dot\"></span>".to_string(),
        }
    }

@@ -87,9 +75,9 @@ impl DetailedConnectivity {
            DetailedConnectivity::Uninitialized => "Not started".to_string(),
            DetailedConnectivity::Connecting => stock_str::connecting(context).await,
            DetailedConnectivity::Working => stock_str::updating(context).await,
-            DetailedConnectivity::InterruptingIdle
-            | DetailedConnectivity::Connected
-            | DetailedConnectivity::Idle => stock_str::connected(context).await,
+            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
+                stock_str::connected(context).await
+            }
            DetailedConnectivity::NotConfigured => "Not configured".to_string(),
        }
    }
@@ -106,9 +94,9 @@ impl DetailedConnectivity {
            // We don't know any more than that the last message was sent successfully;
            // since sending the last message, connectivity could have changed, which we don't notice
            // until another message is sent
-            DetailedConnectivity::InterruptingIdle
-            | DetailedConnectivity::Connected
-            | DetailedConnectivity::Idle => stock_str::last_msg_sent_successfully(context).await,
+            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
+                stock_str::last_msg_sent_successfully(context).await
+            }
            DetailedConnectivity::NotConfigured => "Not configured".to_string(),
        }
    }
@@ -120,9 +108,8 @@ impl DetailedConnectivity {
            DetailedConnectivity::Connecting => false,
            DetailedConnectivity::Working => false,
            DetailedConnectivity::InterruptingIdle => false,
-            DetailedConnectivity::Connected => false, // Just connected, there may still be work to do.
+            DetailedConnectivity::Connected => true,
            DetailedConnectivity::NotConfigured => true,
-            DetailedConnectivity::Idle => true,
        }
    }
 }
@@ -154,9 +141,6 @@ impl ConnectivityStore {
    pub(crate) async fn set_not_configured(&self, context: &Context) {
        self.set(context, DetailedConnectivity::NotConfigured).await;
    }
-    pub(crate) async fn set_idle(&self, context: &Context) {
-        self.set(context, DetailedConnectivity::Idle).await;
-    }

    async fn get_detailed(&self) -> DetailedConnectivity {
        self.0.lock().await.deref().clone()
@@ -180,7 +164,6 @@ pub(crate) async fn idle_interrupted(inbox: ConnectivityStore, oboxes: Vec<Conne
    // return Connected until DC is completely done with fetching folders; this also
    // includes scan_folders() which happens on the inbox thread.
    if *connectivity_lock == DetailedConnectivity::Connected
-        || *connectivity_lock == DetailedConnectivity::Idle
        || *connectivity_lock == DetailedConnectivity::NotConfigured
    {
        *connectivity_lock = DetailedConnectivity::InterruptingIdle;
@@ -189,9 +172,7 @@ pub(crate) async fn idle_interrupted(inbox: ConnectivityStore, oboxes: Vec<Conne

    for state in oboxes {
        let mut connectivity_lock = state.0.lock().await;
-        if *connectivity_lock == DetailedConnectivity::Connected
-            || *connectivity_lock == DetailedConnectivity::Idle
-        {
+        if *connectivity_lock == DetailedConnectivity::Connected {
            *connectivity_lock = DetailedConnectivity::InterruptingIdle;
        }
    }
--- a/src/securejoin.rs
+++ b/src/securejoin.rs
@@ -287,7 +287,10 @@ pub(crate) async fn handle_securejoin_handshake(
        .get_header(HeaderDef::SecureJoin)
        .context("Not a Secure-Join message")?;

-    info!(context, "Received secure-join message {step:?}.");
+    info!(
+        context,
+        ">>>>>>>>>>>>>>>>>>>>>>>>> secure-join message \'{}\' received", step,
+    );

    let join_vg = step.starts_with("vg-");

@@ -313,6 +316,7 @@ pub(crate) async fn handle_securejoin_handshake(
                warn!(context, "Secure-join denied (bad invitenumber).");
                return Ok(HandshakeMessage::Ignore);
            }
+            info!(context, "Secure-join requested.",);

            inviter_progress(context, contact_id, 300);

@@ -550,7 +554,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
    let step = mime_message
        .get_header(HeaderDef::SecureJoin)
        .context("Not a Secure-Join message")?;
-    info!(context, "Observing secure-join message {step:?}.");
+    info!(context, "observing secure-join message \'{}\'", step);

    match step.as_str() {
        "vg-request-with-auth"
--- a/src/securejoin/bobstate.rs
+++ b/src/securejoin/bobstate.rs
@@ -291,7 +291,7 @@ impl BobState {
    ) -> Result<Option<BobHandshakeStage>> {
        info!(
            context,
-            "Bob Step 4 - handling {{vc,vg}}-auth-required message."
+            "Bob Step 4 - handling vc-auth-require/vg-auth-required message"
        );
        if !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint())) {
            let reason = if mime_message.was_encrypted() {
@@ -333,7 +333,7 @@ impl BobState {
    ) -> Result<Option<BobHandshakeStage>> {
        info!(
            context,
-            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message."
+            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message"
        );
        mark_peer_as_verified(
            context,
--- a/src/sql.rs
+++ b/src/sql.rs
@@ -131,6 +131,23 @@ impl Sql {
        // drop closes the connection
    }

+    /// Flushes the WAL file.
+    pub(crate) async fn checkpoint(&self, context: &Context) -> Result<()> {
+        let busy = self
+            .call_write(move |conn| {
+                let busy = conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", (), |row| {
+                    let busy: bool = row.get(0)?;
+                    Ok(busy)
+                })?;
+                Ok(busy)
+            })
+            .await?;
+        if busy {
+            warn!(context, "Failed to checkpoint WAL");
+        }
+        Ok(())
+    }
+
    /// Imports the database from a separate file with the given passphrase.
    pub(crate) async fn import(&self, path: &Path, passphrase: String) -> Result<()> {
        let path_str = path
@@ -1 +1 @@
 -12-12
 -12-02