start with debouncing -> doesn't work because I don't know how to integrate it into the context

this is the first step to debouncing

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.74.0
+      RUSTUP_TOOLCHAIN: 1.75.0
     steps:
       - uses: actions/checkout@v3
       - name: Install rustfmt and clippy
@@ -76,11 +76,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.74.0
+            rust: 1.75.0
           - os: windows-latest
-            rust: 1.74.0
+            rust: 1.75.0
           - os: macos-latest
-            rust: 1.74.0
+            rust: 1.75.0
 
           # Minimum Supported Rust Version = 1.70.0
           - os: ubuntu-latest

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [1.132.1] - 2023-12-12
+
+### Features / Changes
+
+- Add "From:" to protected headers for signed-only messages.
+- Sync user actions for ad-hoc groups across devices ([#5065](https://github.com/deltachat/deltachat-core-rust/pull/5065)).
+
+### Fixes
+
+- Add padlock to empty part if the whole message is empty.
+- Renew IDLE timeout on keepalives and reduce it to 5 minutes.
+- connectivity: Return false from `all_work_done()` immediately after connecting (iOS notification fix).
+
+### API-Changes
+
+- deltachat-jsonrpc-client: add `Account.{import,export}_self_keys`.
+
+### CI
+
+- Update to Rust 1.74.1.
+
 ## [1.132.0] - 2023-12-06
 
 ### Features / Changes
@@ -3334,3 +3355,5 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.131.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.6...v1.131.7
 [1.131.8]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.7...v1.131.8
 [1.131.9]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.8...v1.131.9
+[1.132.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.9...v1.132.0
+[1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.132.0"
+version = "1.132.1"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.70"
@@ -40,7 +40,7 @@ ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
 async-channel = "2.0.0"
-async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
+async-imap = { version = "0.9.5", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
@@ -59,7 +59,7 @@ hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
 image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh = { git = "https://github.com/n0-computer/iroh", branch = "maint-0.4", default-features = false }
+iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -78,11 +78,11 @@ qrcodegen = "1.7.0"
 quick-xml = "0.31"
 rand = "0.8"
 regex = "1.9"
-reqwest = { version = "0.11.20", features = ["json"] }
+reqwest = { version = "0.11.23", features = ["json"] }
 rusqlite = { version = "0.30", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
-serde_json = "1.0"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"

README.md

@@ -27,7 +27,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -121,7 +121,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.132.0"
+version = "1.132.1"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -6230,6 +6230,22 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
 
+/**
+ * Inform UI that Order (and content as in chat ids) of the chatlist changed.
+ *
+ * Sometimes this is emitted together with `DC_EVENT_UI_CHATLIST_ITEM_CHANGED` such as on `DC_EVENT_INCOMING_MSG`.
+ */
+
+#define DC_EVENT_UI_CHATLIST_CHANGED          2200
+
+/**
+ * Inform UI that all or a single chat list item changed and needs to be rerendered
+ * If `chat_id` is set to 0, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
+ * 
+ * @param data1 (int) chat_id chat id of chatlist item to be rerendered, if chat_id = 0 all (cached & visible) items need to be rerendered
+ */
+
+#define DC_EVENT_UI_CHATLIST_ITEM_CHANGED     2201
 
 /**
  * @}

deltachat-ffi/src/lib.rs

@@ -558,6 +558,8 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::SelfavatarChanged => 2110,
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
+        EventType::UIChatListChanged => 2200,
+        EventType::UIChatListItemChanged { .. } => 2201,
     }
 }
 
@@ -584,7 +586,8 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
         | EventType::IncomingMsgBunch { .. }
-        | EventType::ErrorSelfNotInGroup(_) => 0,
+        | EventType::ErrorSelfNotInGroup(_)
+        | EventType::UIChatListChanged => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -609,6 +612,9 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         }
         EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
         EventType::WebxdcInstanceDeleted { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::UIChatListItemChanged { chat_id } => {
+            chat_id.unwrap_or_default().to_u32() as libc::c_int
+        }
     }
 }
 
@@ -643,7 +649,9 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::SelfavatarChanged => 0,
+        | EventType::SelfavatarChanged
+        | EventType::UIChatListChanged
+        | EventType::UIChatListItemChanged { .. } => 0,
         EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
@@ -705,7 +713,9 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
         | EventType::WebxdcInstanceDeleted { .. }
-        | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
+        | EventType::ChatEphemeralTimerModified { .. }
+        | EventType::UIChatListItemChanged { .. }
+        | EventType::UIChatListChanged => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
                 comment.to_c_string().unwrap_or_default().into_raw()

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.132.0"
+version = "1.132.1"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -17,11 +17,11 @@ deltachat = { path = ".." }
 num-traits = "0.2"
 schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.8.0"
+tempfile = "3.9.0"
 log = "0.4"
 async-channel = { version = "2.0.0" }
-futures = { version = "0.3.28" }
-serde_json = "1.0.105"
+futures = { version = "0.3.30" }
+serde_json = "1"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
 tokio = { version = "1.33.0" }

deltachat-jsonrpc/src/api/types/chat.rs

@@ -85,7 +85,7 @@ impl FullChat {
         let can_send = chat.can_send(context).await?;
 
         let was_seen_recently = if chat.get_type() == Chattype::Single {
-            match contact_ids.get(0) {
+            match contact_ids.first() {
                 Some(contact) => Contact::get_by_id(context, *contact)
                     .await
                     .context("failed to load contact for was_seen_recently")?

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -102,7 +102,7 @@ pub(crate) async fn get_chat_list_item_by_id(
     let self_in_group = chat_contacts.contains(&ContactId::SELF);
 
     let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
-        let contact = chat_contacts.get(0);
+        let contact = chat_contacts.first();
         let was_seen_recently = match contact {
             Some(contact) => Contact::get_by_id(ctx, *contact)
                 .await

deltachat-jsonrpc/src/api/types/events.rs

@@ -301,6 +301,18 @@ pub enum EventType {
     WebxdcInstanceDeleted {
         msg_id: u32,
     },
+
+    /// Inform UI that Order (and content as in chat ids) of the chatlist changed.
+    ///
+    /// Sometimes this is emitted together with `UIChatListItemChanged` such as on IncomingMessage.
+    UIChatListChanged,
+
+    /// Inform UI that a single chat list item changed and needs to be rerendered
+    /// If `chat_id` is set to None, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
+    #[serde(rename_all = "camelCase")]
+    UIChatListItemChanged {
+        chat_id: Option<u32>,
+    },
 }
 
 impl From<CoreEventType> for EventType {
@@ -406,6 +418,10 @@ impl From<CoreEventType> for EventType {
             CoreEventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
                 msg_id: msg_id.to_u32(),
             },
+            CoreEventType::UIChatListItemChanged { chat_id } => UIChatListItemChanged {
+                chat_id: chat_id.map(|id| id.to_u32()),
+            },
+            CoreEventType::UIChatListChanged => UIChatListChanged,
         }
     }
 }

deltachat-jsonrpc/typescript/package.json

@@ -53,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.132.0"
+  "version": "1.132.1"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.132.0"
+version = "1.132.1"
 license = "MPL-2.0"
 edition = "2021"
 

deltachat-repl/src/main.rs

@@ -481,7 +481,10 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    let _ = pretty_env_logger::try_init();
+    pretty_env_logger::formatted_timed_builder()
+        .parse_default_env()
+        .filter_module("deltachat_repl", log::LevelFilter::Info)
+        .init();
 
     let args = std::env::args().collect();
     start(args).await?;

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -300,3 +300,13 @@ class Account:
     def import_backup(self, path, passphrase: str = "") -> None:
         """Import backup."""
         self._rpc.import_backup(self.id, str(path), passphrase)
+
+    def export_self_keys(self, path) -> None:
+        """Export keys."""
+        passphrase = ""  # Setting passphrase is currently not supported.
+        self._rpc.export_self_keys(self.id, str(path), passphrase)
+
+    def import_self_keys(self, path) -> None:
+        """Import keys."""
+        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
+        self._rpc.import_self_keys(self.id, str(path), passphrase)

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,9 +1,10 @@
 import logging
 
+import pytest
 from deltachat_rpc_client import Chat, SpecialContactId
 
 
-def test_qr_setup_contact(acfactory) -> None:
+def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     qr_code, _svg = alice.get_qr_code()
@@ -23,13 +24,26 @@ def test_qr_setup_contact(acfactory) -> None:
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
+    # Test that if Bob changes the key, backwards verification is lost.
+    logging.info("Bob 2 is created")
+    bob2 = acfactory.new_configured_account()
+    bob2.export_self_keys(tmp_path)
 
-def test_qr_securejoin(acfactory):
+    logging.info("Bob imports a key")
+    bob.import_self_keys(tmp_path / "private-key-default.asc")
+
+    assert bob.get_config("key_id") == "2"
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert not bob_contact_alice_snapshot.is_verified
+
+
+@pytest.mark.parametrize("protect", [True, False])
+def test_qr_securejoin(acfactory, protect):
     alice, bob = acfactory.get_online_accounts(2)
 
     logging.info("Alice creates a verified group")
-    alice_chat = alice.create_group("Verified group", protect=True)
-    assert alice_chat.get_basic_snapshot().is_protected
+    alice_chat = alice.create_group("Verified group", protect=protect)
+    assert alice_chat.get_basic_snapshot().is_protected == protect
 
     logging.info("Bob joins verified group")
     qr_code, _svg = alice_chat.get_qr_code()
@@ -53,7 +67,7 @@ def test_qr_securejoin(acfactory):
 
     snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
-    assert snapshot.chat.get_basic_snapshot().is_protected
+    assert snapshot.chat.get_basic_snapshot().is_protected == protect
 
     # Test that Bob verified Alice's profile.
     bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))

deltachat-rpc-client/tests/test_something.py

@@ -140,12 +140,9 @@ def test_chat(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
     assert snapshot.chat_id == chat_id
@@ -224,12 +221,9 @@ def test_message(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
 
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
@@ -331,7 +325,7 @@ def test_wait_next_messages(acfactory) -> None:
         next_messages_task = executor.submit(bot.wait_next_messages)
 
         bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
         alice_chat_bot = alice_contact_bot.create_chat()
         alice_chat_bot.send_text("Hello!")
 
@@ -341,7 +335,7 @@ def test_wait_next_messages(acfactory) -> None:
         assert snapshot.text == "Hello!"
 
 
-def test_import_export(acfactory, tmp_path) -> None:
+def test_import_export_backup(acfactory, tmp_path) -> None:
     alice = acfactory.new_configured_account()
     alice.export_backup(tmp_path)
 
@@ -352,6 +346,31 @@ def test_import_export(acfactory, tmp_path) -> None:
     assert alice2.manager.get_system_info()
 
 
+def test_import_export_keys(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello Bob!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Bob!"
+
+    # Alice resetups account, but keeps the key.
+    alice_keys_path = tmp_path / "alice_keys"
+    alice_keys_path.mkdir()
+    alice.export_self_keys(alice_keys_path)
+    alice = acfactory.resetup_account(alice)
+    alice.import_self_keys(alice_keys_path)
+
+    snapshot.chat.accept()
+    snapshot.chat.send_text("Hello Alice!")
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Alice!"
+    assert snapshot.show_padlock
+
+
 def test_openrpc_command_line() -> None:
     """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
     out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
@@ -377,3 +396,46 @@ def test_provider_info(rpc) -> None:
     rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
+
+
+def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+
+    # Bob creates chat manually so chat with Alice is accepted.
+    alice_chat_bob = alice_contact_bob.create_chat()
+
+    # Alice sends a message to Bob.
+    alice_chat_bob.send_text("Hello Bob!")
+    event = bob.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = bob.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+
+    # Bob sends a message to Alice.
+    bob_chat_alice = snapshot.chat
+    bob_chat_alice.accept()
+    bob_chat_alice.send_text("Hello Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock
+
+    # Alice reads Bob's message.
+    message.mark_seen()
+    while True:
+        event = bob.wait_for_event()
+        if event.kind == EventType.MSG_READ:
+            break
+
+    # Bob sends a message to Alice, it should also be encrypted.
+    bob_chat_alice.send_text("Hi Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.132.0"
+version = "1.132.1"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -17,7 +17,7 @@ anyhow = "1"
 env_logger = { version = "0.10.0" }
 futures-lite = "2.0.0"
 log = "0.4"
-serde_json = "1.0.105"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "1.33.0", features = ["io-std"] }
 tokio-util = "0.7.9"

deny.toml

@@ -53,7 +53,6 @@ skip = [
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },
-     { name = "socket2", version = "0.4.9" },
      { name = "spin", version = "<0.9.6" },
      { name = "spki", version = "0.6.0" },
      { name = "syn", version = "1.0.109" },
@@ -102,5 +101,4 @@ github = [
        "async-email",
        "deltachat",
        "djc",
-       "n0-computer", # iroh
 ]

node/constants.js

@@ -60,6 +60,8 @@ module.exports = {
   DC_EVENT_SELFAVATAR_CHANGED: 2110,
   DC_EVENT_SMTP_CONNECTED: 101,
   DC_EVENT_SMTP_MESSAGE_SENT: 103,
+  DC_EVENT_UI_CHATLIST_CHANGED: 2200,
+  DC_EVENT_UI_CHATLIST_ITEM_CHANGED: 2201,
   DC_EVENT_WARNING: 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED: 2121,
   DC_EVENT_WEBXDC_STATUS_UPDATE: 2120,

node/events.js

@@ -35,5 +35,7 @@ module.exports = {
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
-  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED'
+  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2200: 'DC_EVENT_UI_CHATLIST_CHANGED',
+  2201: 'DC_EVENT_UI_CHATLIST_ITEM_CHANGED'
 }

node/lib/constants.ts

@@ -60,6 +60,8 @@ export enum C {
   DC_EVENT_SELFAVATAR_CHANGED = 2110,
   DC_EVENT_SMTP_CONNECTED = 101,
   DC_EVENT_SMTP_MESSAGE_SENT = 103,
+  DC_EVENT_UI_CHATLIST_CHANGED = 2200,
+  DC_EVENT_UI_CHATLIST_ITEM_CHANGED = 2201,
   DC_EVENT_WARNING = 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED = 2121,
   DC_EVENT_WEBXDC_STATUS_UPDATE = 2120,
@@ -321,4 +323,6 @@ export const EventId2EventName: { [key: number]: string } = {
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2200: 'DC_EVENT_UI_CHATLIST_CHANGED',
+  2201: 'DC_EVENT_UI_CHATLIST_ITEM_CHANGED',
 }

package.json

@@ -8,7 +8,7 @@
   "devDependencies": {
     "@types/debug": "^4.1.7",
     "@types/node": "^20.8.10",
-    "chai": "^4.2.0",
+    "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
     "esm": "^3.2.25",
     "mocha": "^8.2.1",
@@ -56,5 +56,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.132.0"
+  "version": "1.132.1"
 }

python/tests/test_1_online.py

@@ -1979,6 +1979,32 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
 
 
+def test_all_work_done(acfactory, lp):
+    """
+    Tests that calling start_io() immediately followed by maybe_network()
+    and then waiting for all_work_done() reliably fetches the messages
+    delivered while account was offline.
+    In other words, connectivity should not change to a state
+    where all_work_done() returns true until IMAP connection goes idle.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    ac1.stop_io()
+    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+    ac1.direct_imap.select_config_folder("inbox")
+    with ac1.direct_imap.idle() as idle1:
+        ac2.create_chat(ac1).send_text("Hi")
+        idle1.wait_for_new_message()
+
+    ac1.start_io()
+    ac1.maybe_network()
+    ac1._evtracker.wait_for_all_work_done()
+    msgs = ac1.create_chat(ac2).get_messages()
+    assert len(msgs) == 1
+    assert msgs[0].text == "Hi"
+
+
 def test_fetch_deleted_msg(acfactory, lp):
     """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
     hundreds of times, because uid_next was not updated.

release-date.in

@@ -1 +1 @@
-2023-12-06
+2023-12-12

src/accounts.rs

@@ -546,8 +546,12 @@ impl Config {
             }
             if self.inner.selected_account == id {
                 // reset selected account
-                self.inner.selected_account =
-                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                self.inner.selected_account = self
+                    .inner
+                    .accounts
+                    .first()
+                    .map(|e| e.id)
+                    .unwrap_or_default();
             }
         }
 

src/chat.rs

@@ -19,8 +19,8 @@ use crate::chatlist::Chatlist;
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_LAST_SPECIAL,
-    DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS,
+    self, Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK,
+    DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS,
 };
 use crate::contact::{self, Contact, ContactAddress, ContactId, Origin};
 use crate::context::Context;
@@ -46,6 +46,7 @@ use crate::tools::{
     create_smeared_timestamps, get_abs_path, gm2local_offset, improve_single_line_input,
     smeared_time, strip_rtlo_characters, time, IsNoneOrEmpty,
 };
+use crate::ui_events;
 use crate::webxdc::WEBXDC_SUFFIX;
 
 /// An chat item, such as a message or a marker.
@@ -209,6 +210,30 @@ impl ChatId {
         self == DC_CHAT_ID_ALLDONE_HINT
     }
 
+    /// Returns [`ChatId`] of a chat that `msg` belongs to.
+    ///
+    /// Checks that `msg` is assigned to the right chat.
+    pub(crate) fn lookup_by_message(msg: &Message) -> Option<Self> {
+        if msg.chat_id == DC_CHAT_ID_TRASH {
+            return None;
+        }
+        if msg.download_state != DownloadState::Done
+            // TODO (2023-09-12): Added for backward compatibility with versions that did not have
+            // `DownloadState::Undecipherable`. Remove eventually with the comment in
+            // `MimeMessage::from_bytes()`.
+            || msg
+                .error
+                .as_ref()
+                .filter(|e| e.starts_with("Decrypting failed:"))
+                .is_some()
+        {
+            // If `msg` is not fully downloaded or undecipherable, it may have been assigned to the
+            // wrong chat (they often get assigned to the 1:1 chat with the sender).
+            return None;
+        }
+        Some(msg.chat_id)
+    }
+
     /// Returns the [`ChatId`] for the 1:1 chat with `contact_id`
     /// if it exists and is not blocked.
     ///
@@ -285,6 +310,8 @@ impl ChatId {
             }
         };
         context.emit_msgs_changed_without_ids();
+        ui_events::emit_chatlist_changed(context);
+        ui_events::emit_chatlist_item_changed(context, chat_id);
         Ok(chat_id)
     }
 
@@ -374,6 +401,7 @@ impl ChatId {
 
     pub(crate) async fn block_ex(self, context: &Context, sync: sync::Sync) -> Result<()> {
         let chat = Chat::load_from_db(context, self).await?;
+        let mut delete = false;
 
         match chat.typ {
             Chattype::Broadcast => {
@@ -392,7 +420,7 @@ impl ChatId {
             }
             Chattype::Group => {
                 info!(context, "Can't block groups yet, deleting the chat.");
-                self.delete(context).await?;
+                delete = true;
             }
             Chattype::Mailinglist => {
                 if self.set_blocked(context, Blocked::Yes).await? {
@@ -400,6 +428,7 @@ impl ChatId {
                 }
             }
         }
+        ui_events::emit_chatlist_changed(context);
 
         if sync.into() {
             // NB: For a 1:1 chat this currently triggers `Contact::block()` on other devices.
@@ -408,6 +437,9 @@ impl ChatId {
                 .log_err(context)
                 .ok();
         }
+        if delete {
+            self.delete(context).await?;
+        }
         Ok(())
     }
 
@@ -419,6 +451,8 @@ impl ChatId {
     pub(crate) async fn unblock_ex(self, context: &Context, sync: sync::Sync) -> Result<()> {
         self.set_blocked(context, Blocked::Not).await?;
 
+        ui_events::emit_chatlist_changed(context);
+
         if sync.into() {
             let chat = Chat::load_from_db(context, self).await?;
             // TODO: For a 1:1 chat this currently triggers `Contact::unblock()` on other devices.
@@ -429,6 +463,7 @@ impl ChatId {
                 .log_err(context)
                 .ok();
         }
+
         Ok(())
     }
 
@@ -472,6 +507,7 @@ impl ChatId {
 
         if self.set_blocked(context, Blocked::Not).await? {
             context.emit_event(EventType::ChatModified(self));
+            ui_events::emit_chatlist_item_changed(context, self);
         }
 
         if sync.into() {
@@ -514,6 +550,7 @@ impl ChatId {
             .await?;
 
         context.emit_event(EventType::ChatModified(self));
+        ui_events::emit_chatlist_item_changed(context, self);
 
         // make sure, the receivers will get all keys
         self.reset_gossiped_timestamp(context).await?;
@@ -550,7 +587,7 @@ impl ChatId {
     ///
     /// `timestamp_sort` is used as the timestamp of the added message
     /// and should be the timestamp of the change happening.
-    pub(crate) async fn set_protection(
+    async fn set_protection_for_timestamp_sort(
         self,
         context: &Context,
         protect: ProtectionStatus,
@@ -562,6 +599,7 @@ impl ChatId {
                 if protection_status_modified {
                     self.add_protection_msg(context, protect, contact_id, timestamp_sort)
                         .await?;
+                    ui_events::emit_chatlist_item_changed(context, self);
                 }
                 Ok(())
             }
@@ -572,6 +610,24 @@ impl ChatId {
         }
     }
 
+    /// Sets protection and sends or adds a message.
+    ///
+    /// `timestamp_sent` is the "sent" timestamp of a message caused the protection state change.
+    pub(crate) async fn set_protection(
+        self,
+        context: &Context,
+        protect: ProtectionStatus,
+        timestamp_sent: i64,
+        contact_id: Option<ContactId>,
+    ) -> Result<()> {
+        let sort_to_bottom = true;
+        let ts = self
+            .calc_sort_timestamp(context, timestamp_sent, sort_to_bottom, false)
+            .await?;
+        self.set_protection_for_timestamp_sort(context, protect, ts, contact_id)
+            .await
+    }
+
     /// Sets the 1:1 chat with the given address to ProtectionStatus::Protected,
     /// and posts a `SystemMessage::ChatProtectionEnabled` into it.
     ///
@@ -579,6 +635,7 @@ impl ChatId {
     pub(crate) async fn set_protection_for_contact(
         context: &Context,
         contact_id: ContactId,
+        timestamp: i64,
     ) -> Result<()> {
         let chat_id = ChatId::create_for_contact_with_blocked(context, contact_id, Blocked::Yes)
             .await
@@ -587,7 +644,7 @@ impl ChatId {
             .set_protection(
                 context,
                 ProtectionStatus::Protected,
-                smeared_time(context),
+                timestamp,
                 Some(contact_id),
             )
             .await?;
@@ -629,6 +686,8 @@ impl ChatId {
             .await?;
 
         context.emit_msgs_changed_without_ids();
+        ui_events::emit_chatlist_changed(context);
+        ui_events::emit_chatlist_item_changed(context, self);
 
         if sync.into() {
             let chat = Chat::load_from_db(context, self).await?;
@@ -735,6 +794,7 @@ impl ChatId {
             .await?;
 
         context.emit_msgs_changed_without_ids();
+        ui_events::emit_chatlist_changed(context);
 
         context.set_config(Config::LastHousekeeping, None).await?;
         context.scheduler.interrupt_inbox().await;
@@ -744,6 +804,7 @@ impl ChatId {
             msg.text = stock_str::self_deleted_msg_body(context).await;
             add_device_msg(context, None, Some(&mut msg)).await?;
         }
+        ui_events::emit_chatlist_changed(context);
 
         Ok(())
     }
@@ -1124,47 +1185,46 @@ impl ChatId {
         Ok(self.get_param(context).await?.exists(Param::Devicetalk))
     }
 
-    async fn parent_query<T, F>(self, context: &Context, fields: &str, f: F) -> Result<Option<T>>
+    async fn parent_query<T, F>(
+        self,
+        context: &Context,
+        fields: &str,
+        state_out_min: MessageState,
+        f: F,
+    ) -> Result<Option<T>>
     where
         F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
         T: Send + 'static,
     {
         let sql = &context.sql;
-        // Do not reply to not fully downloaded messages. Such a message could be a group chat
-        // message that we assigned to 1:1 chat.
         let query = format!(
             "SELECT {fields} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden AND download_state={} \
+             FROM msgs \
+             WHERE chat_id=? \
+             AND ((state BETWEEN {} AND {}) OR (state >= {})) \
+             AND NOT hidden \
+             AND download_state={} \
              ORDER BY timestamp DESC, id DESC \
              LIMIT 1;",
-             DownloadState::Done as u32,
+            MessageState::InFresh as u32,
+            MessageState::InSeen as u32,
+            state_out_min as u32,
+            // Do not reply to not fully downloaded messages. Such a message could be a group chat
+            // message that we assigned to 1:1 chat.
+            DownloadState::Done as u32,
         );
-        let row = sql
-            .query_row_optional(
-                &query,
-                (
-                    self,
-                    MessageState::OutPreparing,
-                    MessageState::OutDraft,
-                    // We don't filter `OutPending` and `OutFailed` messages because the new message
-                    // for which `parent_query()` is done may assume that it will be received in a
-                    // context affected by those messages, e.g. they could add new members to a
-                    // group and the new message will contain them in "To:". Anyway recipients must
-                    // be prepared to orphaned references.
-                ),
-                f,
-            )
-            .await?;
-        Ok(row)
+        sql.query_row_optional(&query, (self,), f).await
     }
 
     async fn get_parent_mime_headers(
         self,
         context: &Context,
+        state_out_min: MessageState,
     ) -> Result<Option<(String, String, String)>> {
         self.parent_query(
             context,
             "rfc724_mid, mime_in_reply_to, mime_references",
+            state_out_min,
             |row: &rusqlite::Row| {
                 let rfc724_mid: String = row.get(0)?;
                 let mime_in_reply_to: String = row.get(1)?;
@@ -1291,6 +1351,64 @@ impl ChatId {
             .unwrap_or_default();
         Ok(protection_status)
     }
+
+    /// Returns the sort timestamp for a new message in the chat.
+    ///
+    /// `message_timestamp` should be either the message "sent" timestamp or a timestamp of the
+    /// corresponding event in case of a system message (usually the current system time).
+    /// `always_sort_to_bottom` makes this ajust the returned timestamp up so that the message goes
+    /// to the chat bottom.
+    /// `incoming` -- whether the message is incoming.
+    pub(crate) async fn calc_sort_timestamp(
+        self,
+        context: &Context,
+        message_timestamp: i64,
+        always_sort_to_bottom: bool,
+        incoming: bool,
+    ) -> Result<i64> {
+        let mut sort_timestamp = cmp::min(message_timestamp, smeared_time(context));
+
+        let last_msg_time: Option<i64> = if always_sort_to_bottom {
+            // get newest message for this chat
+
+            // Let hidden messages also be ordered with protection messages because hidden messages
+            // also can be or not be verified, so let's preserve this information -- even it's not
+            // used currently, it can be useful in the future versions.
+            context
+                .sql
+                .query_get_value(
+                    "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state!=?",
+                    (self, MessageState::OutDraft),
+                )
+                .await?
+        } else if incoming {
+            // get newest non fresh message for this chat.
+
+            // If a user hasn't been online for some time, the Inbox is fetched first and then the
+            // Sentbox. In order for Inbox and Sent messages to be allowed to mingle, outgoing
+            // messages are purely sorted by their sent timestamp. NB: The Inbox must be fetched
+            // first otherwise Inbox messages would be always below old Sentbox messages. We could
+            // take in the query below only incoming messages, but then new incoming messages would
+            // mingle with just sent outgoing ones and apear somewhere in the middle of the chat.
+            context
+                .sql
+                .query_get_value(
+                    "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND hidden=0 AND state>?",
+                    (self, MessageState::InFresh),
+                )
+                .await?
+        } else {
+            None
+        };
+
+        if let Some(last_msg_time) = last_msg_time {
+            if last_msg_time > sort_timestamp {
+                sort_timestamp = last_msg_time;
+            }
+        }
+
+        Ok(sort_timestamp)
+    }
 }
 
 impl std::fmt::Display for ChatId {
@@ -1741,7 +1859,15 @@ impl Chat {
         // we do not set In-Reply-To/References in this case.
         if !self.is_self_talk() {
             if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                self.id.get_parent_mime_headers(context).await?
+                // We don't filter `OutPending` and `OutFailed` messages because the new message for
+                // which `parent_query()` is done may assume that it will be received in a context
+                // affected by those messages, e.g. they could add new members to a group and the
+                // new message will contain them in "To:". Anyway recipients must be prepared to
+                // orphaned references.
+                self
+                    .id
+                    .get_parent_mime_headers(context, MessageState::OutPending)
+                    .await?
             {
                 // "In-Reply-To:" is not changed if it is set manually.
                 // This does not affect "References:" header, it will contain "default parent" (the
@@ -1959,10 +2085,26 @@ impl Chat {
                 Ok(r)
             }
             Chattype::Broadcast | Chattype::Group | Chattype::Mailinglist => {
-                if self.grpid.is_empty() {
-                    return Ok(None);
+                if !self.grpid.is_empty() {
+                    return Ok(Some(SyncId::Grpid(self.grpid.clone())));
                 }
-                Ok(Some(SyncId::Grpid(self.grpid.clone())))
+
+                let Some((parent_rfc724_mid, parent_in_reply_to, _)) = self
+                    .id
+                    .get_parent_mime_headers(context, MessageState::OutDelivered)
+                    .await?
+                else {
+                    warn!(
+                        context,
+                        "Chat::get_sync_id({}): No good message identifying the chat found.",
+                        self.id
+                    );
+                    return Ok(None);
+                };
+                Ok(Some(SyncId::Msgids(vec![
+                    parent_in_reply_to,
+                    parent_rfc724_mid,
+                ])))
             }
         }
     }
@@ -1976,7 +2118,7 @@ impl Chat {
     }
 }
 
-async fn sync(context: &Context, id: SyncId, action: SyncAction) -> Result<()> {
+pub(crate) async fn sync(context: &Context, id: SyncId, action: SyncAction) -> Result<()> {
     context
         .add_sync_item(SyncData::AlterChat { id, action })
         .await?;
@@ -2534,17 +2676,20 @@ pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) ->
 
 /// Tries to send a message synchronously.
 ///
-/// Creates a new message in `smtp` table, then drectly opens an SMTP connection and sends the
-/// message. If this fails, the message remains in the database to be sent later.
+/// Creates jobs in the `smtp` table, then drectly opens an SMTP connection and sends the
+/// message. If this fails, the jobs remain in the database for later sending.
 pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
-    if let Some(rowid) = prepare_send_msg(context, chat_id, msg).await? {
-        let mut smtp = crate::smtp::Smtp::new();
+    let rowids = prepare_send_msg(context, chat_id, msg).await?;
+    if rowids.is_empty() {
+        return Ok(msg.id);
+    }
+    let mut smtp = crate::smtp::Smtp::new();
+    for rowid in rowids {
         send_msg_to_smtp(context, &mut smtp, rowid)
             .await
             .context("failed to send message, queued for later sending")?;
-
-        context.emit_msgs_changed(msg.chat_id, msg.id);
     }
+    context.emit_msgs_changed(msg.chat_id, msg.id);
     Ok(msg.id)
 }
 
@@ -2554,7 +2699,7 @@ async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -
         msg.text = strip_rtlo_characters(&msg.text);
     }
 
-    if prepare_send_msg(context, chat_id, msg).await?.is_some() {
+    if !prepare_send_msg(context, chat_id, msg).await?.is_empty() {
         context.emit_msgs_changed(msg.chat_id, msg.id);
 
         if msg.param.exists(Param::SetLatitude) {
@@ -2567,12 +2712,12 @@ async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -
     Ok(msg.id)
 }
 
-/// Returns rowid from `smtp` table.
+/// Returns row ids of the `smtp` table.
 async fn prepare_send_msg(
     context: &Context,
     chat_id: ChatId,
     msg: &mut Message,
-) -> Result<Option<i64>> {
+) -> Result<Vec<i64>> {
     // prepare_msg() leaves the message state to OutPreparing, we
     // only have to change the state to OutPending in this case.
     // Otherwise we still have to prepare the message, which will set
@@ -2588,20 +2733,16 @@ async fn prepare_send_msg(
         );
         message::update_msg_state(context, msg.id, MessageState::OutPending).await?;
     }
-    let row_id = create_send_msg_job(context, msg).await?;
-    Ok(row_id)
+    create_send_msg_jobs(context, msg).await
 }
 
-/// Constructs a job for sending a message and inserts into `smtp` table.
+/// Constructs jobs for sending a message and inserts them into the `smtp` table.
 ///
-/// Returns rowid if job was created or `None` if SMTP job is not needed, e.g. when sending to a
+/// Returns row ids if jobs were created or an empty `Vec` otherwise, e.g. when sending to a
 /// group with only self and no BCC-to-self configured.
 ///
-/// The caller has to interrupt SMTP loop or otherwise process a new row.
-pub(crate) async fn create_send_msg_job(
-    context: &Context,
-    msg: &mut Message,
-) -> Result<Option<i64>> {
+/// The caller has to interrupt SMTP loop or otherwise process new rows.
+pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result<Vec<i64>> {
     let needs_encryption = msg.param.get_bool(Param::GuaranteeE2ee).unwrap_or_default();
 
     let attach_selfavatar = match shall_attach_selfavatar(context, msg.chat_id).await {
@@ -2620,7 +2761,7 @@ pub(crate) async fn create_send_msg_job(
     let lowercase_from = from.to_lowercase();
 
     // Send BCC to self if it is enabled and we are not going to
-    // delete it immediately.
+    // delete it immediately. `from` must be the last addr, see `receive_imf_inner()` why.
     if context.get_config_bool(Config::BccSelf).await?
         && context.get_config_delete_server_after().await? != Some(0)
         && !recipients
@@ -2638,7 +2779,7 @@ pub(crate) async fn create_send_msg_job(
         );
         msg.id.set_delivered(context).await?;
         msg.state = MessageState::OutDelivered;
-        return Ok(None);
+        return Ok(Vec::new());
     }
 
     let rendered_msg = match mimefactory.render(context).await {
@@ -2698,27 +2839,32 @@ pub(crate) async fn create_send_msg_job(
         msg.update_param(context).await?;
     }
 
-    ensure!(!recipients.is_empty(), "no recipients for smtp job set");
-
-    let recipients = recipients.join(" ");
-
     msg.subject = rendered_msg.subject.clone();
     msg.update_subject(context).await?;
-
-    let row_id = context
-        .sql
-        .insert(
-            "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id)
-             VALUES           (?1,         ?2,         ?3,   ?4)",
-            (
-                &rendered_msg.rfc724_mid,
-                recipients,
-                &rendered_msg.message,
-                msg.id,
-            ),
-        )
-        .await?;
-    Ok(Some(row_id))
+    let chunk_size = context
+        .get_configured_provider()
+        .await?
+        .and_then(|provider| provider.opt.max_smtp_rcpt_to)
+        .map_or(constants::DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
+    let trans_fn = |t: &mut rusqlite::Transaction| {
+        let mut row_ids = Vec::<i64>::new();
+        for recipients_chunk in recipients.chunks(chunk_size) {
+            let recipients_chunk = recipients_chunk.join(" ");
+            let row_id = t.execute(
+                "INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
+                VALUES            (?1,         ?2,         ?3,   ?4)",
+                (
+                    &rendered_msg.rfc724_mid,
+                    recipients_chunk,
+                    &rendered_msg.message,
+                    msg.id,
+                ),
+            )?;
+            row_ids.push(row_id.try_into()?);
+        }
+        Ok(row_ids)
+    };
+    context.sql.transaction(trans_fn).await
 }
 
 /// Sends a text message to the given chat.
@@ -2948,7 +3094,9 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()>
             .await?;
         for chat_id_in_archive in chat_ids_in_archive {
             context.emit_event(EventType::MsgsNoticed(chat_id_in_archive));
+            ui_events::emit_chatlist_item_changed(context, chat_id_in_archive);
         }
+        ui_events::emit_chatlist_item_changed(context, DC_CHAT_ID_ARCHIVED_LINK);
     } else {
         let exists = context
             .sql
@@ -2975,6 +3123,7 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()>
     }
 
     context.emit_event(EventType::MsgsNoticed(chat_id));
+    ui_events::emit_chatlist_item_changed(context, chat_id);
 
     Ok(())
 }
@@ -3042,6 +3191,7 @@ pub(crate) async fn mark_old_messages_as_noticed(
 
     for c in changed_chats {
         context.emit_event(EventType::MsgsNoticed(c));
+        ui_events::emit_chatlist_item_changed(context, c);
     }
 
     Ok(())
@@ -3204,10 +3354,12 @@ pub async fn create_group_chat(
     }
 
     context.emit_msgs_changed_without_ids();
+    ui_events::emit_chatlist_changed(context);
+    ui_events::emit_chatlist_item_changed(context, chat_id);
 
     if protect == ProtectionStatus::Protected {
         chat_id
-            .set_protection(context, protect, timestamp, None)
+            .set_protection_for_timestamp_sort(context, protect, timestamp, None)
             .await?;
     }
 
@@ -3291,11 +3443,14 @@ pub(crate) async fn create_broadcast_list_ex(
     let chat_id = ChatId::new(u32::try_from(row_id)?);
 
     context.emit_msgs_changed_without_ids();
+    ui_events::emit_chatlist_changed(context);
+
     if sync.into() {
         let id = SyncId::Grpid(grpid);
         let action = SyncAction::CreateBroadcast(chat_name);
         self::sync(context, id, action).await.log_err(context).ok();
     }
+
     Ok(chat_id)
 }
 
@@ -3438,7 +3593,7 @@ pub(crate) async fn add_contact_to_chat_ex(
         if chat.is_protected() && !contact.is_verified(context).await? {
             error!(
                 context,
-                "Only bidirectional verified contacts can be added to protected chats."
+                "Cannot add non-bidirectionally verified contact {contact_id} to protected chat {chat_id}."
             );
             return Ok(false);
         }
@@ -3566,6 +3721,7 @@ pub(crate) async fn set_muted_ex(
         .await
         .context(format!("Failed to set mute duration for {chat_id}"))?;
     context.emit_event(EventType::ChatModified(chat_id));
+    ui_events::emit_chatlist_item_changed(context, chat_id);
     if sync.into() {
         let chat = Chat::load_from_db(context, chat_id).await?;
         chat.sync(context, SyncAction::SetMuted(duration))
@@ -3726,6 +3882,7 @@ async fn rename_ex(
                 sync = Nosync;
             }
             context.emit_event(EventType::ChatModified(chat_id));
+            ui_events::emit_chatlist_item_changed(context, chat_id);
             success = true;
         }
     }
@@ -3786,6 +3943,7 @@ pub async fn set_chat_profile_image(
         context.emit_msgs_changed(chat_id, msg.id);
     }
     context.emit_event(EventType::ChatModified(chat_id));
+    ui_events::emit_chatlist_item_changed(context, chat_id);
     Ok(())
 }
 
@@ -3874,7 +4032,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
                 .prepare_msg_raw(context, &mut msg, None, curr_timestamp)
                 .await?;
             curr_timestamp += 1;
-            if create_send_msg_job(context, &mut msg).await?.is_some() {
+            if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
                 context.scheduler.interrupt_smtp().await;
             }
         }
@@ -3931,7 +4089,9 @@ pub async fn resend_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             chat_id: msg.chat_id,
             msg_id: msg.id,
         });
-        if create_send_msg_job(context, &mut msg).await?.is_some() {
+        // note(treefit): only matters if it is the last message in chat (but probably to expensive to check, debounce also solves it)
+        ui_events::emit_chatlist_item_changed(context, msg.chat_id);
+        if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
             context.scheduler.interrupt_smtp().await;
         }
     }
@@ -4242,8 +4402,8 @@ async fn set_contacts_by_addrs(context: &Context, id: ChatId, addrs: &[String])
 pub(crate) enum SyncId {
     ContactAddr(String),
     Grpid(String),
-    // NOTE: Ad-hoc groups lack an identifier that can be used across devices so
-    // block/mute/etc. actions on them are not synchronized to other devices.
+    /// "Message-ID"-s, from oldest to latest. Used for ad-hoc groups.
+    Msgids(Vec<String>),
 }
 
 /// An action synchronised to other devices.
@@ -4266,12 +4426,13 @@ impl Context {
     pub(crate) async fn sync_alter_chat(&self, id: &SyncId, action: &SyncAction) -> Result<()> {
         let chat_id = match id {
             SyncId::ContactAddr(addr) => {
-                let Some(contact_id) =
-                    Contact::lookup_id_by_addr_ex(self, addr, Origin::Unknown, None).await?
-                else {
-                    warn!(self, "sync_alter_chat: No contact for addr '{addr}'.");
+                if let SyncAction::Rename(to) = action {
+                    Contact::create_ex(self, Nosync, to, addr).await?;
                     return Ok(());
-                };
+                }
+                let contact_id = Contact::lookup_id_by_addr_ex(self, addr, Origin::Unknown, None)
+                    .await?
+                    .with_context(|| format!("No contact for addr '{addr}'"))?;
                 match action {
                     SyncAction::Block => {
                         return contact::set_blocked(self, Nosync, contact_id, true).await
@@ -4281,22 +4442,26 @@ impl Context {
                     }
                     _ => (),
                 }
-                let Some(chat_id) = ChatId::lookup_by_contact(self, contact_id).await? else {
-                    warn!(self, "sync_alter_chat: No chat for addr '{addr}'.");
-                    return Ok(());
-                };
-                chat_id
+                ChatId::lookup_by_contact(self, contact_id)
+                    .await?
+                    .with_context(|| format!("No chat for addr '{addr}'"))?
             }
             SyncId::Grpid(grpid) => {
                 if let SyncAction::CreateBroadcast(name) = action {
                     create_broadcast_list_ex(self, Nosync, grpid.clone(), name.clone()).await?;
                     return Ok(());
                 }
-                let Some((chat_id, ..)) = get_chat_id_by_grpid(self, grpid).await? else {
-                    warn!(self, "sync_alter_chat: No chat for grpid '{grpid}'.");
-                    return Ok(());
-                };
-                chat_id
+                get_chat_id_by_grpid(self, grpid)
+                    .await?
+                    .with_context(|| format!("No chat for grpid '{grpid}'"))?
+                    .0
+            }
+            SyncId::Msgids(msgids) => {
+                let msg = message::get_latest_by_rfc724_mids(self, msgids)
+                    .await?
+                    .with_context(|| format!("No message found for Message-IDs {msgids:?}"))?;
+                ChatId::lookup_by_message(&msg)
+                    .with_context(|| format!("No chat found for Message-IDs {msgids:?}"))?
             }
         };
         match action {
@@ -5775,7 +5940,7 @@ mod tests {
         // Alice has an SMTP-server replacing the `Message-ID:`-header (as done eg. by outlook.com).
         let sent_msg = alice.pop_sent_msg().await;
         let msg = sent_msg.payload();
-        assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 1);
+        assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 2);
         assert_eq!(msg.match_indices("References: <Gr.").count(), 1);
         let msg = msg.replace("Message-ID: <Gr.", "Message-ID: <XXX");
         assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 0);
@@ -6941,6 +7106,51 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync_adhoc_grp() -> Result<()> {
+        let alice0 = &TestContext::new_alice().await;
+        let alice1 = &TestContext::new_alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let mut chat_ids = Vec::new();
+        for a in [alice0, alice1] {
+            let msg = receive_imf(
+                a,
+                b"Subject: =?utf-8?q?Message_from_alice=40example=2Eorg?=\r\n\
+                    From: alice@example.org\r\n\
+                    To: <bob@example.net>, <fiona@example.org> \r\n\
+                    Date: Mon, 2 Dec 2023 16:59:39 +0000\r\n\
+                    Message-ID: <Mr.alices_original_mail@example.org>\r\n\
+                    Chat-Version: 1.0\r\n\
+                    \r\n\
+                    hi\r\n",
+                false,
+            )
+            .await?
+            .unwrap();
+            chat_ids.push(msg.chat_id);
+        }
+        let chat1 = Chat::load_from_db(alice1, chat_ids[1]).await?;
+        assert_eq!(chat1.typ, Chattype::Group);
+        assert!(chat1.grpid.is_empty());
+
+        // Test synchronisation on chat blocking because it causes chat deletion currently and thus
+        // requires generating a sync message in advance.
+        chat_ids[0].block(alice0).await?;
+        sync(alice0, alice1).await;
+        assert!(Chat::load_from_db(alice1, chat_ids[1]).await.is_err());
+        assert!(
+            !alice1
+                .sql
+                .exists("SELECT COUNT(*) FROM chats WHERE id=?", (chat_ids[1],))
+                .await?
+        );
+
+        Ok(())
+    }
+
     /// Tests syncing of chat visibility on a self-chat. This way we test:
     /// - Self-chat synchronisation.
     /// - That sync messages don't unarchive the self-chat.

src/config.rs

@@ -18,7 +18,7 @@ use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
 use crate::sync::{self, Sync::*, SyncData};
-use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
+use crate::tools::{get_abs_path, improve_single_line_input};
 
 /// The available configuration keys.
 #[derive(
@@ -343,6 +343,10 @@ pub enum Config {
     /// until `chat_id.accept()` is called.
     #[strum(props(default = "0"))]
     VerifiedOneOnOneChats,
+
+    /// Row ID of the key in the `keypairs` table
+    /// used for signatures, encryption to self and included in `Autocrypt` header.
+    KeyId,
 }
 
 impl Config {
@@ -627,8 +631,6 @@ impl Context {
     ///
     /// This should only be used by test code and during configure.
     pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
-        let old_addr = self.get_config(Config::ConfiguredAddr).await?;
-
         // add old primary address (if exists) to secondary addresses
         let mut secondary_addrs = self.get_all_self_addrs().await?;
         // never store a primary address also as a secondary
@@ -642,17 +644,6 @@ impl Context {
         self.set_config(Config::ConfiguredAddr, Some(primary_new))
             .await?;
 
-        if let Some(old_addr) = old_addr {
-            let old_addr = EmailAddress::new(&old_addr)?;
-            let old_keypair = crate::key::load_keypair(self, &old_addr).await?;
-
-            if let Some(mut old_keypair) = old_keypair {
-                old_keypair.addr = EmailAddress::new(primary_new)?;
-                crate::key::store_self_keypair(self, &old_keypair, crate::key::KeyPairUse::Default)
-                    .await?;
-            }
-        }
-
         Ok(())
     }
 

src/configure/auto_mozilla.rs

@@ -1,6 +1,7 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: <https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
+//! RFC draft: <https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html>
+//! Archived original documentation: <https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
 use std::io::BufRead;
 use std::str::FromStr;
 

src/constants.rs

@@ -5,6 +5,7 @@
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
+use tokio::time::Duration;
 
 use crate::chat::ChatId;
 
@@ -209,6 +210,16 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
 
+// If more recipients are needed in SMTP's `RCPT TO:` header, the recipient list is split into
+// chunks. This does not affect MIME's `To:` header. Can be overwritten by setting
+// `max_smtp_rcpt_to` in the provider db.
+pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
+
+/// How often UI events should be sent out / How much they should be debounced.
+/// Defines the tick rate/delay of the debounce loop for UI events in milliseconds.
+pub(crate) const UI_EVENTS_TICK_RATE: Duration = Duration::from_millis(50); // 50ms which means 20 fps
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/contact.rs

@@ -26,18 +26,19 @@ use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY}
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{load_self_public_key, DcKey};
+use crate::log::LogExt;
 use crate::login_param::LoginParam;
 use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::sql::{self, params_iter};
-use crate::sync::{self, Sync::*, SyncData};
+use crate::sync::{self, Sync::*};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
     EmailAddress,
 };
-use crate::{chat, stock_str};
+use crate::{chat, stock_str, ui_events};
 
 /// Time during which a contact is considered as seen recently.
 const SEEN_RECENTLY_SECONDS: i64 = 600;
@@ -476,6 +477,15 @@ impl Contact {
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
     pub async fn create(context: &Context, name: &str, addr: &str) -> Result<ContactId> {
+        Self::create_ex(context, Sync, name, addr).await
+    }
+
+    pub(crate) async fn create_ex(
+        context: &Context,
+        sync: sync::Sync,
+        name: &str,
+        addr: &str,
+    ) -> Result<ContactId> {
         let name = improve_single_line_input(name);
 
         let (name, addr) = sanitize_name_and_addr(&name, addr);
@@ -496,6 +506,16 @@ impl Contact {
             set_blocked(context, Nosync, contact_id, false).await?;
         }
 
+        if sync.into() {
+            chat::sync(
+                context,
+                chat::SyncId::ContactAddr(addr.to_string()),
+                chat::SyncAction::Rename(name.to_string()),
+            )
+            .await
+            .log_err(context)
+            .ok();
+        }
         Ok(contact_id)
     }
 
@@ -741,6 +761,7 @@ impl Contact {
                             if count > 0 {
                                 // Chat name updated
                                 context.emit_event(EventType::ChatModified(chat_id));
+                                ui_events::emit_chatlist_items_changed_for_contact(context, contact_id);
                             }
                         }
                     }
@@ -777,7 +798,31 @@ impl Contact {
             Ok(row_id)
         }).await?;
 
-        Ok((ContactId::new(row_id), sth_modified))
+        let contact_id = ContactId::new(row_id);
+
+        Ok((contact_id, sth_modified))
+    }
+
+    /// Get all chats the contact is part of
+    pub async fn get_chats_with_contact(
+        context: &Context,
+        contact_id: &ContactId,
+    ) -> Result<Vec<ChatId>> {
+        context
+            .sql
+            .query_map(
+                "SELECT chat_id FROM chats_contacts WHERE contact_id=?",
+                (contact_id,),
+                |row| {
+                    let chat_id: ChatId = row.get(0)?;
+                    Ok(chat_id)
+                },
+                |rows| {
+                    rows.collect::<std::result::Result<Vec<_>, _>>()
+                        .map_err(Into::into)
+                },
+            )
+            .await
     }
 
     /// Add a number of contacts.
@@ -1268,13 +1313,30 @@ impl Contact {
             return Ok(true);
         }
 
-        if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
-            if peerstate.is_using_verified_key() {
-                return Ok(true);
-            }
+        let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? else {
+            return Ok(false);
+        };
+
+        let forward_verified = peerstate.is_using_verified_key();
+        let backward_verified = peerstate.is_backward_verified(context).await?;
+        Ok(forward_verified && backward_verified)
+    }
+
+    /// Returns true if we have a verified key for the contact
+    /// and it is the same as Autocrypt key.
+    /// This is enough to send messages to the contact in verified chat
+    /// and verify received messages, but not enough to display green checkmark
+    /// or add the contact to verified groups.
+    pub async fn is_forward_verified(&self, context: &Context) -> Result<bool> {
+        if self.id == ContactId::SELF {
+            return Ok(true);
         }
 
-        Ok(false)
+        let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? else {
+            return Ok(false);
+        };
+
+        Ok(peerstate.is_using_verified_key())
     }
 
     /// Returns the `ContactId` that verified the contact.
@@ -1480,16 +1542,18 @@ WHERE type=? AND id IN (
                 true => chat::SyncAction::Block,
                 false => chat::SyncAction::Unblock,
             };
-            context
-                .add_sync_item(SyncData::AlterChat {
-                    id: chat::SyncId::ContactAddr(contact.addr.clone()),
-                    action,
-                })
-                .await?;
-            context.send_sync_msg().await?;
+            chat::sync(
+                context,
+                chat::SyncId::ContactAddr(contact.addr.clone()),
+                action,
+            )
+            .await
+            .log_err(context)
+            .ok();
         }
     }
 
+    ui_events::emit_chatlist_changed(context);
     Ok(())
 }
 
@@ -1538,6 +1602,7 @@ pub(crate) async fn set_profile_image(
     if changed {
         contact.update_param(context).await?;
         context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+        ui_events::emit_chatlist_item_changed_for_contacts_dm_chat(context, contact_id);
     }
     Ok(())
 }
@@ -1744,6 +1809,10 @@ impl RecentlySeenLoop {
                         // Timeout, notify about contact.
                         if let Some(contact_id) = contact_id {
                             context.emit_event(EventType::ContactsChanged(Some(*contact_id)));
+                            ui_events::emit_chatlist_item_changed_for_contacts_dm_chat(
+                                &context,
+                                *contact_id,
+                            );
                             unseen_queue.pop();
                         }
                     }
@@ -1773,6 +1842,10 @@ impl RecentlySeenLoop {
                 // Event is already in the past.
                 if let Some(contact_id) = contact_id {
                     context.emit_event(EventType::ContactsChanged(Some(*contact_id)));
+                    ui_events::emit_chatlist_item_changed_for_contacts_dm_chat(
+                        &context,
+                        *contact_id,
+                    );
                 }
                 unseen_queue.pop();
             }
@@ -1885,12 +1958,12 @@ mod tests {
         // Search by name.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         // Search by address.
         let contacts = Contact::get_all(&context.ctx, 0, Some("user")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         let contacts = Contact::get_all(&context.ctx, 0, Some("alice")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1917,7 +1990,7 @@ mod tests {
         // Search by display name (same as manually set name).
         let contacts = Contact::get_all(&context.ctx, 0, Some("someone")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         Ok(())
     }
@@ -2789,4 +2862,33 @@ Hi."#;
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync_create() -> Result<()> {
+        let alice0 = &TestContext::new_alice().await;
+        let alice1 = &TestContext::new_alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        Contact::create(alice0, "Bob", "bob@example.net").await?;
+        test_utils::sync(alice0, alice1).await;
+        let a1b_contact_id =
+            Contact::lookup_id_by_addr(alice1, "bob@example.net", Origin::ManuallyCreated)
+                .await?
+                .unwrap();
+        let a1b_contact = Contact::get_by_id(alice1, a1b_contact_id).await?;
+        assert_eq!(a1b_contact.name, "Bob");
+
+        Contact::create(alice0, "Bob Renamed", "bob@example.net").await?;
+        test_utils::sync(alice0, alice1).await;
+        let id = Contact::lookup_id_by_addr(alice1, "bob@example.net", Origin::ManuallyCreated)
+            .await?
+            .unwrap();
+        assert_eq!(id, a1b_contact_id);
+        let a1b_contact = Contact::get_by_id(alice1, a1b_contact_id).await?;
+        assert_eq!(a1b_contact.name, "Bob Renamed");
+
+        Ok(())
+    }
 }

src/context.rs

@@ -1,5 +1,6 @@
 //! Context module.
 
+use std::borrow::BorrowMut;
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
@@ -28,6 +29,7 @@ use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
 use crate::tools::{duration_to_str, time};
+use crate::ui_events::{self, UIEvents};
 
 /// Builder for the [`Context`].
 ///
@@ -203,6 +205,7 @@ pub struct InnerContext {
     pub(crate) wrong_pw_warning_mutex: Mutex<()>,
     pub(crate) translated_stockstrings: StockStrings,
     pub(crate) events: Events,
+    pub(crate) ui_events: Mutex<UIEvents>,
 
     pub(crate) scheduler: SchedulerState,
     pub(crate) ratelimit: RwLock<Ratelimit>,
@@ -367,6 +370,8 @@ impl Context {
         // without starting I/O.
         new_msgs_notify.notify_one();
 
+        let (ui_events, ui_events_receiver) = UIEvents::new();
+
         let inner = InnerContext {
             id,
             blobdir,
@@ -378,6 +383,7 @@ impl Context {
             wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: stockstrings,
             events,
+            ui_events: Mutex::new(ui_events),
             scheduler: SchedulerState::new(),
             ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow at least 1 message every 10 seconds + a burst of 6.
             quota: RwLock::new(None),
@@ -394,6 +400,8 @@ impl Context {
             inner: Arc::new(inner),
         };
 
+        ctx.inner.ui_events.blocking_lock().start(&ctx, ui_events_receiver);
+
         Ok(ctx)
     }
 
@@ -485,11 +493,15 @@ impl Context {
     /// Emits a MsgsChanged event with specified chat and message ids
     pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
         self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
+        ui_events::emit_chatlist_changed(self);
+        ui_events::emit_chatlist_item_changed(self, chat_id);
     }
 
     /// Emits an IncomingMsg event with specified chat and message ids
     pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
         self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
+        ui_events::emit_chatlist_changed(self);
+        ui_events::emit_chatlist_item_changed(self, chat_id);
     }
 
     /// Returns a receiver for emitted events.
@@ -1318,6 +1330,7 @@ mod tests {
             "socks5_port",
             "socks5_user",
             "socks5_password",
+            "key_id",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();
@@ -1369,7 +1382,7 @@ mod tests {
         assert_eq!(res.len(), 2);
 
         // Message added later is returned first.
-        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.first(), Some(&msg2.id));
         assert_eq!(res.get(1), Some(&msg1.id));
 
         // Global search with longer text does not find any message.
@@ -1586,7 +1599,7 @@ mod tests {
 
         let bob_next_msg_ids = bob.get_next_msgs().await?;
         assert_eq!(bob_next_msg_ids.len(), 1);
-        assert_eq!(bob_next_msg_ids.get(0), Some(&received_msg.id));
+        assert_eq!(bob_next_msg_ids.first(), Some(&received_msg.id));
 
         bob.set_config_u32(Config::LastMsgId, received_msg.id.to_u32())
             .await?;
@@ -1595,7 +1608,7 @@ mod tests {
         // Next messages include self-sent messages.
         let alice_next_msg_ids = alice.get_next_msgs().await?;
         assert_eq!(alice_next_msg_ids.len(), 1);
-        assert_eq!(alice_next_msg_ids.get(0), Some(&sent_msg.sender_msg_id));
+        assert_eq!(alice_next_msg_ids.first(), Some(&sent_msg.sender_msg_id));
 
         alice
             .set_config_u32(Config::LastMsgId, sent_msg.sender_msg_id.to_u32())

src/download.rs

@@ -13,7 +13,7 @@ use crate::imap::{Imap, ImapActionResult};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::tools::time;
-use crate::{stock_str, EventType};
+use crate::{stock_str, ui_events, EventType};
 
 /// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
 ///
@@ -115,6 +115,7 @@ impl MsgId {
             chat_id: msg.chat_id,
             msg_id: self,
         });
+        ui_events::emit_chatlist_item_changed(context, msg.chat_id);
         Ok(())
     }
 }

src/e2ee.rs

@@ -169,11 +169,10 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat;
     use crate::key::DcKey;
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
-    use crate::test_utils::{bob_keypair, TestContext};
+    use crate::test_utils::{bob_keypair, TestContext, TestContextManager};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -217,37 +216,35 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
 
         let chat_alice = alice.create_chat(&bob).await.id;
         let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
         // Bob receives unencrypted message from Alice
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
 
-        // Parsing a message is enough to update peerstate
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
-        // Bob sends encrypted message to Alice
+        // Bob sends empty encrypted message to Alice
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        let sent = bob.pop_sent_msg().await;
+        let sent = bob.send_msg(chat_bob, &mut msg).await;
 
-        // Alice receives encrypted message from Bob
-        let msg = alice.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        // Alice receives an empty encrypted message from Bob.
+        // This is also a regression test for previously existing bug
+        // that resulted in no padlock on encrypted empty messages.
+        let msg = alice.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
 
         let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
             .await?
@@ -259,12 +256,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends encrypted message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -273,12 +268,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -288,12 +281,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -321,6 +312,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
         vec![(Some(peerstate), addr)]

src/events.rs

@@ -4,6 +4,7 @@ use async_channel::{self as channel, Receiver, Sender, TrySendError};
 use pin_project::pin_project;
 
 mod payload;
+pub(crate) mod ui_events;
 
 pub use self::payload::EventType;
 

src/events/payload.rs

@@ -277,4 +277,16 @@ pub enum EventType {
         /// ID of the deleted message.
         msg_id: MsgId,
     },
+
+    /// Inform UI that Order (and content as in chat ids) of the chatlist changed.
+    ///
+    /// Sometimes this is emitted together with `UIChatListItemChanged` such as on IncomingMessage.
+    UIChatListChanged,
+
+    /// Inform UI that a single chat list item changed and needs to be rerendered
+    /// If `chat_id` is set to None, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
+    UIChatListItemChanged {
+        /// ID of the changed chat
+        chat_id: Option<ChatId>,
+    },
 }

src/events/ui_events.rs

@@ -0,0 +1,253 @@
+use crate::{
+    chat::{ChatId, ChatIdBlocked},
+    constants::UI_EVENTS_TICK_RATE,
+    contact::{Contact, ContactId},
+    context::Context,
+    EventType,
+};
+use async_channel::{self as channel, Receiver, Sender};
+use tokio::{
+    task,
+    time::{sleep_until, Instant},
+};
+
+/// order or content of chatlist changes (chat ids, not the actual chatlist item)
+pub(crate) fn emit_chatlist_changed(context: &Context) {
+    context
+        .ui_events
+        .blocking_lock()
+        .send_chat_list_event(context, InternalUIEvent::ChatListChanged)
+}
+
+/// Chatlist item of a specific chat changed
+pub(crate) fn emit_chatlist_item_changed(context: &Context, chat_id: ChatId) {
+    context
+        .ui_events
+        .blocking_lock()
+        .send_chat_list_event(context, InternalUIEvent::ChatListItemChanged(chat_id))
+}
+
+#[allow(unused)]
+/// Used when you don't know which chatlist items changed, this reloads all cached chatlist items in the UI
+/// note(treefit): This is not used right now, but I know there will be a point where someone wants it
+pub(crate) fn emit_unknown_chatlist_items_changed(context: &Context) {
+    context
+        .ui_events
+        .blocking_lock()
+        .send_chat_list_event(context, InternalUIEvent::UnknownChatListItemsChanged)
+}
+
+/// update event for dm chat of contact
+/// used when recently seen changes and when profile image changes
+pub(crate) fn emit_chatlist_item_changed_for_contacts_dm_chat(
+    context: &Context,
+    contact_id: ContactId,
+) {
+    context
+        .ui_events
+        .blocking_lock()
+        .send_chat_list_event(context, InternalUIEvent::ContactDMChatChanged(contact_id))
+}
+
+/// update dm for chats that have the contact
+/// used when contact changes their name or did AEAP for example
+pub(crate) fn emit_chatlist_items_changed_for_contact(context: &Context, contact_id: ContactId) {
+    context
+        .ui_events
+        .blocking_lock()
+        .send_chat_list_event(context, InternalUIEvent::ContactChatsChanged(contact_id));
+}
+
+#[derive(Debug)]
+pub(crate) enum InternalUIEvent {
+    ChatListChanged,
+    ChatListItemChanged(ChatId),
+    UnknownChatListItemsChanged,
+    ContactDMChatChanged(ContactId),
+    ContactChatsChanged(ContactId),
+}
+
+struct EventLoopTickState {
+    chat_list_changed: bool,
+    has_unknown_items: bool,
+    chat_ids: Vec<ChatId>,
+    contact_ids_dm: Vec<ContactId>,
+    contact_ids_chats: Vec<ContactId>,
+}
+
+impl EventLoopTickState {
+    fn new(capacity: usize) -> Self {
+        Self {
+            chat_list_changed: false,
+            has_unknown_items: false,
+            chat_ids: Vec::with_capacity(capacity),
+            contact_ids_dm: Vec::with_capacity(capacity),
+            contact_ids_chats: Vec::with_capacity(capacity),
+        }
+    }
+
+    fn apply_internal_ui_event(&mut self, event: InternalUIEvent) {
+        match event {
+            InternalUIEvent::ChatListChanged => {
+                self.chat_list_changed = true;
+            }
+            InternalUIEvent::ChatListItemChanged(chat_id) => {
+                self.chat_ids.push(chat_id);
+            }
+            InternalUIEvent::UnknownChatListItemsChanged => {
+                self.has_unknown_items = true;
+            }
+            InternalUIEvent::ContactDMChatChanged(contact_id) => {
+                self.contact_ids_dm.push(contact_id);
+            }
+            InternalUIEvent::ContactChatsChanged(contact_id) => {
+                self.contact_ids_chats.push(contact_id);
+            }
+        }
+    }
+
+    async fn emit_chatlist_ui_events(&mut self, context: &Context) {
+        if self.chat_list_changed {
+            context.emit_event(EventType::UIChatListChanged);
+        }
+        if self.has_unknown_items {
+            context.emit_event(EventType::UIChatListItemChanged { chat_id: None });
+            return; // since this refreshes everything no further events are needed
+        }
+
+        for contact_id in self
+            .contact_ids_dm
+            .iter()
+            .filter(|contact| !self.contact_ids_chats.contains(contact))
+            .collect::<Vec<&ContactId>>()
+        {
+            if let Ok(Some(chat_id)) = ChatIdBlocked::lookup_by_contact(context, *contact_id).await
+            {
+                self.chat_ids.push(chat_id.id)
+            }
+        }
+
+        // note:(treefit): could make sense to only update chats where the last message is from the contact, but the db query for that is more expensive
+        for contact_id in &self.contact_ids_chats {
+            match Contact::get_chats_with_contact(context, contact_id).await {
+                Ok(contacts_chat_ids) => {
+                    self.chat_ids.extend(contacts_chat_ids);
+                }
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Error while getting chats for contact {} in chatlist events loop: {}",
+                        contact_id,
+                        err
+                    );
+                }
+            }
+        }
+
+        self.chat_ids.sort();
+        self.chat_ids.dedup();
+
+        // TODO change event so it accepts a list of chat ids to get rid of this loop? wouldn't work with cffi unless we give it out as json
+        for chat_id in &self.chat_ids {
+            context.emit_event(EventType::UIChatListItemChanged {
+                chat_id: Some(*chat_id),
+            })
+        }
+    }
+}
+
+/// Debounces UI events
+#[derive(Debug)]
+pub(crate) struct UIEvents {
+    task_handle: Option<task::JoinHandle<()>>,
+    chatlist_event_queue: Sender<InternalUIEvent>,
+}
+
+impl UIEvents {
+    pub(crate) fn new() -> (Self, Receiver<InternalUIEvent>) {
+        let (chatlist_event_queue, chatlist_event_queue_recv) = channel::unbounded();
+        (
+            Self {
+                task_handle: None,
+                chatlist_event_queue,
+            },
+            chatlist_event_queue_recv,
+        )
+    }
+
+    pub(crate) fn start(
+        &mut self,
+        context: &Context,
+        chatlist_event_queue_recv: Receiver<InternalUIEvent>,
+    ) {
+        if let Some(handle) = self.task_handle {
+            handle.abort()
+        }
+        self.task_handle = Some(task::spawn(Self::run_task(
+            context,
+            chatlist_event_queue_recv,
+        )))
+    }
+
+    async fn run_task(context: &Context, chatlist_event_queue: Receiver<InternalUIEvent>) {
+        loop {
+            match chatlist_event_queue.recv().await {
+                Ok(chatlist_event) => {
+                    let backlog_len = chatlist_event_queue.len();
+                    let mut tick_state = EventLoopTickState::new(backlog_len);
+
+                    tick_state.apply_internal_ui_event(chatlist_event);
+                    // get all events from the queue
+                    while let Ok(event) = chatlist_event_queue.try_recv() {
+                        tick_state.apply_internal_ui_event(event);
+                    }
+
+                    tick_state.emit_chatlist_ui_events(context).await;
+
+                    // cooldown
+                    sleep_until(Instant::now() + UI_EVENTS_TICK_RATE).await;
+                }
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Error receiving an interruption in ui chatlist events loop: {}", err
+                    );
+                    // Maybe the sender side is closed, so terminate the loop to avoid looping indefinitely.
+                    return;
+                }
+            }
+        }
+    }
+
+    pub(crate) fn send_chat_list_event(&self, context: &Context, event: InternalUIEvent) {
+        // todo check if ui events are enabled?
+        if let Err(error) = self.chatlist_event_queue.try_send(event) {
+            warn!(
+                context,
+                "Error receiving an interruption in ui chatlist events loop: {}", error
+            );
+        }
+    }
+}
+
+impl Drop for UIEvents {
+    fn drop(&mut self) {
+        if let Some(handle) = &self.task_handle {
+            handle.abort()
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    // todo tests:
+
+    // send ui events though the UIEventsLoop
+
+    // check that UIEventsLoop really ratelimits the events
+
+    // check that has_unknown_items does not send out any ids before or afterwards
+
+    // if we should make it possible to disable via config then test that as well
+}

src/headerdef.rs

@@ -38,6 +38,9 @@ pub enum HeaderDef {
     /// Mailing list ID defined in [RFC 2919](https://tools.ietf.org/html/rfc2919).
     ListId,
     ListPost,
+
+    /// List-Help header defined in [RFC 2369](https://datatracker.ietf.org/doc/html/rfc2369).
+    ListHelp,
     References,
 
     /// In-Reply-To header containing Message-ID of the parent message.

src/imap.rs

@@ -42,6 +42,7 @@ use crate::socks::Socks5Config;
 use crate::sql;
 use crate::stock_str;
 use crate::tools::{create_id, duration_to_str};
+use crate::ui_events;
 
 pub(crate) mod capabilities;
 mod client;
@@ -94,6 +95,20 @@ pub struct Imap {
     login_failed_once: bool,
 
     pub(crate) connectivity: ConnectivityStore,
+
+    /// Rate limit for IMAP connection usage attempts.
+    ///
+    /// Rate limit is checked before connecting
+    /// and updated right before login attempt.
+    /// It does not limit the number of connection attempts
+    /// if the network is bad as only successful connections are counted.
+    ///
+    /// Main purpose of this rate limit is
+    /// to prevent busy loop in case
+    /// connection gets dropped over and over due to IMAP bug,
+    /// e.g. the server returning invalid response to SELECT command
+    /// immediately after logging in or returning an error in response to LOGIN command
+    /// due to internal server error.
     ratelimit: RwLock<Ratelimit>,
 }
 
@@ -256,7 +271,7 @@ impl Imap {
             session: None,
             login_failed_once: false,
             connectivity: Default::default(),
-            // 1 connection per minute + a burst of 2.
+            // 1 login per minute + a burst of 2.
             ratelimit: RwLock::new(Ratelimit::new(Duration::new(120, 0), 2.0)),
         };
 
@@ -306,6 +321,12 @@ impl Imap {
         }
 
         self.connectivity.set_connecting(context).await;
+
+        // Check rate limit before trying to connect
+        // to avoid connecting and not using the connection
+        // in case we are currently ratelimited.
+        // Otherwise connection may become unusable due to NAT forgetting about it
+        // before we attempt to actually login.
         let ratelimit_duration = self.ratelimit.read().await.until_can_send();
         if !ratelimit_duration.is_zero() {
             warn!(
@@ -316,10 +337,7 @@ impl Imap {
             tokio::time::sleep(ratelimit_duration).await;
         }
 
-        let oauth2 = self.config.lp.oauth2;
-
         info!(context, "Connecting to IMAP server");
-        self.ratelimit.write().await.send();
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
         {
@@ -369,11 +387,13 @@ impl Imap {
                 Client::connect_secure(context, imap_server, imap_port, config.strict_tls).await
             }
         };
-
         let client = connection_res?;
+        self.ratelimit.write().await.send();
+
         let config = &self.config;
         let imap_user: &str = config.lp.user.as_ref();
         let imap_pw: &str = config.lp.password.as_ref();
+        let oauth2 = self.config.lp.oauth2;
 
         let login_res = if oauth2 {
             info!(context, "Logging into IMAP server with OAuth 2");
@@ -975,7 +995,7 @@ impl Imap {
         self.prepare(context).await?;
 
         let all_folders = self
-            .list_folders(context)
+            .list_folders()
             .await
             .context("listing folders for resync")?;
         for folder in all_folders {
@@ -1300,6 +1320,7 @@ impl Imap {
             .with_context(|| format!("failed to set MODSEQ for folder {folder}"))?;
         for updated_chat_id in updated_chat_ids {
             context.emit_event(EventType::MsgsNoticed(updated_chat_id));
+            ui_events::emit_chatlist_item_changed(context, updated_chat_id);
         }
 
         Ok(())

src/imap/idle.rs

@@ -12,7 +12,13 @@ use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
 use crate::log::LogExt;
 
-const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
+/// Timeout after which IDLE is finished
+/// if there are no responses from the server.
+///
+/// If `* OK Still here` keepalives are sent more frequently
+/// than this duration, timeout should never be triggered.
+/// For example, Dovecot sends keepalives every 2 minutes by default.
+const IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
 
 impl Session {
     pub async fn idle(

src/imap/scan_folders.rs

@@ -1,7 +1,7 @@
 use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
-use futures::stream::StreamExt;
+use futures::TryStreamExt;
 
 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
@@ -27,7 +27,7 @@ impl Imap {
         info!(context, "Starting full folder scan");
 
         self.prepare(context).await?;
-        let folders = self.list_folders(context).await?;
+        let folders = self.list_folders().await?;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
@@ -98,21 +98,15 @@ impl Imap {
     }
 
     /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(
-        self: &mut Imap,
-        context: &Context,
-    ) -> Result<Vec<async_imap::types::Name>> {
+    pub async fn list_folders(self: &mut Imap) -> Result<Vec<async_imap::types::Name>> {
         let session = self.session.as_mut();
         let session = session.context("No IMAP connection")?;
         let list = session
             .list(Some(""), Some("*"))
             .await?
-            .filter_map(|f| async {
-                f.context("list_folders() can't get folder")
-                    .log_err(context)
-                    .ok()
-            });
-        Ok(list.collect().await)
+            .try_collect()
+            .await?;
+        Ok(list)
     }
 }
 

src/imex.rs

@@ -375,7 +375,15 @@ async fn imex_inner(
     path: &Path,
     passphrase: Option<String>,
 ) -> Result<()> {
-    info!(context, "Import/export dir: {}", path.display());
+    info!(
+        context,
+        "{} path: {}",
+        match what {
+            ImexMode::ExportSelfKeys | ImexMode::ExportBackup => "Export",
+            ImexMode::ImportSelfKeys | ImexMode::ImportBackup => "Import",
+        },
+        path.display()
+    );
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
@@ -670,7 +678,7 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let keys = context
         .sql
         .query_map(
-            "SELECT id, public_key, private_key, is_default FROM keypairs;",
+            "SELECT id, public_key, private_key, id=(SELECT value FROM config WHERE keyname='key_id') FROM keypairs;",
             (),
             |row| {
                 let id = row.get(0)?;

src/imex/transfer.rs

@@ -638,7 +638,7 @@ mod tests {
         let self_chat = ctx1.get_self_chat().await;
         let msgs = get_chat_msgs(&ctx1, self_chat.id).await.unwrap();
         assert_eq!(msgs.len(), 2);
-        let msgid = match msgs.get(0).unwrap() {
+        let msgid = match msgs.first().unwrap() {
             ChatItem::Message { msg_id } => msg_id,
             _ => panic!("wrong chat item"),
         };

src/key.rs

@@ -18,7 +18,7 @@ use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::log::LogExt;
 use crate::pgp::KeyPair;
-use crate::tools::{time, EmailAddress};
+use crate::tools::EmailAddress;
 
 /// Convenience trait for working with keys.
 ///
@@ -82,10 +82,9 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
     match context
         .sql
         .query_row_optional(
-            r#"SELECT public_key
-               FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1"#,
+            "SELECT public_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
             (),
             |row| {
                 let bytes: Vec<u8> = row.get(0)?;
@@ -106,10 +105,9 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
     match context
         .sql
         .query_row_optional(
-            r#"SELECT private_key
-               FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1"#,
+            "SELECT private_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
             (),
             |row| {
                 let bytes: Vec<u8> = row.get(0)?;
@@ -132,8 +130,7 @@ pub(crate) async fn load_self_secret_keyring(context: &Context) -> Result<Vec<Si
         .query_map(
             r#"SELECT private_key
                FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               ORDER BY is_default DESC"#,
+               ORDER BY id=(SELECT value FROM config WHERE keyname='key_id') DESC"#,
             (),
             |row| row.get::<_, Vec<u8>>(0),
             |keys| keys.collect::<Result<Vec<_>, _>>().map_err(Into::into),
@@ -233,13 +230,10 @@ pub(crate) async fn load_keypair(
     let res = context
         .sql
         .query_row_optional(
-            r#"
-        SELECT public_key, private_key
-          FROM keypairs
-         WHERE addr=?1
-           AND is_default=1;
-        "#,
-            (addr,),
+            "SELECT public_key, private_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
+            (),
             |row| {
                 let pub_bytes: Vec<u8> = row.get(0)?;
                 let sec_bytes: Vec<u8> = row.get(1)?;
@@ -288,42 +282,44 @@ pub async fn store_self_keypair(
     keypair: &KeyPair,
     default: KeyPairUse,
 ) -> Result<()> {
-    context
+    let mut config_cache_lock = context.sql.config_cache.write().await;
+    let new_key_id = context
         .sql
         .transaction(|transaction| {
             let public_key = DcKey::to_bytes(&keypair.public);
             let secret_key = DcKey::to_bytes(&keypair.secret);
-            transaction
-                .execute(
-                    "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-                    (&public_key, &secret_key),
-                )
-                .context("failed to remove old use of key")?;
-            if default == KeyPairUse::Default {
-                transaction
-                    .execute("UPDATE keypairs SET is_default=0;", ())
-                    .context("failed to clear default")?;
-            }
+
             let is_default = match default {
-                KeyPairUse::Default => i32::from(true),
-                KeyPairUse::ReadOnly => i32::from(false),
+                KeyPairUse::Default => true,
+                KeyPairUse::ReadOnly => false,
             };
 
-            let addr = keypair.addr.to_string();
-            let t = time();
-
             transaction
                 .execute(
-                    "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
-                VALUES (?,?,?,?,?);",
-                    (addr, is_default, &public_key, &secret_key, t),
+                    "INSERT OR REPLACE INTO keypairs (public_key, private_key)
+                     VALUES (?,?)",
+                    (&public_key, &secret_key),
                 )
-                .context("failed to insert keypair")?;
+                .context("Failed to insert keypair")?;
 
-            Ok(())
+            if is_default {
+                let new_key_id = transaction.last_insert_rowid();
+                transaction.execute(
+                    "INSERT OR REPLACE INTO config (keyname, value) VALUES ('key_id', ?)",
+                    (new_key_id,),
+                )?;
+                Ok(Some(new_key_id))
+            } else {
+                Ok(None)
+            }
         })
         .await?;
 
+    if let Some(new_key_id) = new_key_id {
+        // Update config cache if transaction succeeded and changed current default key.
+        config_cache_lock.insert("key_id".to_string(), Some(new_key_id.to_string()));
+    }
+
     Ok(())
 }
 

src/location.rs

@@ -14,8 +14,8 @@ use crate::context::Context;
 use crate::events::EventType;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
-use crate::stock_str;
 use crate::tools::{duration_to_str, time};
+use crate::{stock_str, ui_events};
 
 /// Location record.
 #[derive(Debug, Clone, Default)]
@@ -290,6 +290,7 @@ pub async fn send_locations_to_chat(
         chat::add_info_msg(context, chat_id, &stock_str, now).await?;
     }
     context.emit_event(EventType::ChatModified(chat_id));
+    ui_events::emit_chatlist_item_changed(context, chat_id);
     if 0 != seconds {
         context.scheduler.interrupt_location().await;
     }
@@ -787,6 +788,7 @@ async fn maybe_send_locations(context: &Context) -> Result<Option<u64>> {
             let stock_str = stock_str::msg_location_disabled(context).await;
             chat::add_info_msg(context, chat_id, &stock_str, now).await?;
             context.emit_event(EventType::ChatModified(chat_id));
+            ui_events::emit_chatlist_item_changed(context, chat_id);
         }
     }
 
@@ -954,7 +956,7 @@ Content-Disposition: attachment; filename="location.kml"
         assert!(msg.chat_id == bob_chat_id);
         assert_eq!(msg.msg_ids.len(), 1);
 
-        let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.get(0).unwrap()).await?;
+        let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.first().unwrap()).await?;
         assert_eq!(bob_msg.chat_id, bob_chat_id);
         assert_eq!(bob_msg.viewtype, Viewtype::Image);
 

src/message.rs

@@ -30,6 +30,7 @@ use crate::tools::{
     buf_compress, buf_decompress, get_filebytes, get_filemeta, gm2local_offset, read_file, time,
     timestamp_to_str, truncate,
 };
+use crate::ui_events;
 
 /// Message ID, including reserved IDs.
 ///
@@ -138,6 +139,7 @@ WHERE id=?;
             chat_id,
             msg_id: self,
         });
+        ui_events::emit_chatlist_item_changed(context, chat_id);
         Ok(())
     }
 
@@ -1518,9 +1520,12 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
 
     for modified_chat_id in modified_chat_ids {
         context.emit_msgs_changed(modified_chat_id, MsgId::new(0));
+        ui_events::emit_chatlist_item_changed(context, modified_chat_id);
     }
 
     if !msg_ids.is_empty() {
+        context.emit_msgs_changed_without_ids();
+        ui_events::emit_chatlist_changed(context);
         // Run housekeeping to delete unused blobs.
         context.set_config(Config::LastHousekeeping, None).await?;
     }
@@ -1653,6 +1658,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
 
     for updated_chat_id in updated_chat_ids {
         context.emit_event(EventType::MsgsNoticed(updated_chat_id));
+        ui_events::emit_chatlist_item_changed(context, updated_chat_id);
     }
 
     Ok(())
@@ -1663,9 +1669,17 @@ pub(crate) async fn update_msg_state(
     msg_id: MsgId,
     state: MessageState,
 ) -> Result<()> {
+    ensure!(state != MessageState::OutFailed, "use set_msg_failed()!");
+    let error_subst = match state >= MessageState::OutPending {
+        true => ", error=''",
+        false => "",
+    };
     context
         .sql
-        .execute("UPDATE msgs SET state=? WHERE id=?;", (state, msg_id))
+        .execute(
+            &format!("UPDATE msgs SET state=?1 {error_subst} WHERE id=?2 AND (?1!=?3 OR state<?3)"),
+            (state, msg_id, MessageState::OutDelivered),
+        )
         .await?;
     Ok(())
 }
@@ -1705,6 +1719,7 @@ pub(crate) async fn set_msg_failed(
         chat_id: msg.chat_id,
         msg_id: msg.id,
     });
+    ui_events::emit_chatlist_item_changed(context, msg.chat_id);
 
     Ok(())
 }
@@ -1842,6 +1857,24 @@ pub(crate) async fn rfc724_mid_exists_and(
     Ok(res)
 }
 
+/// Given a list of Message-IDs, returns the latest message found in the database.
+///
+/// Only messages that are not in the trash chat are considered.
+pub(crate) async fn get_latest_by_rfc724_mids(
+    context: &Context,
+    mids: &[String],
+) -> Result<Option<Message>> {
+    for id in mids.iter().rev() {
+        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
+            let msg = Message::load_from_db(context, msg_id).await?;
+            if msg.chat_id != DC_CHAT_ID_TRASH {
+                return Ok(Some(msg));
+            }
+        }
+    }
+    Ok(None)
+}
+
 /// How a message is primarily displayed.
 #[derive(
     Debug,

src/mimefactory.rs

@@ -1,5 +1,6 @@
 //! # MIME message production.
 
+use std::collections::HashSet;
 use std::convert::TryInto;
 
 use anyhow::{bail, ensure, Context as _, Result};
@@ -123,7 +124,8 @@ struct MessageHeaders {
     /// Headers that MUST NOT go into IMF header section.
     ///
     /// These are large headers which may hit the header section size limit on the server, such as
-    /// Chat-User-Avatar with a base64-encoded image inside.
+    /// Chat-User-Avatar with a base64-encoded image inside. Also there are headers duplicated here
+    /// that servers mess up with in the IMF header section, like Message-ID.
     pub hidden: Vec<Header>,
 }
 
@@ -517,6 +519,7 @@ impl<'a> MimeFactory<'a> {
         // <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.1.1>.
         let from_header = Header::new_with_value("From".into(), vec![from]).unwrap();
         headers.unprotected.push(from_header.clone());
+        headers.protected.push(from_header);
 
         if let Some(sender_displayname) = &self.sender_displayname {
             let sender =
@@ -558,24 +561,9 @@ impl<'a> MimeFactory<'a> {
             Loaded::Mdn { .. } => create_outgoing_rfc724_mid(None, &self.from_addr),
         };
         let rfc724_mid_headervalue = render_rfc724_mid(&rfc724_mid);
-
-        // Amazon's SMTP servers change the `Message-ID`, just as Outlook's SMTP servers do.
-        // Outlook's servers add an `X-Microsoft-Original-Message-ID` header with the original `Message-ID`,
-        // and when downloading messages we look for this header in order to correctly identify
-        // messages.
-        // Amazon's servers do not add such a header, so we just add it ourselves.
-        if let Some(server) = context.get_config(Config::ConfiguredSendServer).await? {
-            if server.ends_with(".amazonaws.com") {
-                headers.unprotected.push(Header::new(
-                    "X-Microsoft-Original-Message-ID".into(),
-                    rfc724_mid_headervalue.clone(),
-                ))
-            }
-        }
-
-        headers
-            .unprotected
-            .push(Header::new("Message-ID".into(), rfc724_mid_headervalue));
+        let rfc724_mid_header = Header::new("Message-ID".into(), rfc724_mid_headervalue);
+        headers.unprotected.push(rfc724_mid_header.clone());
+        headers.hidden.push(rfc724_mid_header);
 
         // Reply headers as in <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.2>.
         if !self.in_reply_to.is_empty() {
@@ -704,8 +692,6 @@ impl<'a> MimeFactory<'a> {
             )
         };
         let outer_message = if is_encrypted {
-            headers.protected.push(from_header);
-
             // Store protected headers in the inner message.
             let message = headers
                 .protected
@@ -782,30 +768,53 @@ impl<'a> MimeFactory<'a> {
                         .build(),
                 )
                 .header(("Subject".to_string(), "...".to_string()))
-        } else {
-            let message = if headers.hidden.is_empty() {
-                message
-            } else {
-                // Store hidden headers in the inner unencrypted message.
-                let message = headers
-                    .hidden
-                    .into_iter()
-                    .fold(message, |message, header| message.header(header));
+        } else if matches!(self.loaded, Loaded::Mdn { .. }) {
+            // Never add outer multipart/mixed wrapper to MDN
+            // as multipart/report Content-Type is used to recognize MDNs
+            // by Delta Chat receiver and Chatmail servers
+            // allowing them to be unencrypted and not contain Autocrypt header
+            // without resetting Autocrypt encryption or triggering Chatmail filter
+            // that normally only allows encrypted mails.
 
-                PartBuilder::new()
-                    .message_type(MimeMultipartType::Mixed)
-                    .child(message.build())
-            };
+            // Hidden headers are dropped.
 
             // Store protected headers in the outer message.
             let message = headers
                 .protected
+                .iter()
+                .fold(message, |message, header| message.header(header.clone()));
+
+            let protected: HashSet<Header> = HashSet::from_iter(headers.protected.into_iter());
+            for h in headers.unprotected.split_off(0) {
+                if !protected.contains(&h) {
+                    headers.unprotected.push(h);
+                }
+            }
+
+            message
+        } else {
+            // Store hidden headers in the inner unencrypted message.
+            let message = headers
+                .hidden
                 .into_iter()
                 .fold(message, |message, header| message.header(header));
+            let message = PartBuilder::new()
+                .message_type(MimeMultipartType::Mixed)
+                .child(message.build());
 
-            if self.should_skip_autocrypt()
-                || !context.get_config_bool(Config::SignUnencrypted).await?
-            {
+            // Store protected headers in the outer message.
+            let message = headers
+                .protected
+                .iter()
+                .fold(message, |message, header| message.header(header.clone()));
+
+            if skip_autocrypt || !context.get_config_bool(Config::SignUnencrypted).await? {
+                let protected: HashSet<Header> = HashSet::from_iter(headers.protected.into_iter());
+                for h in headers.unprotected.split_off(0) {
+                    if !protected.contains(&h) {
+                        headers.unprotected.push(h);
+                    }
+                }
                 message
             } else {
                 let message = message.header(get_content_type_directives_header());
@@ -994,17 +1003,6 @@ impl<'a> MimeFactory<'a> {
                             "Secure-Join".to_string(),
                             "vg-member-added".to_string(),
                         ));
-                        // FIXME: Old clients require Secure-Join-Fingerprint header. Remove this
-                        // eventually.
-                        let fingerprint = Peerstate::from_addr(context, email_to_add)
-                            .await?
-                            .context("No peerstate found in db")?
-                            .public_key_fingerprint
-                            .context("No public key fingerprint in db for the member to add")?;
-                        headers.protected.push(Header::new(
-                            "Secure-Join-Fingerprint".into(),
-                            fingerprint.hex(),
-                        ));
                     }
                 }
                 SystemMessage::GroupNameChanged => {
@@ -2159,33 +2157,37 @@ mod tests {
         let body = payload.next().unwrap();
 
         assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(outer.match_indices("Message-ID:").count(), 1);
         assert_eq!(outer.match_indices("Subject:").count(), 1);
         assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
         assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
 
         assert_eq!(inner.match_indices("text/plain").count(), 1);
+        assert_eq!(inner.match_indices("Message-ID:").count(), 1);
         assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 1);
         assert_eq!(inner.match_indices("Subject:").count(), 0);
 
         assert_eq!(body.match_indices("this is the text!").count(), 1);
 
         // if another message is sent, that one must not contain the avatar
-        // and no artificial multipart/mixed nesting
         let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(2, "\r\n\r\n");
+        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
         let outer = payload.next().unwrap();
+        let inner = payload.next().unwrap();
         let body = payload.next().unwrap();
 
-        assert_eq!(outer.match_indices("text/plain").count(), 1);
+        assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(outer.match_indices("Message-ID:").count(), 1);
         assert_eq!(outer.match_indices("Subject:").count(), 1);
         assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
-        assert_eq!(outer.match_indices("multipart/mixed").count(), 0);
         assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
 
+        assert_eq!(inner.match_indices("text/plain").count(), 1);
+        assert_eq!(inner.match_indices("Message-ID:").count(), 1);
+        assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 0);
+        assert_eq!(inner.match_indices("Subject:").count(), 0);
+
         assert_eq!(body.match_indices("this is the text!").count(), 1);
-        assert_eq!(body.match_indices("text/plain").count(), 0);
-        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
-        assert_eq!(body.match_indices("Subject:").count(), 0);
 
         Ok(())
     }
@@ -2216,6 +2218,8 @@ mod tests {
 
         let part = payload.next().unwrap();
         assert_eq!(part.match_indices("multipart/signed").count(), 1);
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
         assert_eq!(part.match_indices("Subject:").count(), 0);
         assert_eq!(part.match_indices("Autocrypt:").count(), 1);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
@@ -2226,12 +2230,16 @@ mod tests {
                 .count(),
             1
         );
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 0);
         assert_eq!(part.match_indices("Subject:").count(), 1);
         assert_eq!(part.match_indices("Autocrypt:").count(), 0);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
         let part = payload.next().unwrap();
         assert_eq!(part.match_indices("text/plain").count(), 1);
+        assert_eq!(part.match_indices("From:").count(), 0);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1);
         assert_eq!(part.match_indices("Subject:").count(), 0);
 
@@ -2252,28 +2260,38 @@ mod tests {
             .is_some());
 
         // if another message is sent, that one must not contain the avatar
-        // and no artificial multipart/mixed nesting
         let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
+        let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n");
 
         let part = payload.next().unwrap();
         assert_eq!(part.match_indices("multipart/signed").count(), 1);
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
         assert_eq!(part.match_indices("Subject:").count(), 0);
         assert_eq!(part.match_indices("Autocrypt:").count(), 1);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
         let part = payload.next().unwrap();
-        assert_eq!(part.match_indices("text/plain").count(), 1);
+        assert_eq!(
+            part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+                .count(),
+            1
+        );
+        assert_eq!(part.match_indices("From:").count(), 1);
+        assert_eq!(part.match_indices("Message-ID:").count(), 0);
         assert_eq!(part.match_indices("Subject:").count(), 1);
         assert_eq!(part.match_indices("Autocrypt:").count(), 0);
-        assert_eq!(part.match_indices("multipart/mixed").count(), 0);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
+        let part = payload.next().unwrap();
+        assert_eq!(part.match_indices("text/plain").count(), 1);
+        assert_eq!(body.match_indices("From:").count(), 0);
+        assert_eq!(part.match_indices("Message-ID:").count(), 1);
+        assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+        assert_eq!(part.match_indices("Subject:").count(), 0);
+
         let body = payload.next().unwrap();
         assert_eq!(body.match_indices("this is the text!").count(), 1);
-        assert_eq!(body.match_indices("text/plain").count(), 0);
-        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
-        assert_eq!(body.match_indices("Subject:").count(), 0);
 
         bob.recv_msg(&sent_msg).await;
         let alice_contact = Contact::get_by_id(&bob.ctx, alice_id).await.unwrap();

src/mimeparser.rs

@@ -1,5 +1,6 @@
 //! # MIME message parsing module.
 
+use std::cmp::min;
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::path::Path;
@@ -34,13 +35,12 @@ use crate::message::{
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::{simplify, SimplifiedText};
-use crate::stock_str;
 use crate::sync::SyncItems;
 use crate::tools::{
-    create_smeared_timestamp, get_filemeta, parse_receive_headers, strip_rtlo_characters,
-    truncate_by_lines,
+    create_smeared_timestamp, get_filemeta, parse_receive_headers, smeared_time,
+    strip_rtlo_characters, truncate_by_lines,
 };
-use crate::{location, tools};
+use crate::{location, stock_str, tools, ui_events};
 
 /// A parsed MIME message.
 ///
@@ -118,6 +118,12 @@ pub(crate) struct MimeMessage {
 
     /// Whether the contact sending this should be marked as bot.
     pub(crate) is_bot: bool,
+
+    /// When the message was received, in secs since epoch.
+    pub(crate) timestamp_rcvd: i64,
+    /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized
+    /// clocks, but not too much.
+    pub(crate) timestamp_sent: i64,
 }
 
 #[derive(Debug, PartialEq)]
@@ -386,6 +392,12 @@ impl MimeMessage {
         // Auto-submitted is also set by holiday-notices so we also check `chat-version`
         let is_bot = headers.contains_key("auto-submitted") && headers.contains_key("chat-version");
 
+        let timestamp_rcvd = smeared_time(context);
+        let timestamp_sent = headers
+            .get(HeaderDef::Date.get_headername())
+            .and_then(|value| mailparse::dateparse(value).ok())
+            .map_or(timestamp_rcvd, |value| min(value, timestamp_rcvd + 60));
+
         let mut parser = MimeMessage {
             parts: Vec::new(),
             headers,
@@ -415,6 +427,8 @@ impl MimeMessage {
             decoded_data: Vec::new(),
             hop_info,
             is_bot,
+            timestamp_rcvd,
+            timestamp_sent,
         };
 
         match partial {
@@ -461,20 +475,6 @@ impl MimeMessage {
             parser.decoded_data = mail_raw;
         }
 
-        crate::peerstate::maybe_do_aeap_transition(context, &mut parser).await?;
-        if let Some(peerstate) = &parser.decryption_info.peerstate {
-            peerstate
-                .handle_fingerprint_change(context, message_time)
-                .await?;
-            // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
-            // of an error. Otherwise we don't save peerstate until get here to reduce the number of
-            // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
-            // successfully.
-            if peerstate.prefer_encrypt != EncryptPreference::Mutual {
-                peerstate.save_to_db(&context.sql).await?;
-            }
-        }
-
         Ok(parser)
     }
 
@@ -694,7 +694,7 @@ impl MimeMessage {
                 }
             }
 
-            self.parts.push(part);
+            self.do_add_single_part(part);
         }
 
         if self.headers.contains_key("auto-submitted") {
@@ -913,7 +913,7 @@ impl MimeMessage {
                 skip the rest.  (see
                 <https://k9mail.app/2016/11/24/OpenPGP-Considerations-Part-I.html>
                 for background information why we use encrypted+signed) */
-                if let Some(first) = mail.subparts.get(0) {
+                if let Some(first) = mail.subparts.first() {
                     any_part_added = self
                         .parse_mime_recursive(context, first, is_related)
                         .await?;
@@ -969,7 +969,7 @@ impl MimeMessage {
                             }
                         }
                         Some(_) => {
-                            if let Some(first) = mail.subparts.get(0) {
+                            if let Some(first) = mail.subparts.first() {
                                 any_part_added = self
                                     .parse_mime_recursive(context, first, is_related)
                                     .await?;
@@ -1364,6 +1364,15 @@ impl MimeMessage {
         self.get_mailinglist_header().is_some()
     }
 
+    /// Detects Schleuder mailing list by List-Help header.
+    pub(crate) fn is_schleuder_message(&self) -> bool {
+        if let Some(list_help) = self.get_header(HeaderDef::ListHelp) {
+            list_help == "<https://schleuder.org/>"
+        } else {
+            false
+        }
+    }
+
     pub fn replace_msg_by_error(&mut self, error_msg: &str) {
         self.is_system_message = SystemMessage::Unknown;
         if let Some(part) = self.parts.first_mut() {
@@ -1592,8 +1601,12 @@ impl MimeMessage {
     /// eg. when the user-edited-content is html.
     /// As these footers would appear as repeated, separate text-bubbles,
     /// we remove them.
+    ///
+    /// We make an exception for Schleuder mailing lists
+    /// because they typically create messages with two text parts,
+    /// one for headers and one for the actual contents.
     fn maybe_remove_inline_mailinglist_footer(&mut self) {
-        if self.is_mailinglist_message() {
+        if self.is_mailinglist_message() && !self.is_schleuder_message() {
             let text_part_cnt = self
                 .parts
                 .iter()
@@ -1644,13 +1657,7 @@ impl MimeMessage {
     /// Handle reports
     /// (MDNs = Message Disposition Notification, the message was read
     /// and NDNs = Non delivery notification, the message could not be delivered)
-    pub async fn handle_reports(
-        &self,
-        context: &Context,
-        from_id: ContactId,
-        sent_timestamp: i64,
-        parts: &[Part],
-    ) {
+    pub async fn handle_reports(&self, context: &Context, from_id: ContactId, parts: &[Part]) {
         for report in &self.mdn_reports {
             for original_message_id in report
                 .original_message_id
@@ -1658,7 +1665,7 @@ impl MimeMessage {
                 .chain(&report.additional_message_ids)
             {
                 if let Err(err) =
-                    handle_mdn(context, from_id, original_message_id, sent_timestamp).await
+                    handle_mdn(context, from_id, original_message_id, self.timestamp_sent).await
                 {
                     warn!(context, "Could not handle MDN: {err:#}.");
                 }
@@ -2111,6 +2118,8 @@ async fn handle_mdn(
     {
         update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await?;
         context.emit_event(EventType::MsgRead { chat_id, msg_id });
+        // note(treefit): only matters if it is the last message in chat (but probably to expensive to check, debounce also solves it)
+        ui_events::emit_chatlist_item_changed(context, chat_id);
     }
     Ok(())
 }
@@ -3791,4 +3800,24 @@ Content-Disposition: reaction\n\
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_schleuder() -> Result<()> {
+        let context = TestContext::new_alice().await;
+        let raw = include_bytes!("../test-data/message/schleuder.eml");
+
+        let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
+            .await
+            .unwrap();
+        assert_eq!(msg.parts.len(), 2);
+
+        // Header part.
+        assert_eq!(msg.parts[0].typ, Viewtype::Text);
+
+        // Actual contents part.
+        assert_eq!(msg.parts[1].typ, Viewtype::Text);
+        assert_eq!(msg.parts[1].msg, "hello,\nbye");
+
+        Ok(())
+    }
 }

src/param.rs

@@ -84,7 +84,7 @@ pub enum Param {
     /// For Messages
     Arg2 = b'F',
 
-    /// For Messages
+    /// `Secure-Join-Fingerprint` header for `{vc,vg}-request-with-auth` messages.
     Arg3 = b'G',
 
     /// For Messages

src/peerstate.rs

@@ -6,6 +6,7 @@ use num_traits::FromPrimitive;
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::chat::{self, Chat};
 use crate::chatlist::Chatlist;
+use crate::config::Config;
 use crate::constants::Chattype;
 use crate::contact::{addr_cmp, Contact, ContactAddress, Origin};
 use crate::context::Context;
@@ -14,7 +15,7 @@ use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::message::Message;
 use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
-use crate::stock_str;
+use crate::{stock_str, ui_events};
 
 /// Type of the public key stored inside the peerstate.
 #[derive(Debug)]
@@ -83,6 +84,10 @@ pub struct Peerstate {
     /// The address that introduced secondary verified key.
     pub secondary_verifier: Option<String>,
 
+    /// Row ID of the key in the `keypairs` table
+    /// that we think the peer knows as verified.
+    pub backward_verified_key_id: Option<i64>,
+
     /// True if it was detected
     /// that the fingerprint of the key used in chats with
     /// opportunistic encryption was changed after Peerstate creation.
@@ -108,6 +113,7 @@ impl Peerstate {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         }
     }
@@ -137,6 +143,7 @@ impl Peerstate {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         }
     }
@@ -148,7 +155,8 @@ impl Peerstate {
                      verified_key, verified_key_fingerprint, \
                      verifier, \
                      secondary_verified_key, secondary_verified_key_fingerprint, \
-                     secondary_verifier \
+                     secondary_verifier, \
+                     backward_verified_key_id \
                      FROM acpeerstates \
                      WHERE addr=? COLLATE NOCASE LIMIT 1;";
         Self::from_stmt(context, query, (addr,)).await
@@ -164,7 +172,8 @@ impl Peerstate {
                      verified_key, verified_key_fingerprint, \
                      verifier, \
                      secondary_verified_key, secondary_verified_key_fingerprint, \
-                     secondary_verifier \
+                     secondary_verifier, \
+                     backward_verified_key_id \
                      FROM acpeerstates  \
                      WHERE public_key_fingerprint=? \
                      OR gossip_key_fingerprint=? \
@@ -187,7 +196,8 @@ impl Peerstate {
                      verified_key, verified_key_fingerprint, \
                      verifier, \
                      secondary_verified_key, secondary_verified_key_fingerprint, \
-                     secondary_verifier \
+                     secondary_verifier, \
+                     backward_verified_key_id \
                      FROM acpeerstates  \
                      WHERE verified_key_fingerprint=? \
                      OR addr=? COLLATE NOCASE \
@@ -255,6 +265,7 @@ impl Peerstate {
                         let secondary_verifier: Option<String> = row.get("secondary_verifier")?;
                         secondary_verifier.filter(|s| !s.is_empty())
                     },
+                    backward_verified_key_id: row.get("backward_verified_key_id")?,
                     fingerprint_changed: false,
                 };
 
@@ -435,6 +446,17 @@ impl Peerstate {
         verified.is_some() && verified == self.peek_key_fingerprint(false)
     }
 
+    pub(crate) async fn is_backward_verified(&self, context: &Context) -> Result<bool> {
+        let Some(backward_verified_key_id) = self.backward_verified_key_id else {
+            return Ok(false);
+        };
+
+        let self_key_id = context.get_config_i64(Config::KeyId).await?;
+
+        let backward_verified = backward_verified_key_id == self_key_id;
+        Ok(backward_verified)
+    }
+
     /// Set this peerstate to verified
     /// Make sure to call `self.save_to_db` to save these changes
     /// Params:
@@ -510,8 +532,9 @@ impl Peerstate {
                 secondary_verified_key,
                 secondary_verified_key_fingerprint,
                 secondary_verifier,
+                backward_verified_key_id,
                 addr)
-                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                 ON CONFLICT (addr)
                 DO UPDATE SET
                   last_seen = excluded.last_seen,
@@ -527,7 +550,8 @@ impl Peerstate {
                   verifier = excluded.verifier,
                   secondary_verified_key = excluded.secondary_verified_key,
                   secondary_verified_key_fingerprint = excluded.secondary_verified_key_fingerprint,
-                  secondary_verifier = excluded.secondary_verifier",
+                  secondary_verifier = excluded.secondary_verifier,
+                  backward_verified_key_id = excluded.backward_verified_key_id",
             (
                 self.last_seen,
                 self.last_seen_autocrypt,
@@ -545,6 +569,7 @@ impl Peerstate {
                     .as_ref()
                     .map(|fp| fp.hex()),
                 self.secondary_verifier.as_deref().unwrap_or(""),
+                self.backward_verified_key_id,
                 &self.addr,
             ),
         )
@@ -670,6 +695,9 @@ impl Peerstate {
             .await?;
         }
 
+        ui_events::emit_chatlist_changed(context);
+        // update the chats the contact is part of
+        ui_events::emit_chatlist_items_changed_for_contact(context, contact_id);
         Ok(())
     }
 
@@ -696,44 +724,46 @@ pub(crate) async fn maybe_do_aeap_transition(
     mime_parser: &mut crate::mimeparser::MimeMessage,
 ) -> Result<()> {
     let info = &mime_parser.decryption_info;
-    if let Some(peerstate) = &info.peerstate {
-        // If the from addr is different from the peerstate address we know,
-        // we may want to do an AEAP transition.
-        if !addr_cmp(&peerstate.addr, &mime_parser.from.addr)
-                // Check if it's a chat message; we do this to avoid
-                // some accidental transitions if someone writes from multiple
-                // addresses with an MUA.
-                && mime_parser.has_chat_version()
-                // Check if the message is signed correctly.
-                // Although checking `from_is_signed` below is sufficient, let's play it safe.
-                && !mime_parser.signatures.is_empty()
-                // Check if the From: address was also in the signed part of the email.
-                // Without this check, an attacker could replay a message from Alice 
-                // to Bob. Then Bob's device would do an AEAP transition from Alice's
-                // to the attacker's address, allowing for easier phishing.
-                && mime_parser.from_is_signed
-                && info.message_time > peerstate.last_seen
-        {
-            let info = &mut mime_parser.decryption_info;
-            let peerstate = info.peerstate.as_mut().context("no peerstate??")?;
-            // Add info messages to chats with this (verified) contact
-            //
-            peerstate
-                .handle_setup_change(
-                    context,
-                    info.message_time,
-                    PeerstateChange::Aeap(info.from.clone()),
-                )
-                .await?;
+    let Some(peerstate) = &info.peerstate else {
+        return Ok(());
+    };
 
-            peerstate.addr = info.from.clone();
-            let header = info.autocrypt_header.as_ref().context(
-                "Internal error: Tried to do an AEAP transition without an autocrypt header??",
-            )?;
-            peerstate.apply_header(header, info.message_time);
+    // If the from addr is different from the peerstate address we know,
+    // we may want to do an AEAP transition.
+    if !addr_cmp(&peerstate.addr, &mime_parser.from.addr)
+            // Check if it's a chat message; we do this to avoid
+            // some accidental transitions if someone writes from multiple
+            // addresses with an MUA.
+            && mime_parser.has_chat_version()
+            // Check if the message is signed correctly.
+            // Although checking `from_is_signed` below is sufficient, let's play it safe.
+            && !mime_parser.signatures.is_empty()
+            // Check if the From: address was also in the signed part of the email.
+            // Without this check, an attacker could replay a message from Alice
+            // to Bob. Then Bob's device would do an AEAP transition from Alice's
+            // to the attacker's address, allowing for easier phishing.
+            && mime_parser.from_is_signed
+            && info.message_time > peerstate.last_seen
+    {
+        let info = &mut mime_parser.decryption_info;
+        let peerstate = info.peerstate.as_mut().context("no peerstate??")?;
+        // Add info messages to chats with this (verified) contact
+        //
+        peerstate
+            .handle_setup_change(
+                context,
+                info.message_time,
+                PeerstateChange::Aeap(info.from.clone()),
+            )
+            .await?;
 
-            peerstate.save_to_db(&context.sql).await?;
-        }
+        peerstate.addr = info.from.clone();
+        let header = info.autocrypt_header.as_ref().context(
+            "Internal error: Tried to do an AEAP transition without an autocrypt header??",
+        )?;
+        peerstate.apply_header(header, info.message_time);
+
+        peerstate.save_to_db(&context.sql).await?;
     }
 
     Ok(())
@@ -804,6 +834,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 
@@ -847,6 +878,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 
@@ -883,6 +915,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 
@@ -949,6 +982,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
 

src/qr.rs

@@ -1057,6 +1057,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
         assert!(

src/reaction.rs

@@ -425,7 +425,7 @@ Content-Disposition: reaction\n\
         let contacts = reactions.contacts();
         assert_eq!(contacts.len(), 1);
 
-        assert_eq!(contacts.get(0), Some(&bob_id));
+        assert_eq!(contacts.first(), Some(&bob_id));
         let bob_reaction = reactions.get(bob_id);
         assert_eq!(bob_reaction.is_empty(), false);
         assert_eq!(bob_reaction.emojis(), vec!["👍"]);
@@ -526,7 +526,7 @@ Here's my footer -- bob@example.net"
         assert_eq!(reactions.to_string(), "👍1");
         let contacts = reactions.contacts();
         assert_eq!(contacts.len(), 1);
-        let bob_id = contacts.get(0).unwrap();
+        let bob_id = contacts.first().unwrap();
         let bob_reaction = reactions.get(*bob_id);
         assert_eq!(bob_reaction.is_empty(), false);
         assert_eq!(bob_reaction.emojis(), vec!["👍"]);
@@ -578,13 +578,13 @@ Here's my footer -- bob@example.net"
         )
         .await?
         .unwrap();
-        let alice_msg_id = *alice_received_message.msg_ids.get(0).unwrap();
+        let alice_msg_id = *alice_received_message.msg_ids.first().unwrap();
 
         // Bob downloads own message on the other device.
         let bob_received_message = receive_imf(&bob, msg_full.as_bytes(), false)
             .await?
             .unwrap();
-        let bob_msg_id = *bob_received_message.msg_ids.get(0).unwrap();
+        let bob_msg_id = *bob_received_message.msg_ids.first().unwrap();
 
         // Bob reacts to own message.
         send_reaction(&bob, bob_msg_id, "👍").await.unwrap();

src/receive_imf.rs

@@ -1,6 +1,5 @@
 //! Internet Message Format reception pipeline.
 
-use std::cmp::min;
 use std::collections::HashSet;
 use std::convert::TryFrom;
 
@@ -10,6 +9,7 @@ use num_traits::FromPrimitive;
 use once_cell::sync::Lazy;
 use regex::Regex;
 
+use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, ShowEmails, DC_CHAT_ID_TRASH};
@@ -23,7 +23,6 @@ use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::imap::{markseen_on_imap_table, GENERATED_PREFIX};
-use crate::location;
 use crate::log::LogExt;
 use crate::message::{
     self, rfc724_mid_exists, rfc724_mid_exists_and, Message, MessageState, MessengerMessage, MsgId,
@@ -38,10 +37,9 @@ use crate::simplify;
 use crate::sql;
 use crate::stock_str;
 use crate::sync::Sync::*;
-use crate::tools::{
-    buf_compress, extract_grpid_from_rfc724_mid, smeared_time, strip_rtlo_characters,
-};
+use crate::tools::{buf_compress, extract_grpid_from_rfc724_mid, strip_rtlo_characters};
 use crate::{contact, imap};
+use crate::{location, ui_events};
 
 /// This is the struct that is returned after receiving one email (aka MIME message).
 ///
@@ -130,7 +128,8 @@ async fn insert_tombstone(context: &Context, rfc724_mid: &str) -> Result<MsgId>
 /// returns `Ok(None)`.
 ///
 /// If `is_partial_download` is set, it contains the full message size in bytes.
-/// Do not confuse that with `replace_partial_download` that will be set when the full message is loaded later.
+/// Do not confuse that with `replace_msg_id` that will be set when the full message is loaded
+/// later.
 pub(crate) async fn receive_imf_inner(
     context: &Context,
     rfc724_mid: &str,
@@ -171,28 +170,99 @@ pub(crate) async fn receive_imf_inner(
         Ok(mime_parser) => mime_parser,
     };
 
-    info!(context, "Receiving message {rfc724_mid:?}, seen={seen}...");
+    crate::peerstate::maybe_do_aeap_transition(context, &mut mime_parser).await?;
+    if let Some(peerstate) = &mime_parser.decryption_info.peerstate {
+        peerstate
+            .handle_fingerprint_change(context, mime_parser.timestamp_sent)
+            .await?;
+        // When peerstate is set to Mutual, it's saved immediately to not lose that fact in case
+        // of an error. Otherwise we don't save peerstate until get here to reduce the number of
+        // calls to save_to_db() and not to degrade encryption if a mail wasn't parsed
+        // successfully.
+        if peerstate.prefer_encrypt != EncryptPreference::Mutual {
+            peerstate.save_to_db(&context.sql).await?;
+        }
+    }
+
+    let rfc724_mid_orig = &mime_parser
+        .get_rfc724_mid()
+        .unwrap_or(rfc724_mid.to_string());
+    info!(
+        context,
+        "Receiving message {rfc724_mid_orig:?}, seen={seen}...",
+    );
+    let incoming = !context.is_self_addr(&mime_parser.from.addr).await?;
+
+    // For the case if we missed a successful SMTP response. Be optimistic that the message is
+    // delivered also.
+    let delivered = !incoming && {
+        let self_addr = context.get_primary_self_addr().await?;
+        context
+            .sql
+            .execute(
+                "DELETE FROM smtp \
+                WHERE rfc724_mid=?1 AND (recipients LIKE ?2 OR recipients LIKE ('% ' || ?2))",
+                (rfc724_mid_orig, &self_addr),
+            )
+            .await?;
+        !context
+            .sql
+            .exists(
+                "SELECT COUNT(*) FROM smtp WHERE rfc724_mid=?",
+                (rfc724_mid_orig,),
+            )
+            .await?
+    };
+
+    async fn on_msg_in_db(
+        context: &Context,
+        msg_id: MsgId,
+        delivered: bool,
+    ) -> Result<Option<ReceivedMsg>> {
+        if delivered {
+            msg_id.set_delivered(context).await?;
+        }
+        Ok(None)
+    }
 
     // check, if the mail is already in our database.
     // make sure, this check is done eg. before securejoin-processing.
-    let (replace_partial_download, replace_chat_id) =
-        if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
-            let msg = Message::load_from_db(context, old_msg_id).await?;
-            if msg.download_state() != DownloadState::Done && is_partial_download.is_none() {
-                // the message was partially downloaded before and is fully downloaded now.
-                info!(
-                    context,
-                    "Message already partly in DB, replacing by full message."
-                );
-                (Some(old_msg_id), Some(msg.chat_id))
-            } else {
-                // the message was probably moved around.
-                info!(context, "Message already in DB, doing nothing.");
-                return Ok(None);
-            }
+    let (replace_msg_id, replace_chat_id);
+    if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
+        if is_partial_download.is_some() {
+            info!(
+                context,
+                "Got a partial download and message is already in DB."
+            );
+            return on_msg_in_db(context, old_msg_id, delivered).await;
+        }
+        let msg = Message::load_from_db(context, old_msg_id).await?;
+        replace_msg_id = Some(old_msg_id);
+        replace_chat_id = if msg.download_state() != DownloadState::Done {
+            // the message was partially downloaded before and is fully downloaded now.
+            info!(
+                context,
+                "Message already partly in DB, replacing by full message."
+            );
+            Some(msg.chat_id)
         } else {
-            (None, None)
+            None
         };
+    } else {
+        replace_msg_id = if rfc724_mid_orig != rfc724_mid {
+            message::rfc724_mid_exists(context, rfc724_mid_orig).await?
+        } else {
+            None
+        };
+        replace_chat_id = None;
+    }
+
+    if replace_chat_id.is_some() {
+        // Need to update chat id in the db.
+    } else if let Some(msg_id) = replace_msg_id {
+        info!(context, "Message is already downloaded.");
+        return on_msg_in_db(context, msg_id, delivered).await;
+    };
 
     let prevent_rename =
         mime_parser.is_mailinglist_message() || mime_parser.get_header(HeaderDef::Sender).is_some();
@@ -217,8 +287,6 @@ pub(crate) async fn receive_imf_inner(
             }
         };
 
-    let incoming = from_id != ContactId::SELF;
-
     let to_ids = add_or_lookup_contacts_by_address_list(
         context,
         &mime_parser.recipients,
@@ -232,15 +300,6 @@ pub(crate) async fn receive_imf_inner(
     )
     .await?;
 
-    let rcvd_timestamp = smeared_time(context);
-
-    // Sender timestamp is allowed to be a bit in the future due to
-    // unsynchronized clocks, but not too much.
-    let sent_timestamp = mime_parser
-        .get_header(HeaderDef::Date)
-        .and_then(|value| mailparse::dateparse(value).ok())
-        .map_or(rcvd_timestamp, |value| min(value, rcvd_timestamp + 60));
-
     update_verified_keys(context, &mut mime_parser, from_id).await?;
 
     let received_msg;
@@ -256,7 +315,7 @@ pub(crate) async fn receive_imf_inner(
             mime_parser.decryption_info.peerstate =
                 Peerstate::from_addr(context, contact.get_addr()).await?;
         } else {
-            let to_id = to_ids.get(0).copied().unwrap_or_default();
+            let to_id = to_ids.first().copied().unwrap_or_default();
             // handshake may mark contacts as verified and must be processed before chats are created
             res = observe_securejoin_on_other_device(context, &mime_parser, to_id)
                 .await
@@ -269,7 +328,7 @@ pub(crate) async fn receive_imf_inner(
                 received_msg = Some(ReceivedMsg {
                     chat_id: DC_CHAT_ID_TRASH,
                     state: MessageState::InSeen,
-                    sort_timestamp: sent_timestamp,
+                    sort_timestamp: mime_parser.timestamp_sent,
                     msg_ids: vec![msg_id],
                     needs_delete_job: res == securejoin::HandshakeMessage::Done,
                     #[cfg(test)]
@@ -287,6 +346,24 @@ pub(crate) async fn receive_imf_inner(
     let verified_encryption =
         has_verified_encryption(context, &mime_parser, from_id, &to_ids).await?;
 
+    if verified_encryption == VerifiedEncryption::Verified
+        && mime_parser.get_header(HeaderDef::ChatVerified).is_some()
+    {
+        if let Some(peerstate) = &mut mime_parser.decryption_info.peerstate {
+            // NOTE: it might be better to remember ID of the key
+            // that we used to decrypt the message, but
+            // it is unlikely that default key ever changes
+            // as it only happens when user imports a new default key.
+            //
+            // Backward verification is not security-critical,
+            // it is only needed to avoid adding user who does not
+            // have our key as verified to protected chats.
+            peerstate.backward_verified_key_id =
+                Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+            peerstate.save_to_db(&context.sql).await?;
+        }
+    }
+
     let received_msg = if let Some(received_msg) = received_msg {
         received_msg
     } else {
@@ -297,13 +374,11 @@ pub(crate) async fn receive_imf_inner(
             imf_raw,
             incoming,
             &to_ids,
-            rfc724_mid,
-            sent_timestamp,
-            rcvd_timestamp,
+            rfc724_mid_orig,
             from_id,
-            seen || replace_partial_download.is_some(),
+            seen,
             is_partial_download,
-            replace_partial_download,
+            replace_msg_id,
             fetching_existing_messages,
             prevent_rename,
             verified_encryption,
@@ -313,7 +388,7 @@ pub(crate) async fn receive_imf_inner(
     };
 
     if !from_id.is_special() {
-        contact::update_last_seen(context, from_id, sent_timestamp).await?;
+        contact::update_last_seen(context, from_id, mime_parser.timestamp_sent).await?;
     }
 
     // Update gossiped timestamp for the chat if someone else or our other device sent
@@ -330,9 +405,9 @@ pub(crate) async fn receive_imf_inner(
             context,
             "Received message contains Autocrypt-Gossip for all members of {chat_id}, updating timestamp."
         );
-        if chat_id.get_gossiped_timestamp(context).await? < sent_timestamp {
+        if chat_id.get_gossiped_timestamp(context).await? < mime_parser.timestamp_sent {
             chat_id
-                .set_gossiped_timestamp(context, sent_timestamp)
+                .set_gossiped_timestamp(context, mime_parser.timestamp_sent)
                 .await?;
         }
     }
@@ -369,7 +444,11 @@ pub(crate) async fn receive_imf_inner(
     if let Some(avatar_action) = &mime_parser.user_avatar {
         if from_id != ContactId::UNDEFINED
             && context
-                .update_contacts_timestamp(from_id, Param::AvatarTimestamp, sent_timestamp)
+                .update_contacts_timestamp(
+                    from_id,
+                    Param::AvatarTimestamp,
+                    mime_parser.timestamp_sent,
+                )
                 .await?
         {
             if let Err(err) = contact::set_profile_image(
@@ -390,7 +469,11 @@ pub(crate) async fn receive_imf_inner(
         if !mime_parser.is_mailinglist_message()
             && from_id != ContactId::UNDEFINED
             && context
-                .update_contacts_timestamp(from_id, Param::StatusTimestamp, sent_timestamp)
+                .update_contacts_timestamp(
+                    from_id,
+                    Param::StatusTimestamp,
+                    mime_parser.timestamp_sent,
+                )
                 .await?
         {
             if let Err(err) = contact::set_status(
@@ -411,20 +494,34 @@ pub(crate) async fn receive_imf_inner(
     let delete_server_after = context.get_config_delete_server_after().await?;
 
     if !received_msg.msg_ids.is_empty() {
-        if received_msg.needs_delete_job
+        let target = if received_msg.needs_delete_job
             || (delete_server_after == Some(0) && is_partial_download.is_none())
         {
-            let target = context.get_delete_msgs_target().await?;
+            Some(context.get_delete_msgs_target().await?)
+        } else {
+            None
+        };
+        if target.is_some() || rfc724_mid_orig != rfc724_mid {
+            let target_subst = match &target {
+                Some(_) => "target=?1,",
+                None => "",
+            };
             context
                 .sql
                 .execute(
-                    "UPDATE imap SET target=? WHERE rfc724_mid=?",
-                    (target, rfc724_mid),
+                    &format!("UPDATE imap SET {target_subst} rfc724_mid=?2 WHERE rfc724_mid=?3"),
+                    (
+                        target.as_deref().unwrap_or_default(),
+                        rfc724_mid_orig,
+                        rfc724_mid,
+                    ),
                 )
                 .await?;
-        } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() {
+        }
+        if target.is_none() && !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version()
+        {
             // This is a Delta Chat MDN. Mark as read.
-            markseen_on_imap_table(context, rfc724_mid).await?;
+            markseen_on_imap_table(context, rfc724_mid_orig).await?;
         }
     }
 
@@ -439,7 +536,7 @@ pub(crate) async fn receive_imf_inner(
     context.new_msgs_notify.notify_one();
 
     mime_parser
-        .handle_reports(context, from_id, sent_timestamp, &mime_parser.parts)
+        .handle_reports(context, from_id, &mime_parser.parts)
         .await;
 
     from_id.mark_bot(context, mime_parser.is_bot).await?;
@@ -510,8 +607,6 @@ async fn add_parts(
     incoming: bool,
     to_ids: &[ContactId],
     rfc724_mid: &str,
-    sent_timestamp: i64,
-    rcvd_timestamp: i64,
     from_id: ContactId,
     seen: bool,
     is_partial_download: Option<u32>,
@@ -520,6 +615,10 @@ async fn add_parts(
     prevent_rename: bool,
     verified_encryption: VerifiedEncryption,
 ) -> Result<ReceivedMsg> {
+    let rfc724_mid_orig = &mime_parser
+        .get_rfc724_mid()
+        .unwrap_or(rfc724_mid.to_string());
+
     let mut chat_id = None;
     let mut chat_id_blocked = Blocked::Not;
 
@@ -650,7 +749,6 @@ async fn add_parts(
                 from_id,
                 to_ids,
                 &verified_encryption,
-                sent_timestamp,
             )
             .await?
             {
@@ -696,7 +794,6 @@ async fn add_parts(
             group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
-                sent_timestamp,
                 group_chat_id,
                 from_id,
                 to_ids,
@@ -714,7 +811,6 @@ async fn add_parts(
                     allow_creation,
                     mailinglist_header,
                     mime_parser,
-                    sent_timestamp,
                 )
                 .await?
                 {
@@ -725,7 +821,7 @@ async fn add_parts(
         }
 
         if let Some(chat_id) = chat_id {
-            apply_mailinglist_changes(context, mime_parser, sent_timestamp, chat_id).await?;
+            apply_mailinglist_changes(context, mime_parser, chat_id).await?;
         }
 
         // if contact renaming is prevented (for mailinglists and bots),
@@ -811,11 +907,13 @@ async fn add_parts(
                         // The message itself will be sorted under the device message since the device
                         // message is `MessageState::InNoticed`, which means that all following
                         // messages are sorted under it.
-                        let sort_timestamp =
-                            calc_sort_timestamp(context, sent_timestamp, chat_id, true, incoming)
-                                .await?;
                         chat_id
-                            .set_protection(context, new_protection, sort_timestamp, Some(from_id))
+                            .set_protection(
+                                context,
+                                new_protection,
+                                mime_parser.timestamp_sent,
+                                Some(from_id),
+                            )
                             .await?;
                     }
                 }
@@ -839,7 +937,7 @@ async fn add_parts(
         // the mail is on the IMAP server, probably it is also delivered.
         // We cannot recreate other states (read, error).
         state = MessageState::OutDelivered;
-        to_id = to_ids.get(0).copied().unwrap_or_default();
+        to_id = to_ids.first().copied().unwrap_or_default();
 
         let self_sent =
             from_id == ContactId::SELF && to_ids.len() == 1 && to_ids.contains(&ContactId::SELF);
@@ -883,7 +981,6 @@ async fn add_parts(
                     from_id,
                     to_ids,
                     &verified_encryption,
-                    sent_timestamp,
                 )
                 .await?
                 {
@@ -921,7 +1018,6 @@ async fn add_parts(
             group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
-                sent_timestamp,
                 chat_id,
                 from_id,
                 to_ids,
@@ -1010,8 +1106,15 @@ async fn add_parts(
     };
 
     let in_fresh = state == MessageState::InFresh;
-    let sort_timestamp =
-        calc_sort_timestamp(context, sent_timestamp, chat_id, false, incoming).await?;
+    let sort_to_bottom = false;
+    let sort_timestamp = chat_id
+        .calc_sort_timestamp(
+            context,
+            mime_parser.timestamp_sent,
+            sort_to_bottom,
+            incoming,
+        )
+        .await?;
 
     // Apply ephemeral timer changes to the chat.
     //
@@ -1041,7 +1144,11 @@ async fn add_parts(
                 "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} to avoid rollback.",
             );
         } else if chat_id
-            .update_timestamp(context, Param::EphemeralSettingsTimestamp, sent_timestamp)
+            .update_timestamp(
+                context,
+                Param::EphemeralSettingsTimestamp,
+                mime_parser.timestamp_sent,
+            )
             .await?
         {
             if let Err(err) = chat_id
@@ -1244,7 +1351,7 @@ async fn add_parts(
             match ephemeral_timer {
                 EphemeralTimer::Disabled => 0,
                 EphemeralTimer::Enabled { duration } => {
-                    rcvd_timestamp.saturating_add(duration.into())
+                    mime_parser.timestamp_rcvd.saturating_add(duration.into())
                 }
             }
         };
@@ -1292,13 +1399,13 @@ RETURNING id
 "#)?;
                 let row_id: MsgId = stmt.query_row(params![
                     replace_msg_id,
-                    rfc724_mid,
+                    rfc724_mid_orig,
                     if trash { DC_CHAT_ID_TRASH } else { chat_id },
                     if trash { ContactId::UNDEFINED } else { from_id },
                     if trash { ContactId::UNDEFINED } else { to_id },
                     sort_timestamp,
-                    sent_timestamp,
-                    rcvd_timestamp,
+                    mime_parser.timestamp_sent,
+                    mime_parser.timestamp_rcvd,
                     typ,
                     state,
                     is_dc_message,
@@ -1470,53 +1577,6 @@ async fn save_locations(
     Ok(())
 }
 
-async fn calc_sort_timestamp(
-    context: &Context,
-    message_timestamp: i64,
-    chat_id: ChatId,
-    always_sort_to_bottom: bool,
-    incoming: bool,
-) -> Result<i64> {
-    let mut sort_timestamp = min(message_timestamp, smeared_time(context));
-
-    let last_msg_time: Option<i64> = if always_sort_to_bottom {
-        // get newest message for this chat
-        context
-            .sql
-            .query_get_value(
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=?",
-                (chat_id,),
-            )
-            .await?
-    } else if incoming {
-        // get newest non fresh message for this chat.
-
-        // If a user hasn't been online for some time, the Inbox is fetched first and then the
-        // Sentbox. In order for Inbox and Sent messages to be allowed to mingle, outgoing messages
-        // are purely sorted by their sent timestamp. NB: The Inbox must be fetched first otherwise
-        // Inbox messages would be always below old Sentbox messages. We could take in the query
-        // below only incoming messages, but then new incoming messages would mingle with just sent
-        // outgoing ones and apear somewhere in the middle of the chat.
-        context
-            .sql
-            .query_get_value(
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
-                (chat_id, MessageState::InFresh),
-            )
-            .await?
-    } else {
-        None
-    };
-
-    if let Some(last_msg_time) = last_msg_time {
-        if last_msg_time > sort_timestamp {
-            sort_timestamp = last_msg_time;
-        }
-    }
-
-    Ok(sort_timestamp)
-}
-
 async fn lookup_chat_by_reply(
     context: &Context,
     mime_parser: &MimeMessage,
@@ -1529,27 +1589,10 @@ async fn lookup_chat_by_reply(
     let Some(parent) = parent else {
         return Ok(None);
     };
-
-    let parent_chat = Chat::load_from_db(context, parent.chat_id).await?;
-
-    if parent.download_state != DownloadState::Done
-        // TODO (2023-09-12): Added for backward compatibility with versions that did not have
-        // `DownloadState::Undecipherable`. Remove eventually with the comment in
-        // `MimeMessage::from_bytes()`.
-        || parent
-            .error
-            .as_ref()
-            .filter(|e| e.starts_with("Decrypting failed:"))
-            .is_some()
-    {
-        // If the parent msg is not fully downloaded or undecipherable, it may have been
-        // assigned to the wrong chat (they often get assigned to the 1:1 chat with the sender).
+    let Some(parent_chat_id) = ChatId::lookup_by_message(parent) else {
         return Ok(None);
-    }
-
-    if parent_chat.id == DC_CHAT_ID_TRASH {
-        return Ok(None);
-    }
+    };
+    let parent_chat = Chat::load_from_db(context, parent_chat_id).await?;
 
     // If this was a private message just to self, it was probably a private reply.
     // It should not go into the group then, but into the private chat.
@@ -1622,7 +1665,6 @@ async fn create_or_lookup_group(
     from_id: ContactId,
     to_ids: &[ContactId],
     verified_encryption: &VerifiedEncryption,
-    timestamp: i64,
 ) -> Result<Option<(ChatId, Blocked)>> {
     let grpid = if let Some(grpid) = try_getting_grpid(mime_parser) {
         grpid
@@ -1635,7 +1677,7 @@ async fn create_or_lookup_group(
             member_ids.push(ContactId::SELF);
         }
 
-        let res = create_adhoc_group(context, mime_parser, create_blocked, &member_ids, timestamp)
+        let res = create_adhoc_group(context, mime_parser, create_blocked, &member_ids)
             .await
             .context("could not create ad hoc group")?
             .map(|chat_id| (chat_id, create_blocked));
@@ -1717,7 +1759,7 @@ async fn create_or_lookup_group(
             create_blocked,
             create_protected,
             None,
-            timestamp,
+            mime_parser.timestamp_sent,
         )
         .await
         .with_context(|| format!("Failed to create group '{grpname}' for grpid={grpid}"))?;
@@ -1736,6 +1778,8 @@ async fn create_or_lookup_group(
         chat::add_to_chat_contacts_table(context, new_chat_id, &members).await?;
 
         context.emit_event(EventType::ChatModified(new_chat_id));
+        ui_events::emit_chatlist_changed(context);
+        ui_events::emit_chatlist_item_changed(context, new_chat_id);
     }
 
     if let Some(chat_id) = chat_id {
@@ -1764,7 +1808,6 @@ async fn create_or_lookup_group(
 async fn apply_group_changes(
     context: &Context,
     mime_parser: &mut MimeMessage,
-    sent_timestamp: i64,
     chat_id: ChatId,
     from_id: ContactId,
     to_ids: &[ContactId],
@@ -1802,7 +1845,11 @@ async fn apply_group_changes(
     let allow_member_list_changes = !is_partial_download
         && is_from_in_chat
         && chat_id
-            .update_timestamp(context, Param::MemberListTimestamp, sent_timestamp)
+            .update_timestamp(
+                context,
+                Param::MemberListTimestamp,
+                mime_parser.timestamp_sent,
+            )
             .await?;
 
     // Whether to rebuild the member list from scratch.
@@ -1841,7 +1888,7 @@ async fn apply_group_changes(
                 .set_protection(
                     context,
                     ProtectionStatus::Protected,
-                    smeared_time(context),
+                    mime_parser.timestamp_sent,
                     Some(from_id),
                 )
                 .await?;
@@ -1895,7 +1942,11 @@ async fn apply_group_changes(
             .filter(|grpname| grpname.len() < 200)
         {
             if chat_id
-                .update_timestamp(context, Param::GroupNameTimestamp, sent_timestamp)
+                .update_timestamp(
+                    context,
+                    Param::GroupNameTimestamp,
+                    mime_parser.timestamp_sent,
+                )
                 .await?
             {
                 info!(context, "Updating grpname for chat {chat_id}.");
@@ -2002,7 +2053,7 @@ async fn apply_group_changes(
             info!(context, "Group-avatar change for {chat_id}.");
             if chat
                 .param
-                .update_timestamp(Param::AvatarTimestamp, sent_timestamp)?
+                .update_timestamp(Param::AvatarTimestamp, mime_parser.timestamp_sent)?
             {
                 match avatar_action {
                     AvatarAction::Change(profile_image) => {
@@ -2020,6 +2071,7 @@ async fn apply_group_changes(
 
     if send_event_chat_modified {
         context.emit_event(EventType::ChatModified(chat_id));
+        ui_events::emit_chatlist_item_changed(context, chat_id);
     }
     Ok((group_changes_msgs, better_msg))
 }
@@ -2051,7 +2103,6 @@ async fn create_or_lookup_mailinglist(
     allow_creation: bool,
     list_id_header: &str,
     mime_parser: &MimeMessage,
-    timestamp: i64,
 ) -> Result<Option<(ChatId, Blocked)>> {
     let listid = mailinglist_header_listid(list_id_header)?;
 
@@ -2083,7 +2134,7 @@ async fn create_or_lookup_mailinglist(
             blocked,
             ProtectionStatus::Unprotected,
             param,
-            timestamp,
+            mime_parser.timestamp_sent,
         )
         .await
         .with_context(|| {
@@ -2171,7 +2222,6 @@ fn compute_mailinglist_name(
 async fn apply_mailinglist_changes(
     context: &Context,
     mime_parser: &MimeMessage,
-    sent_timestamp: i64,
     chat_id: ChatId,
 ) -> Result<()> {
     let Some(mailinglist_header) = mime_parser.get_mailinglist_header() else {
@@ -2187,7 +2237,11 @@ async fn apply_mailinglist_changes(
     let new_name = compute_mailinglist_name(mailinglist_header, listid, mime_parser);
     if chat.name != new_name
         && chat_id
-            .update_timestamp(context, Param::GroupNameTimestamp, sent_timestamp)
+            .update_timestamp(
+                context,
+                Param::GroupNameTimestamp,
+                mime_parser.timestamp_sent,
+            )
             .await?
     {
         info!(context, "Updating listname for chat {chat_id}.");
@@ -2268,7 +2322,6 @@ async fn create_adhoc_group(
     mime_parser: &MimeMessage,
     create_blocked: Blocked,
     member_ids: &[ContactId],
-    timestamp: i64,
 ) -> Result<Option<ChatId>> {
     if mime_parser.is_mailinglist_message() {
         return Ok(None);
@@ -2307,7 +2360,7 @@ async fn create_adhoc_group(
         create_blocked,
         ProtectionStatus::Unprotected,
         None,
-        timestamp,
+        mime_parser.timestamp_sent,
     )
     .await?;
 
@@ -2318,10 +2371,13 @@ async fn create_adhoc_group(
     chat::add_to_chat_contacts_table(context, new_chat_id, member_ids).await?;
 
     context.emit_event(EventType::ChatModified(new_chat_id));
+    ui_events::emit_chatlist_changed(context);
+    ui_events::emit_chatlist_item_changed(context, new_chat_id);
 
     Ok(Some(new_chat_id))
 }
 
+#[derive(Debug, PartialEq, Eq)]
 enum VerifiedEncryption {
     Verified,
     NotVerified(String), // The string contains the reason why it's not verified
@@ -2494,18 +2550,23 @@ async fn mark_recipients_as_verified(
                     info!(context, "{verifier_addr} has verified {to_addr}.");
                     if let Some(fp) = peerstate.gossip_key_fingerprint.clone() {
                         peerstate.set_verified(PeerstateKeyType::GossipKey, fp, verifier_addr)?;
+                        peerstate.backward_verified_key_id =
+                            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
                         peerstate.save_to_db(&context.sql).await?;
 
-                        if !is_verified {
-                            let (to_contact_id, _) = Contact::add_or_lookup(
-                                context,
-                                "",
-                                &ContactAddress::new(&to_addr)?,
-                                Origin::Hidden,
-                            )
-                            .await?;
-                            ChatId::set_protection_for_contact(context, to_contact_id).await?;
-                        }
+                        let (to_contact_id, _) = Contact::add_or_lookup(
+                            context,
+                            "",
+                            &ContactAddress::new(&to_addr)?,
+                            Origin::Hidden,
+                        )
+                        .await?;
+                        ChatId::set_protection_for_contact(
+                            context,
+                            to_contact_id,
+                            mimeparser.timestamp_sent,
+                        )
+                        .await?;
                     }
                 } else {
                     // The contact already has a verified key.
@@ -2543,20 +2604,7 @@ async fn get_previous_message(
 ///
 /// Only messages that are not in the trash chat are considered.
 async fn get_rfc724_mid_in_list(context: &Context, mid_list: &str) -> Result<Option<Message>> {
-    if mid_list.is_empty() {
-        return Ok(None);
-    }
-
-    for id in parse_message_ids(mid_list).iter().rev() {
-        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
-            let msg = Message::load_from_db(context, msg_id).await?;
-            if msg.chat_id != DC_CHAT_ID_TRASH {
-                return Ok(Some(msg));
-            }
-        }
-    }
-
-    Ok(None)
+    message::get_latest_by_rfc724_mids(context, &parse_message_ids(mid_list)).await
 }
 
 /// Returns the last message referenced from References: header found in the database.

src/receive_imf/tests.rs

@@ -2,11 +2,11 @@ use tokio::fs;
 
 use super::*;
 use crate::aheader::EncryptPreference;
+use crate::chat::{self, get_chat_msgs, ChatItem, ChatVisibility};
 use crate::chat::{
     add_contact_to_chat, add_to_chat_contacts_table, create_group_chat, get_chat_contacts,
     is_contact_in_chat, remove_contact_from_chat, send_text_msg,
 };
-use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::{DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS};
@@ -628,7 +628,7 @@ async fn test_parse_ndn(
     rfc724_mid_outgoing: &str,
     raw_ndn: &[u8],
     error_msg: Option<&str>,
-) {
+) -> (TestContext, MsgId) {
     let t = TestContext::new().await;
     t.configure_addr(self_addr).await;
 
@@ -675,6 +675,40 @@ async fn test_parse_ndn(
     );
 
     assert_eq!(msg.error(), error_msg.map(|error| error.to_string()));
+    (t, msg_id)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_resend_after_ndn() -> Result<()> {
+    let (t, msg_id) = test_parse_ndn(
+            "alice@testrun.org",
+            "hcksocnsofoejx@five.chat",
+            "Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org",
+            include_bytes!("../../test-data/message/testrun_ndn.eml"),
+            Some("Undelivered Mail Returned to Sender – This is the mail system at host hq5.merlinux.eu.\n\nI\'m sorry to have to inform you that your message could not\nbe delivered to one or more recipients. It\'s attached below.\n\nFor further assistance, please send mail to postmaster.\n\nIf you do so, please include this problem report. You can\ndelete your own text from the attached returned message.\n\n                   The mail system\n\n<hcksocnsofoejx@five.chat>: host mail.five.chat[195.62.125.103] said: 550 5.1.1\n    <hcksocnsofoejx@five.chat>: Recipient address rejected: User unknown in\n    virtual mailbox table (in reply to RCPT TO command)"),
+        )
+        .await;
+    chat::resend_msgs(&t, &[msg_id]).await?;
+    let msg = Message::load_from_db(&t, msg_id).await?;
+    assert_eq!(msg.state, MessageState::OutPending);
+    assert_eq!(msg.error(), None);
+    // Alice receives a BCC-self copy of their message.
+    receive_imf(
+        &t,
+        "To: hcksocnsofoejx@five.chat\n\
+        From: alice@testrun.org\n\
+        Date: Today, 2 January 2024 00:00:00 -300\n\
+        Message-ID: Mr.A7pTA5IgrUA.q4bP41vAJOp@testrun.org\n\
+        \n\
+        hi"
+        .as_bytes(),
+        false,
+    )
+    .await?;
+    let msg = t.get_last_msg().await;
+    assert_eq!(msg.state, MessageState::OutDelivered);
+    assert_eq!(msg.error(), None);
+    Ok(())
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -3223,6 +3257,22 @@ async fn test_thunderbird_unsigned_with_unencrypted_subject() -> Result<()> {
     Ok(())
 }
 
+/// Tests that DC takes the correct Message-ID from the encrypted message part, not the unencrypted
+/// one messed up by the server.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_messed_up_message_id() -> Result<()> {
+    let t = TestContext::new_bob().await;
+
+    let raw = include_bytes!("../../test-data/message/messed_up_message_id.eml");
+    receive_imf(&t, raw, false).await?;
+    assert_eq!(
+        t.get_last_msg().await.rfc724_mid,
+        "0bb9ffe1-2596-d997-95b4-1fef8cc4808e@example.org"
+    );
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_mua_user_adds_member() -> Result<()> {
     let t = TestContext::new_alice().await;

src/scheduler.rs

@@ -590,7 +590,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
         .log_err(ctx)
         .ok();
 
-    connection.connectivity.set_connected(ctx).await;
+    connection.connectivity.set_idle(ctx).await;
 
     ctx.emit_event(EventType::ImapInboxIdle);
     let Some(session) = connection.session.take() else {
@@ -727,7 +727,7 @@ async fn smtp_loop(
             // Fake Idle
             info!(ctx, "smtp fake idle - started");
             match &connection.last_send_error {
-                None => connection.connectivity.set_connected(&ctx).await,
+                None => connection.connectivity.set_idle(&ctx).await,
                 Some(err) => connection.connectivity.set_err(&ctx, err).await,
             }
 

src/scheduler/connectivity.rs

@@ -33,10 +33,19 @@ enum DetailedConnectivity {
     #[default]
     Uninitialized,
     Connecting,
-    Working,
-    InterruptingIdle,
+
+    /// Connection is just established, but there may be work to do.
     Connected,
 
+    /// There is actual work to do, e.g. there are messages in SMTP queue
+    /// or we detected a message that should be downloaded.
+    Working,
+
+    InterruptingIdle,
+
+    /// Connection is established and is idle.
+    Idle,
+
     /// The folder was configured not to be watched or configured_*_folder is not set
     NotConfigured,
 }
@@ -54,6 +63,8 @@ impl DetailedConnectivity {
             // Just don't return a connectivity, probably the folder is configured not to be
             // watched or there is e.g. no "Sent" folder, so we are not interested in it
             DetailedConnectivity::NotConfigured => None,
+
+            DetailedConnectivity::Idle => Some(Connectivity::Connected),
         }
     }
 
@@ -65,7 +76,8 @@ impl DetailedConnectivity {
             DetailedConnectivity::Connecting => "<span class=\"yellow dot\"></span>".to_string(),
             DetailedConnectivity::Working
             | DetailedConnectivity::InterruptingIdle
-            | DetailedConnectivity::Connected => "<span class=\"green dot\"></span>".to_string(),
+            | DetailedConnectivity::Connected
+            | DetailedConnectivity::Idle => "<span class=\"green dot\"></span>".to_string(),
         }
     }
 
@@ -75,9 +87,9 @@ impl DetailedConnectivity {
             DetailedConnectivity::Uninitialized => "Not started".to_string(),
             DetailedConnectivity::Connecting => stock_str::connecting(context).await,
             DetailedConnectivity::Working => stock_str::updating(context).await,
-            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                stock_str::connected(context).await
-            }
+            DetailedConnectivity::InterruptingIdle
+            | DetailedConnectivity::Connected
+            | DetailedConnectivity::Idle => stock_str::connected(context).await,
             DetailedConnectivity::NotConfigured => "Not configured".to_string(),
         }
     }
@@ -94,9 +106,9 @@ impl DetailedConnectivity {
             // We don't know any more than that the last message was sent successfully;
             // since sending the last message, connectivity could have changed, which we don't notice
             // until another message is sent
-            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                stock_str::last_msg_sent_successfully(context).await
-            }
+            DetailedConnectivity::InterruptingIdle
+            | DetailedConnectivity::Connected
+            | DetailedConnectivity::Idle => stock_str::last_msg_sent_successfully(context).await,
             DetailedConnectivity::NotConfigured => "Not configured".to_string(),
         }
     }
@@ -108,8 +120,9 @@ impl DetailedConnectivity {
             DetailedConnectivity::Connecting => false,
             DetailedConnectivity::Working => false,
             DetailedConnectivity::InterruptingIdle => false,
-            DetailedConnectivity::Connected => true,
+            DetailedConnectivity::Connected => false, // Just connected, there may still be work to do.
             DetailedConnectivity::NotConfigured => true,
+            DetailedConnectivity::Idle => true,
         }
     }
 }
@@ -141,6 +154,9 @@ impl ConnectivityStore {
     pub(crate) async fn set_not_configured(&self, context: &Context) {
         self.set(context, DetailedConnectivity::NotConfigured).await;
     }
+    pub(crate) async fn set_idle(&self, context: &Context) {
+        self.set(context, DetailedConnectivity::Idle).await;
+    }
 
     async fn get_detailed(&self) -> DetailedConnectivity {
         self.0.lock().await.deref().clone()
@@ -164,6 +180,7 @@ pub(crate) async fn idle_interrupted(inbox: ConnectivityStore, oboxes: Vec<Conne
     // return Connected until DC is completely done with fetching folders; this also
     // includes scan_folders() which happens on the inbox thread.
     if *connectivity_lock == DetailedConnectivity::Connected
+        || *connectivity_lock == DetailedConnectivity::Idle
         || *connectivity_lock == DetailedConnectivity::NotConfigured
     {
         *connectivity_lock = DetailedConnectivity::InterruptingIdle;
@@ -172,7 +189,9 @@ pub(crate) async fn idle_interrupted(inbox: ConnectivityStore, oboxes: Vec<Conne
 
     for state in oboxes {
         let mut connectivity_lock = state.0.lock().await;
-        if *connectivity_lock == DetailedConnectivity::Connected {
+        if *connectivity_lock == DetailedConnectivity::Connected
+            || *connectivity_lock == DetailedConnectivity::Idle
+        {
             *connectivity_lock = DetailedConnectivity::InterruptingIdle;
         }
     }

src/securejoin.rs

@@ -20,10 +20,12 @@ use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateKeyType};
 use crate::qr::check_qr;
+use crate::securejoin::bob::JoinerProgress;
 use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::token;
 use crate::tools::time;
+use crate::ui_events;
 
 mod bob;
 mod bobstate;
@@ -173,7 +175,6 @@ async fn send_alice_handshake_msg(
     context: &Context,
     contact_id: ContactId,
     step: &str,
-    fingerprint: Option<Fingerprint>,
 ) -> Result<()> {
     let mut msg = Message {
         viewtype: Viewtype::Text,
@@ -183,9 +184,6 @@ async fn send_alice_handshake_msg(
     };
     msg.param.set_cmd(SystemMessage::SecurejoinMessage);
     msg.param.set(Param::Arg, step);
-    if let Some(fp) = fingerprint {
-        msg.param.set(Param::Arg3, fp.hex());
-    }
     msg.param.set_int(Param::GuaranteeE2ee, 1);
     chat::send_msg(
         context,
@@ -204,7 +202,9 @@ async fn info_chat_id(context: &Context, contact_id: ContactId) -> Result<ChatId
     Ok(chat_id_blocked.id)
 }
 
-async fn fingerprint_equals_sender(
+/// Checks fingerprint and marks the contact as forward verified
+/// if fingerprint matches.
+async fn verify_sender_by_fingerprint(
     context: &Context,
     fingerprint: &Fingerprint,
     contact_id: ContactId,
@@ -223,13 +223,17 @@ async fn fingerprint_equals_sender(
         }
     };
 
-    if let Some(peerstate) = peerstate {
+    if let Some(mut peerstate) = peerstate {
         if peerstate
             .public_key_fingerprint
             .as_ref()
             .filter(|&fp| fp == fingerprint)
             .is_some()
         {
+            let verifier = contact.get_addr().to_owned();
+            peerstate.set_verified(PeerstateKeyType::PublicKey, fingerprint.clone(), verifier)?;
+            peerstate.prefer_encrypt = EncryptPreference::Mutual;
+            peerstate.save_to_db(&context.sql).await?;
             return Ok(true);
         }
     }
@@ -327,7 +331,6 @@ pub(crate) async fn handle_securejoin_handshake(
                 context,
                 contact_id,
                 &format!("{}-auth-required", &step[..2]),
-                None,
             )
             .await
             .context("failed sending auth-required handshake message")?;
@@ -372,7 +375,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            if !fingerprint_equals_sender(context, &fingerprint, contact_id).await? {
+            if !verify_sender_by_fingerprint(context, &fingerprint, contact_id).await? {
                 could_not_establish_secure_connection(
                     context,
                     contact_id,
@@ -384,20 +387,17 @@ pub(crate) async fn handle_securejoin_handshake(
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
-            let auth_0 = match mime_message.get_header(HeaderDef::SecureJoinAuth) {
-                Some(auth) => auth,
-                None => {
-                    could_not_establish_secure_connection(
-                        context,
-                        contact_id,
-                        info_chat_id(context, contact_id).await?,
-                        "Auth not provided.",
-                    )
-                    .await?;
-                    return Ok(HandshakeMessage::Ignore);
-                }
+            let Some(auth) = mime_message.get_header(HeaderDef::SecureJoinAuth) else {
+                could_not_establish_secure_connection(
+                    context,
+                    contact_id,
+                    info_chat_id(context, contact_id).await?,
+                    "Auth not provided.",
+                )
+                .await?;
+                return Ok(HandshakeMessage::Ignore);
             };
-            if !token::exists(context, token::Namespace::Auth, auth_0).await {
+            if !token::exists(context, token::Namespace::Auth, auth).await {
                 could_not_establish_secure_connection(
                     context,
                     contact_id,
@@ -411,8 +411,14 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?
                 .get_addr()
                 .to_owned();
-            let fingerprint_found =
-                mark_peer_as_verified(context, fingerprint.clone(), contact_addr).await?;
+            let backward_verified = true;
+            let fingerprint_found = mark_peer_as_verified(
+                context,
+                fingerprint.clone(),
+                contact_addr,
+                backward_verified,
+            )
+            .await?;
             if !fingerprint_found {
                 could_not_establish_secure_connection(
                     context,
@@ -441,7 +447,13 @@ pub(crate) async fn handle_securejoin_handshake(
                 };
                 match chat::get_chat_id_by_grpid(context, field_grpid).await? {
                     Some((group_chat_id, _, _)) => {
-                        secure_connection_established(context, contact_id, group_chat_id).await?;
+                        secure_connection_established(
+                            context,
+                            contact_id,
+                            group_chat_id,
+                            mime_message.timestamp_sent,
+                        )
+                        .await?;
                         chat::add_contact_to_chat_ex(
                             context,
                             Nosync,
@@ -461,16 +473,12 @@ pub(crate) async fn handle_securejoin_handshake(
                     context,
                     contact_id,
                     info_chat_id(context, contact_id).await?,
+                    mime_message.timestamp_sent,
                 )
                 .await?;
-                send_alice_handshake_msg(
-                    context,
-                    contact_id,
-                    "vc-contact-confirm",
-                    Some(fingerprint),
-                )
-                .await
-                .context("failed sending vc-contact-confirm message")?;
+                send_alice_handshake_msg(context, contact_id, "vc-contact-confirm")
+                    .await
+                    .context("failed sending vc-contact-confirm message")?;
 
                 inviter_progress(context, contact_id, 1000);
             }
@@ -480,11 +488,18 @@ pub(crate) async fn handle_securejoin_handshake(
         ====             Bob - the joiner's side             ====
         ====   Step 7 in "Setup verified contact" protocol   ====
         =======================================================*/
-        "vc-contact-confirm" => match BobState::from_db(&context.sql).await? {
-            Some(bobstate) => bob::handle_contact_confirm(context, bobstate, mime_message).await,
-            None => Ok(HandshakeMessage::Ignore),
-        },
+        "vc-contact-confirm" => {
+            if let Some(mut bobstate) = BobState::from_db(&context.sql).await? {
+                if !bobstate.is_msg_expected(context, step.as_str()) {
+                    warn!(context, "Unexpected vc-contact-confirm.");
+                    return Ok(HandshakeMessage::Ignore);
+                }
 
+                bobstate.step_contact_confirm(context).await?;
+                bobstate.emit_progress(context, JoinerProgress::Succeeded);
+            }
+            Ok(HandshakeMessage::Ignore)
+        }
         "vg-member-added" => {
             let Some(member_added) = mime_message
                 .get_header(HeaderDef::ChatGroupMemberAdded)
@@ -492,23 +507,27 @@ pub(crate) async fn handle_securejoin_handshake(
             else {
                 warn!(
                     context,
-                    "vg-member-added without Chat-Group-Member-Added header"
+                    "vg-member-added without Chat-Group-Member-Added header."
                 );
                 return Ok(HandshakeMessage::Propagate);
             };
             if !context.is_self_addr(member_added).await? {
                 info!(
                     context,
-                    "Member {member_added} added by unrelated SecureJoin process"
+                    "Member {member_added} added by unrelated SecureJoin process."
                 );
                 return Ok(HandshakeMessage::Propagate);
             }
-            match BobState::from_db(&context.sql).await? {
-                Some(bobstate) => {
-                    bob::handle_contact_confirm(context, bobstate, mime_message).await
+            if let Some(mut bobstate) = BobState::from_db(&context.sql).await? {
+                if !bobstate.is_msg_expected(context, step.as_str()) {
+                    warn!(context, "Unexpected vg-member-added.");
+                    return Ok(HandshakeMessage::Propagate);
                 }
-                None => Ok(HandshakeMessage::Propagate),
+
+                bobstate.step_contact_confirm(context).await?;
+                bobstate.emit_progress(context, JoinerProgress::Succeeded);
             }
+            Ok(HandshakeMessage::Propagate)
         }
 
         "vg-member-added-received" | "vc-contact-confirm-received" => {
@@ -522,23 +541,25 @@ pub(crate) async fn handle_securejoin_handshake(
     }
 }
 
-/// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen.
+/// Observe self-sent Securejoin message.
 ///
-/// in a multi-device-setup, there may be other devices that "see" the handshake messages.
-/// if the seen messages seen are self-sent messages encrypted+signed correctly with our key,
-/// we can make some conclusions of it:
+/// In a multi-device-setup, there may be other devices that "see" the handshake messages.
+/// If we see self-sent messages encrypted+signed correctly with our key,
+/// we can make some conclusions of it.
 ///
-/// - if we see the self-sent-message vg-member-added/vc-contact-confirm,
-///   we know that we're an inviter-observer.
-///   The inviting device has marked a peer as verified on vg-request-with-auth/vc-request-with-auth
-///   before sending vg-member-added/vc-contact-confirm - so, if we observe vg-member-added/vc-contact-confirm,
-///   we can mark the peer as verified as well.
+/// If we see self-sent {vc,vg}-request-with-auth,
+/// we know that we are Bob (joiner-observer)
+/// that just marked peer (Alice) as forward-verified
+/// either after receiving {vc,vg}-auth-required
+/// or immediately after scanning the QR-code
+/// if the key was already known.
 ///
-/// - if we see the self-sent-message vg-request-with-auth/vc-request-with-auth
-///   we know that we're an joiner-observer.
-///   the joining device has marked the peer as verified
-///   before sending vg-request-with-auth/vc-request-with-auth - so, if we observe vg-member-added-received,
-///   we can mark the peer as verified as well.
+/// If we see self-sent vc-contact-confirm or vg-member-added message,
+/// we know that we are Alice (inviter-observer)
+/// that just marked peer (Bob) as forward (and backward)-verified
+/// in response to correct vc-request-with-auth message.
+///
+/// In both cases we can mark the peer as forward-verified.
 pub(crate) async fn observe_securejoin_on_other_device(
     context: &Context,
     mime_message: &MimeMessage,
@@ -552,122 +573,96 @@ pub(crate) async fn observe_securejoin_on_other_device(
         .context("Not a Secure-Join message")?;
     info!(context, "Observing secure-join message {step:?}.");
 
-    match step.as_str() {
-        "vg-request-with-auth"
-        | "vc-request-with-auth"
-        | "vg-member-added"
-        | "vc-contact-confirm" => {
-            if !encrypted_and_signed(
-                context,
-                mime_message,
-                get_self_fingerprint(context).await.as_ref(),
-            ) {
-                could_not_establish_secure_connection(
-                    context,
-                    contact_id,
-                    info_chat_id(context, contact_id).await?,
-                    "Message not encrypted correctly.",
-                )
-                .await?;
-                return Ok(HandshakeMessage::Ignore);
-            }
-            let addr = Contact::get_by_id(context, contact_id)
-                .await?
-                .get_addr()
-                .to_lowercase();
-            if mime_message.gossiped_addr.contains(&addr) {
-                let mut peerstate = match Peerstate::from_addr(context, &addr).await? {
-                    Some(p) => p,
-                    None => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_id,
-                            info_chat_id(context, contact_id).await?,
-                            &format!("No peerstate in db for '{}' at step {}", &addr, step),
-                        )
-                        .await?;
-                        return Ok(HandshakeMessage::Ignore);
-                    }
-                };
-                let fingerprint = match peerstate.gossip_key_fingerprint.clone() {
-                    Some(fp) => fp,
-                    None => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_id,
-                            info_chat_id(context, contact_id).await?,
-                            &format!(
-                                "No gossip key fingerprint in db for '{}' at step {}",
-                                &addr, step,
-                            ),
-                        )
-                        .await?;
-                        return Ok(HandshakeMessage::Ignore);
-                    }
-                };
-                peerstate.set_verified(PeerstateKeyType::GossipKey, fingerprint, addr)?;
-                peerstate.prefer_encrypt = EncryptPreference::Mutual;
-                peerstate.save_to_db(&context.sql).await.unwrap_or_default();
+    if !matches!(
+        step.as_str(),
+        "vg-request-with-auth" | "vc-request-with-auth" | "vg-member-added" | "vc-contact-confirm"
+    ) {
+        return Ok(HandshakeMessage::Ignore);
+    };
 
-                ChatId::set_protection_for_contact(context, contact_id).await?;
-            } else if let Some(fingerprint) =
-                mime_message.get_header(HeaderDef::SecureJoinFingerprint)
-            {
-                // FIXME: Old versions of DC send this header instead of gossips. Remove this
-                // eventually.
-                let fingerprint = fingerprint.parse()?;
-                let fingerprint_found = mark_peer_as_verified(
-                    context,
-                    fingerprint,
-                    Contact::get_by_id(context, contact_id)
-                        .await?
-                        .get_addr()
-                        .to_owned(),
-                )
-                .await?;
-                if !fingerprint_found {
-                    could_not_establish_secure_connection(
-                        context,
-                        contact_id,
-                        info_chat_id(context, contact_id).await?,
-                        format!("Fingerprint mismatch on observing {step}.").as_ref(),
-                    )
-                    .await?;
-                    return Ok(HandshakeMessage::Ignore);
-                }
-            } else {
-                could_not_establish_secure_connection(
-                    context,
-                    contact_id,
-                    info_chat_id(context, contact_id).await?,
-                    &format!(
-                        "No gossip header for '{}' at step {}, please update Delta Chat on all \
+    if !encrypted_and_signed(
+        context,
+        mime_message,
+        get_self_fingerprint(context).await.as_ref(),
+    ) {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            "Message not encrypted correctly.",
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    }
+
+    let addr = Contact::get_by_id(context, contact_id)
+        .await?
+        .get_addr()
+        .to_lowercase();
+
+    if !mime_message.gossiped_addr.contains(&addr) {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            &format!(
+                "No gossip header for '{}' at step {}, please update Delta Chat on all \
                         your devices.",
-                        &addr, step,
-                    ),
-                )
-                .await?;
-                return Ok(HandshakeMessage::Ignore);
-            }
-            if step.as_str() == "vg-member-added" {
-                inviter_progress(context, contact_id, 800);
-            }
-            if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" {
-                inviter_progress(context, contact_id, 1000);
-            }
-            if step.as_str() == "vg-request-with-auth" || step.as_str() == "vc-request-with-auth" {
-                // This actually reflects what happens on the first device (which does the secure
-                // join) and causes a subsequent "vg-member-added" message to create an unblocked
-                // verified group.
-                ChatId::create_for_contact_with_blocked(context, contact_id, Blocked::Not).await?;
-            }
-            Ok(if step.as_str() == "vg-member-added" {
-                HandshakeMessage::Propagate
-            } else {
-                HandshakeMessage::Ignore
-            })
-        }
-        _ => Ok(HandshakeMessage::Ignore),
+                &addr, step,
+            ),
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    }
+
+    let Some(mut peerstate) = Peerstate::from_addr(context, &addr).await? else {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            &format!("No peerstate in db for '{}' at step {}", &addr, step),
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    };
+
+    let Some(fingerprint) = peerstate.gossip_key_fingerprint.clone() else {
+        could_not_establish_secure_connection(
+            context,
+            contact_id,
+            info_chat_id(context, contact_id).await?,
+            &format!(
+                "No gossip key fingerprint in db for '{}' at step {}",
+                &addr, step,
+            ),
+        )
+        .await?;
+        return Ok(HandshakeMessage::Ignore);
+    };
+    peerstate.set_verified(PeerstateKeyType::GossipKey, fingerprint, addr)?;
+    peerstate.prefer_encrypt = EncryptPreference::Mutual;
+    peerstate.save_to_db(&context.sql).await?;
+
+    ChatId::set_protection_for_contact(context, contact_id, mime_message.timestamp_sent).await?;
+
+    if step.as_str() == "vg-member-added" {
+        inviter_progress(context, contact_id, 800);
+    }
+    if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" {
+        inviter_progress(context, contact_id, 1000);
+    }
+
+    if step.as_str() == "vg-request-with-auth" || step.as_str() == "vc-request-with-auth" {
+        // This actually reflects what happens on the first device (which does the secure
+        // join) and causes a subsequent "vg-member-added" message to create an unblocked
+        // verified group.
+        ChatId::create_for_contact_with_blocked(context, contact_id, Blocked::Not).await?;
+    }
+
+    if step.as_str() == "vg-member-added" {
+        Ok(HandshakeMessage::Propagate)
+    } else {
+        Ok(HandshakeMessage::Ignore)
     }
 }
 
@@ -675,24 +670,21 @@ async fn secure_connection_established(
     context: &Context,
     contact_id: ContactId,
     chat_id: ChatId,
+    timestamp: i64,
 ) -> Result<()> {
-    if context
-        .get_config_bool(Config::VerifiedOneOnOneChats)
+    let private_chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Yes)
         .await?
-    {
-        let private_chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Yes)
-            .await?
-            .id;
-        private_chat_id
-            .set_protection(
-                context,
-                ProtectionStatus::Protected,
-                time(),
-                Some(contact_id),
-            )
-            .await?;
-    }
+        .id;
+    private_chat_id
+        .set_protection(
+            context,
+            ProtectionStatus::Protected,
+            timestamp,
+            Some(contact_id),
+        )
+        .await?;
     context.emit_event(EventType::ChatModified(chat_id));
+    ui_events::emit_chatlist_item_changed(context, chat_id);
     Ok(())
 }
 
@@ -719,13 +711,18 @@ async fn mark_peer_as_verified(
     context: &Context,
     fingerprint: Fingerprint,
     verifier: String,
+    backward_verified: bool,
 ) -> Result<bool> {
     let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, &fingerprint).await? else {
         return Ok(false);
     };
     peerstate.set_verified(PeerstateKeyType::PublicKey, fingerprint, verifier)?;
     peerstate.prefer_encrypt = EncryptPreference::Mutual;
-    peerstate.save_to_db(&context.sql).await.unwrap_or_default();
+    if backward_verified {
+        peerstate.backward_verified_key_id =
+            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+    }
+    peerstate.save_to_db(&context.sql).await?;
     Ok(true)
 }
 
@@ -975,6 +972,7 @@ mod tests {
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
         peerstate.save_to_db(&bob.ctx.sql).await?;
@@ -1308,4 +1306,70 @@ First thread."#;
 
         Ok(())
     }
+
+    /// Tests that Bob gets Alice as verified
+    /// if `vc-contact-confirm` is lost but Alice then sends
+    /// a message to Bob in a verified 1:1 chat with a `Chat-Verified` header.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_lost_contact_confirm() {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+        alice
+            .set_config(Config::VerifiedOneOnOneChats, Some("1"))
+            .await
+            .unwrap();
+        bob.set_config(Config::VerifiedOneOnOneChats, Some("1"))
+            .await
+            .unwrap();
+
+        let qr = get_securejoin_qr(&alice.ctx, None).await.unwrap();
+        join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        // vc-request
+        let sent = bob.pop_sent_msg().await;
+        alice.recv_msg(&sent).await;
+
+        // vc-auth-required
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        // vc-request-with-auth
+        let sent = bob.pop_sent_msg().await;
+        alice.recv_msg(&sent).await;
+
+        // Alice has Bob verified now.
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown)
+                .await
+                .expect("Error looking up contact")
+                .expect("Contact not found");
+        let contact_bob = Contact::get_by_id(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(contact_bob.is_verified(&alice.ctx).await.unwrap(), true);
+
+        // Alice sends vc-contact-confirm, but it gets lost.
+        let _sent_vc_contact_confirm = alice.pop_sent_msg().await;
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob, "alice@example.org", Origin::Unknown)
+                .await
+                .expect("Error looking up contact")
+                .expect("Contact not found");
+        let contact_alice = Contact::get_by_id(&bob, contact_alice_id).await.unwrap();
+        assert_eq!(contact_alice.is_verified(&bob).await.unwrap(), false);
+
+        // Alice sends a text message to Bob.
+        let received_hello = tcm.send_recv(&alice, &bob, "Hello!").await;
+        let chat_id = received_hello.chat_id;
+        let chat = Chat::load_from_db(&bob, chat_id).await.unwrap();
+        assert_eq!(chat.is_protected(), true);
+
+        // Received text message in a verified 1:1 chat results in backward verification
+        // and Bob now marks alice as verified.
+        let contact_alice = Contact::get_by_id(&bob, contact_alice_id).await.unwrap();
+        assert_eq!(contact_alice.is_verified(&bob).await.unwrap(), true);
+    }
 }

src/securejoin/bob.rs

@@ -9,7 +9,6 @@ use super::bobstate::{BobHandshakeStage, BobState};
 use super::qrinvite::QrInvite;
 use super::HandshakeMessage;
 use crate::chat::{is_contact_in_chat, ChatId, ProtectionStatus};
-use crate::config::Config;
 use crate::constants::{Blocked, Chattype};
 use crate::contact::Contact;
 use crate::context::Context;
@@ -83,65 +82,31 @@ pub(super) async fn handle_auth_required(
     context: &Context,
     message: &MimeMessage,
 ) -> Result<HandshakeMessage> {
-    match BobState::from_db(&context.sql).await? {
-        Some(mut bobstate) => match bobstate.handle_message(context, message).await? {
-            Some(BobHandshakeStage::Terminated(why)) => {
-                bobstate.notify_aborted(context, why).await?;
-                Ok(HandshakeMessage::Done)
-            }
-            Some(_stage) => {
-                if bobstate.is_join_group() {
-                    // The message reads "Alice replied, waiting to be added to the group…",
-                    // so only show it on secure-join and not on setup-contact.
-                    let contact_id = bobstate.invite().contact_id();
-                    let msg = stock_str::secure_join_replies(context, contact_id).await;
-                    let chat_id = bobstate.joining_chat_id(context).await?;
-                    chat::add_info_msg(context, chat_id, &msg, time()).await?;
-                }
-                bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
-                Ok(HandshakeMessage::Done)
-            }
-            None => Ok(HandshakeMessage::Ignore),
-        },
-        None => Ok(HandshakeMessage::Ignore),
-    }
-}
-
-/// Handles `vc-contact-confirm` and `vg-member-added` handshake messages.
-///
-/// # Bob - the joiner's side
-/// ## Step 7 in the "Setup Contact protocol"
-pub(super) async fn handle_contact_confirm(
-    context: &Context,
-    mut bobstate: BobState,
-    message: &MimeMessage,
-) -> Result<HandshakeMessage> {
-    let retval = if bobstate.is_join_group() {
-        HandshakeMessage::Propagate
-    } else {
-        HandshakeMessage::Ignore
+    let Some(mut bobstate) = BobState::from_db(&context.sql).await? else {
+        return Ok(HandshakeMessage::Ignore);
     };
-    match bobstate.handle_message(context, message).await? {
+
+    match bobstate.handle_auth_required(context, message).await? {
         Some(BobHandshakeStage::Terminated(why)) => {
             bobstate.notify_aborted(context, why).await?;
             Ok(HandshakeMessage::Done)
         }
-        Some(BobHandshakeStage::Completed) => {
-            // Note this goes to the 1:1 chat, as when joining a group we implicitly also
-            // verify both contacts (this could be a bug/security issue, see
-            // e.g. https://github.com/deltachat/deltachat-core-rust/issues/1177).
-            bobstate.notify_peer_verified(context).await?;
-            bobstate.emit_progress(context, JoinerProgress::Succeeded);
-            Ok(retval)
+        Some(_stage) => {
+            if bobstate.is_join_group() {
+                // The message reads "Alice replied, waiting to be added to the group…",
+                // so only show it on secure-join and not on setup-contact.
+                let contact_id = bobstate.invite().contact_id();
+                let msg = stock_str::secure_join_replies(context, contact_id).await;
+                let chat_id = bobstate.joining_chat_id(context).await?;
+                chat::add_info_msg(context, chat_id, &msg, time()).await?;
+            }
+            bobstate
+                .set_peer_verified(context, message.timestamp_sent)
+                .await?;
+            bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
+            Ok(HandshakeMessage::Done)
         }
-        Some(_) => {
-            warn!(
-                context,
-                "Impossible state returned from handling handshake message"
-            );
-            Ok(retval)
-        }
-        None => Ok(retval),
+        None => Ok(HandshakeMessage::Ignore),
     }
 }
 
@@ -154,7 +119,7 @@ impl BobState {
         }
     }
 
-    fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
+    pub(crate) fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
         let contact_id = self.invite().contact_id();
         context.emit_event(EventType::SecurejoinJoinerProgress {
             contact_id,
@@ -217,28 +182,17 @@ impl BobState {
         Ok(())
     }
 
-    /// Notifies the user that the SecureJoin peer is verified.
-    ///
-    /// This creates an info message in the chat being joined.
-    async fn notify_peer_verified(&self, context: &Context) -> Result<()> {
+    /// Turns 1:1 chat with SecureJoin peer into protected chat.
+    pub(crate) async fn set_peer_verified(&self, context: &Context, timestamp: i64) -> Result<()> {
         let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        let chat_id = self.joining_chat_id(context).await?;
-
-        if context
-            .get_config_bool(Config::VerifiedOneOnOneChats)
-            .await?
-        {
-            self.alice_chat()
-                .set_protection(
-                    context,
-                    ProtectionStatus::Protected,
-                    time(),
-                    Some(contact.id),
-                )
-                .await?;
-        }
-
-        context.emit_event(EventType::ChatModified(chat_id));
+        self.alice_chat()
+            .set_protection(
+                context,
+                ProtectionStatus::Protected,
+                timestamp,
+                Some(contact.id),
+            )
+            .await?;
         Ok(())
     }
 }
@@ -247,7 +201,7 @@ impl BobState {
 ///
 /// This has an `From<JoinerProgress> for usize` impl yielding numbers between 0 and a 1000
 /// which can be shown as a progress bar.
-enum JoinerProgress {
+pub(crate) enum JoinerProgress {
     /// An error occurred.
     Error,
     /// vg-vc-request-with-auth sent.

src/securejoin/bobstate.rs

@@ -11,8 +11,9 @@ use anyhow::Result;
 use rusqlite::Connection;
 
 use super::qrinvite::QrInvite;
-use super::{encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified};
+use super::{encrypted_and_signed, verify_sender_by_fingerprint};
 use crate::chat::{self, ChatId};
+use crate::config::Config;
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::events::EventType;
@@ -21,7 +22,9 @@ use crate::key::{load_self_public_key, DcKey};
 use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
+use crate::securejoin::Peerstate;
 use crate::sql::Sql;
+use crate::tools::time;
 
 /// The stage of the [`BobState`] securejoin handshake protocol state machine.
 ///
@@ -30,14 +33,9 @@ use crate::sql::Sql;
 #[derive(Clone, Copy, Debug, Display)]
 pub enum BobHandshakeStage {
     /// Step 2 completed: (vc|vg)-request message sent.
-    ///
-    /// Note that this is only ever returned by [`BobState::start_protocol`] and never by
-    /// [`BobState::handle_message`].
     RequestSent,
     /// Step 4 completed: (vc|vg)-request-with-auth message sent.
     RequestWithAuthSent,
-    /// The protocol completed successfully.
-    Completed,
     /// The protocol prematurely terminated with given reason.
     Terminated(&'static str),
 }
@@ -92,21 +90,26 @@ impl BobState {
         invite: QrInvite,
         chat_id: ChatId,
     ) -> Result<(Self, BobHandshakeStage, Vec<Self>)> {
-        let (stage, next) =
-            if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await?
-            {
-                // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
-                info!(context, "Taking securejoin protocol shortcut");
-                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
-                    .await?;
-                (
-                    BobHandshakeStage::RequestWithAuthSent,
-                    SecureJoinStep::ContactConfirm,
-                )
-            } else {
-                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
-                (BobHandshakeStage::RequestSent, SecureJoinStep::AuthRequired)
-            };
+        let peer_verified =
+            verify_sender_by_fingerprint(context, invite.fingerprint(), invite.contact_id())
+                .await?;
+
+        let (stage, next);
+        if peer_verified {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+
+            stage = BobHandshakeStage::RequestWithAuthSent;
+            next = SecureJoinStep::ContactConfirm;
+        } else {
+            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
+
+            stage = BobHandshakeStage::RequestSent;
+            next = SecureJoinStep::AuthRequired;
+        };
+
         let (id, aborted_states) =
             Self::insert_new_db_entry(context, next, invite.clone(), chat_id).await?;
         let state = Self {
@@ -115,6 +118,12 @@ impl BobState {
             next,
             chat_id,
         };
+
+        if peer_verified {
+            // Mark 1:1 chat as verified already.
+            state.set_peer_verified(context, time()).await?;
+        }
+
         Ok((state, stage, aborted_states))
     }
 
@@ -230,13 +239,13 @@ impl BobState {
         Ok(())
     }
 
-    /// Handles the given message for the securejoin handshake for Bob.
+    /// Handles {vc,vg}-auth-required message of the securejoin handshake for Bob.
     ///
     /// If the message was not used for this handshake `None` is returned, otherwise the new
-    /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
-    /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
+    /// stage is returned.  Once [`BobHandshakeStage::Terminated`] is reached this
+    /// [`BobState`] should be destroyed,
     /// further calling it will just result in the messages being unused by this handshake.
-    pub(crate) async fn handle_message(
+    pub(crate) async fn handle_auth_required(
         &mut self,
         context: &Context,
         mime_message: &MimeMessage,
@@ -256,39 +265,7 @@ impl BobState {
             info!(context, "{} message out of sync for BobState", step);
             return Ok(None);
         }
-        match step.as_str() {
-            "vg-auth-required" | "vc-auth-required" => {
-                self.step_auth_required(context, mime_message).await
-            }
-            "vg-member-added" | "vc-contact-confirm" => {
-                self.step_contact_confirm(context, mime_message).await
-            }
-            _ => {
-                warn!(context, "Invalid step for BobState: {}", step);
-                Ok(None)
-            }
-        }
-    }
 
-    /// Returns `true` if the message is expected according to the protocol.
-    fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
-        let variant_matches = match self.invite {
-            QrInvite::Contact { .. } => step.starts_with("vc-"),
-            QrInvite::Group { .. } => step.starts_with("vg-"),
-        };
-        let step_matches = self.next.matches(context, step);
-        variant_matches && step_matches
-    }
-
-    /// Handles a *vc-auth-required* or *vg-auth-required* message.
-    ///
-    /// # Bob - the joiner's side
-    /// ## Step 4 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    async fn step_auth_required(
-        &mut self,
-        context: &Context,
-        mime_message: &MimeMessage,
-    ) -> Result<Option<BobHandshakeStage>> {
         info!(
             context,
             "Bob Step 4 - handling {{vc,vg}}-auth-required message."
@@ -303,14 +280,19 @@ impl BobState {
                 .await?;
             return Ok(Some(BobHandshakeStage::Terminated(reason)));
         }
-        if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.invite.contact_id())
-            .await?
+        if !verify_sender_by_fingerprint(
+            context,
+            self.invite.fingerprint(),
+            self.invite.contact_id(),
+        )
+        .await?
         {
             self.update_next(&context.sql, SecureJoinStep::Terminated)
                 .await?;
             return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
         }
         info!(context, "Fingerprint verified.",);
+
         self.update_next(&context.sql, SecureJoinStep::ContactConfirm)
             .await?;
         self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
@@ -318,36 +300,39 @@ impl BobState {
         Ok(Some(BobHandshakeStage::RequestWithAuthSent))
     }
 
+    /// Returns `true` if the message is expected according to the protocol.
+    pub(crate) fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
+        let variant_matches = match self.invite {
+            QrInvite::Contact { .. } => step.starts_with("vc-"),
+            QrInvite::Group { .. } => step.starts_with("vg-"),
+        };
+        let step_matches = self.next.matches(context, step);
+        variant_matches && step_matches
+    }
+
     /// Handles a *vc-contact-confirm* or *vg-member-added* message.
     ///
     /// # Bob - the joiner's side
     /// ## Step 7 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    ///
-    /// This deviates from the protocol by also sending a confirmation message in response
-    /// to the *vc-contact-confirm* message.  This has no specific value to the protocol and
-    /// is only done out of symmetry with *vg-member-added* handling.
-    async fn step_contact_confirm(
-        &mut self,
-        context: &Context,
-        mime_message: &MimeMessage,
-    ) -> Result<Option<BobHandshakeStage>> {
-        info!(
-            context,
-            "Bob Step 7 - handling vc-contact-confirm/vg-member-added message."
-        );
-        mark_peer_as_verified(
-            context,
-            self.invite.fingerprint().clone(),
-            mime_message.from.addr.to_string(),
-        )
-        .await?;
+    pub(crate) async fn step_contact_confirm(&mut self, context: &Context) -> Result<()> {
+        let fingerprint = self.invite.fingerprint();
+        let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, fingerprint).await?
+        else {
+            return Ok(());
+        };
+
+        // Mark peer as backward verified.
+        peerstate.backward_verified_key_id =
+            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+        peerstate.save_to_db(&context.sql).await?;
+
         Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
             .await?;
         context.emit_event(EventType::ContactsChanged(None));
 
         self.update_next(&context.sql, SecureJoinStep::Completed)
             .await?;
-        Ok(Some(BobHandshakeStage::Completed))
+        Ok(())
     }
 
     /// Sends the requested handshake message to Alice.
@@ -389,13 +374,13 @@ async fn send_handshake_message(
             // Sends the Secure-Join-Auth header in mimefactory.rs.
             msg.param.set(Param::Arg2, invite.authcode());
             msg.param.set_int(Param::GuaranteeE2ee, 1);
+
+            // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+            let bob_fp = load_self_public_key(context).await?.fingerprint();
+            msg.param.set(Param::Arg3, bob_fp.hex());
         }
     };
 
-    // Sends our own fingerprint in the Secure-Join-Fingerprint header.
-    let bob_fp = load_self_public_key(context).await?.fingerprint();
-    msg.param.set(Param::Arg3, bob_fp.hex());
-
     // Sends the grpid in the Secure-Join-Group header.
     if let QrInvite::Group { ref grpid, .. } = invite {
         msg.param.set(Param::Arg4, grpid);

src/smtp.rs

@@ -595,7 +595,13 @@ pub(crate) async fn send_msg_to_smtp(
     match status {
         SendResult::Retry => Err(format_err!("Retry")),
         SendResult::Success => {
-            msg_id.set_delivered(context).await?;
+            if !context
+                .sql
+                .exists("SELECT COUNT(*) FROM smtp WHERE msg_id=?", (msg_id,))
+                .await?
+            {
+                msg_id.set_delivered(context).await?;
+            }
             Ok(())
         }
         SendResult::Failure(err) => Err(format_err!("{}", err)),

src/smtp/send.rs

@@ -9,11 +9,6 @@ use crate::events::EventType;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is split to chunks.
-// this does not affect MIME'e `To:` header.
-// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
-pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
-
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Envelope error: {}", _0)]
@@ -43,40 +38,30 @@ impl Smtp {
         }
 
         let message_len_bytes = message.len();
+        let recipients_display = recipients
+            .iter()
+            .map(|x| x.as_ref())
+            .collect::<Vec<&str>>()
+            .join(",");
 
-        let chunk_size = context
-            .get_configured_provider()
-            .await?
-            .and_then(|provider| provider.opt.max_smtp_rcpt_to)
-            .map_or(DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
+        let envelope =
+            Envelope::new(self.from.clone(), recipients.to_vec()).map_err(Error::Envelope)?;
+        let mail = SendableEmail::new(envelope, message);
 
-        for recipients_chunk in recipients.chunks(chunk_size) {
-            let recipients_display = recipients_chunk
-                .iter()
-                .map(|x| x.as_ref())
-                .collect::<Vec<&str>>()
-                .join(",");
+        if let Some(ref mut transport) = self.transport {
+            transport.send(mail).await.map_err(Error::SmtpSend)?;
 
-            let envelope = Envelope::new(self.from.clone(), recipients_chunk.to_vec())
-                .map_err(Error::Envelope)?;
-            let mail = SendableEmail::new(envelope, message);
-
-            if let Some(ref mut transport) = self.transport {
-                transport.send(mail).await.map_err(Error::SmtpSend)?;
-
-                let info_msg = format!(
-                    "Message len={message_len_bytes} was SMTP-sent to {recipients_display}"
-                );
-                info!(context, "{info_msg}.");
-                context.emit_event(EventType::SmtpMessageSent(info_msg));
-                self.last_success = Some(std::time::SystemTime::now());
-            } else {
-                warn!(
-                    context,
-                    "uh? SMTP has no transport, failed to send to {}", recipients_display
-                );
-                return Err(Error::NoTransport);
-            }
+            let info_msg =
+                format!("Message len={message_len_bytes} was SMTP-sent to {recipients_display}");
+            info!(context, "{info_msg}.");
+            context.emit_event(EventType::SmtpMessageSent(info_msg));
+            self.last_success = Some(std::time::SystemTime::now());
+        } else {
+            warn!(
+                context,
+                "uh? SMTP has no transport, failed to send to {}", recipients_display
+            );
+            return Err(Error::NoTransport);
         }
         Ok(())
     }

src/sql.rs

@@ -572,22 +572,13 @@ impl Sql {
     pub async fn set_raw_config(&self, key: &str, value: Option<&str>) -> Result<()> {
         let mut lock = self.config_cache.write().await;
         if let Some(value) = value {
-            let exists = self
-                .exists("SELECT COUNT(*) FROM config WHERE keyname=?;", (key,))
-                .await?;
-
-            if exists {
-                self.execute("UPDATE config SET value=? WHERE keyname=?;", (value, key))
-                    .await?;
-            } else {
-                self.execute(
-                    "INSERT INTO config (keyname, value) VALUES (?, ?);",
-                    (key, value),
-                )
-                .await?;
-            }
+            self.execute(
+                "INSERT OR REPLACE INTO config (keyname, value) VALUES (?, ?)",
+                (key, value),
+            )
+            .await?;
         } else {
-            self.execute("DELETE FROM config WHERE keyname=?;", (key,))
+            self.execute("DELETE FROM config WHERE keyname=?", (key,))
                 .await?;
         }
         lock.insert(key.to_string(), value.map(|s| s.to_string()));
@@ -608,7 +599,7 @@ impl Sql {
 
         let mut lock = self.config_cache.write().await;
         let value = self
-            .query_get_value("SELECT value FROM config WHERE keyname=?;", (key,))
+            .query_get_value("SELECT value FROM config WHERE keyname=?", (key,))
             .await
             .context(format!("failed to fetch raw config: {key}"))?;
         lock.insert(key.to_string(), value.clone());

src/sql/migrations.rs

@@ -1,11 +1,13 @@
 //! Migrations module.
 
 use anyhow::{Context as _, Result};
+use rusqlite::OptionalExtension;
 
 use crate::config::Config;
-use crate::constants::ShowEmails;
+use crate::constants::{self, ShowEmails};
 use crate::context::Context;
 use crate::imap;
+use crate::message::MsgId;
 use crate::provider::get_provider_by_domain;
 use crate::sql::Sql;
 use crate::tools::EmailAddress;
@@ -785,6 +787,119 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
         .await?;
     }
 
+    if dbversion < 106 {
+        // Recreate `config` table with UNIQUE constraint on `keyname`.
+        sql.execute_migration(
+            "CREATE TABLE new_config (
+               id INTEGER PRIMARY KEY,
+               keyname TEXT UNIQUE,
+               value TEXT NOT NULL
+             );
+             INSERT OR IGNORE INTO new_config SELECT
+               id, keyname, value
+             FROM config;
+             DROP TABLE config;
+             ALTER TABLE new_config RENAME TO config;
+             CREATE INDEX config_index1 ON config (keyname);",
+            106,
+        )
+        .await?;
+    }
+
+    if dbversion < 107 {
+        sql.execute_migration(
+            "CREATE TABLE new_keypairs (
+               id INTEGER PRIMARY KEY AUTOINCREMENT,
+               private_key UNIQUE NOT NULL,
+               public_key UNIQUE NOT NULL
+             );
+             INSERT OR IGNORE INTO new_keypairs SELECT id, private_key, public_key FROM keypairs;
+
+             INSERT OR IGNORE
+             INTO config (keyname, value)
+             VALUES
+             ('key_id', (SELECT id FROM new_keypairs
+                         WHERE private_key=
+                           (SELECT private_key FROM keypairs
+                            WHERE addr=(SELECT value FROM config WHERE keyname='configured_addr')
+                            AND is_default=1)));
+
+             -- We do not drop the old `keypairs` table for now,
+             -- but move it to `old_keypairs`. We can remove it later
+             -- in next migrations. This may be needed for recovery
+             -- in case something is wrong with the migration.
+             ALTER TABLE keypairs RENAME TO old_keypairs;
+             ALTER TABLE new_keypairs RENAME TO keypairs;
+             ",
+            107,
+        )
+        .await?;
+    }
+
+    if dbversion < 108 {
+        let version = 108;
+        let chunk_size = context
+            .get_configured_provider()
+            .await?
+            .and_then(|provider| provider.opt.max_smtp_rcpt_to)
+            .map_or(constants::DEFAULT_MAX_SMTP_RCPT_TO, usize::from);
+        sql.transaction(move |trans| {
+            Sql::set_db_version_trans(trans, version)?;
+            let id_max =
+                trans.query_row("SELECT IFNULL((SELECT MAX(id) FROM smtp), 0)", (), |row| {
+                    let id_max: i64 = row.get(0)?;
+                    Ok(id_max)
+                })?;
+            while let Some((id, rfc724_mid, mime, msg_id, recipients, retries)) = trans
+                .query_row(
+                    "SELECT id, rfc724_mid, mime, msg_id, recipients, retries FROM smtp \
+                    WHERE id<=? LIMIT 1",
+                    (id_max,),
+                    |row| {
+                        let id: i64 = row.get(0)?;
+                        let rfc724_mid: String = row.get(1)?;
+                        let mime: String = row.get(2)?;
+                        let msg_id: MsgId = row.get(3)?;
+                        let recipients: String = row.get(4)?;
+                        let retries: i64 = row.get(5)?;
+                        Ok((id, rfc724_mid, mime, msg_id, recipients, retries))
+                    },
+                )
+                .optional()?
+            {
+                trans.execute("DELETE FROM smtp WHERE id=?", (id,))?;
+                let recipients = recipients.split(' ').collect::<Vec<_>>();
+                for recipients in recipients.chunks(chunk_size) {
+                    let recipients = recipients.join(" ");
+                    trans.execute(
+                        "INSERT INTO smtp (rfc724_mid, mime, msg_id, recipients, retries) \
+                        VALUES (?, ?, ?, ?, ?)",
+                        (&rfc724_mid, &mime, msg_id, recipients, retries),
+                    )?;
+                }
+            }
+            Ok(())
+        })
+        .await
+        .with_context(|| format!("migration failed for version {version}"))?;
+
+        sql.set_db_version_in_cache(version).await?;
+    }
+
+    if dbversion < 109 {
+        sql.execute_migration(
+            r#"ALTER TABLE acpeerstates
+               ADD COLUMN backward_verified_key_id -- What we think the contact has as our verified key
+               INTEGER;
+               UPDATE acpeerstates
+               SET backward_verified_key_id=(SELECT value FROM config WHERE keyname='key_id')
+               WHERE verified_key IS NOT NULL
+               "#,
+            109,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?
@@ -824,6 +939,12 @@ impl Sql {
         Ok(())
     }
 
+    async fn set_db_version_in_cache(&self, version: i32) -> Result<()> {
+        let mut lock = self.config_cache.write().await;
+        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
+        Ok(())
+    }
+
     async fn execute_migration(&self, query: &str, version: i32) -> Result<()> {
         self.transaction(move |transaction| {
             Self::set_db_version_trans(transaction, version)?;
@@ -834,10 +955,6 @@ impl Sql {
         .await
         .with_context(|| format!("execute_migration failed for version {version}"))?;
 
-        let mut lock = self.config_cache.write().await;
-        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
-        drop(lock);
-
-        Ok(())
+        self.set_db_version_in_cache(version).await
     }
 }

src/sync.rs

@@ -447,7 +447,7 @@ mod tests {
         )?;
         assert_eq!(sync_items.items.len(), 1);
         let SyncDataOrUnknown::SyncData(AlterChat { id, action }) =
-            &sync_items.items.get(0).unwrap().data
+            &sync_items.items.first().unwrap().data
         else {
             bail!("bad item");
         };
@@ -491,7 +491,7 @@ mod tests {
 
         assert_eq!(sync_items.items.len(), 1);
         if let SyncDataOrUnknown::SyncData(AddQrToken(token)) =
-            &sync_items.items.get(0).unwrap().data
+            &sync_items.items.first().unwrap().data
         {
             assert_eq!(token.invitenumber, "in");
             assert_eq!(token.auth, "yip");

src/test_utils.rs

@@ -1047,7 +1047,8 @@ fn print_logevent(logevent: &LogEvent) {
     }
 }
 
-/// Saves the other account's public key as verified.
+/// Saves the other account's public key as verified
+/// and peerstate as backwards verified.
 pub(crate) async fn mark_as_verified(this: &TestContext, other: &TestContext) {
     let mut peerstate = Peerstate::from_header(
         &EncryptHelper::new(other).await.unwrap().get_aheader(),
@@ -1063,6 +1064,7 @@ pub(crate) async fn mark_as_verified(this: &TestContext, other: &TestContext) {
 
     peerstate.verified_key = peerstate.public_key.clone();
     peerstate.verified_key_fingerprint = peerstate.public_key_fingerprint.clone();
+    peerstate.backward_verified_key_id = Some(this.get_config_i64(Config::KeyId).await.unwrap());
 
     peerstate.save_to_db(&this.sql).await.unwrap();
 }

src/webxdc.rs

@@ -1241,7 +1241,7 @@ mod tests {
         )
         .await?
         .unwrap();
-        assert_eq!(*received_msg.msg_ids.get(0).unwrap(), bob_instance.id);
+        assert_eq!(*received_msg.msg_ids.first().unwrap(), bob_instance.id);
         let bob_instance = bob.get_last_msg().await;
         assert_eq!(bob_instance.viewtype, Viewtype::Webxdc);
         assert_eq!(bob_instance.download_state, DownloadState::Done);

test-data/message/messed_up_message_id.eml

@@ -0,0 +1,93 @@
+From - Thu, 24 Nov 2022 19:06:16 GMT
+X-Mozilla-Status: 0001
+X-Mozilla-Status2: 00800000
+Message-ID: <0bb9ffe1-2596-d997-95b4-1fef8cc4808f@example.org>
+Date: Thu, 24 Nov 2022 20:05:57 +0100
+MIME-Version: 1.0
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101
+ Thunderbird/102.4.2
+From: Alice <alice@example.org>
+To: bob@example.net
+Content-Language: en-US
+Autocrypt: addr=alice@example.org; keydata=
+ xjMEXlh13RYJKwYBBAHaRw8BAQdAzfVIAleCXMJrq8VeLlEVof6ITCviMktKjmcBKAu4m5DN
+ GUFsaWNlIDxhbGljZUBleGFtcGxlLm9yZz7CkAQTFggAOBYhBC5vossjtTLXKGNLWGSwj2Gp
+ 7ZRDBQJeWHXdAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEGSwj2Gp7ZRDE3oA/i4M
+ CyDMTsjWqDZoQwX/A/GoTO2/V0wKPhjJJy/8m2pMAPkBjOnGOtx2SZpQvJGTa9h804RY6iDr
+ RuI8A/8tEEXAA844BF5Ydd0SCisGAQQBl1UBBQEBB0AG7cjWy2SFAU8KnltlubVW67rFiyfp
+ 01JrRe6Xqy22HQMBCAfCeAQYFggAIBYhBC5vossjtTLXKGNLWGSwj2Gp7ZRDBQJeWHXdAhsM
+ AAoJEGSwj2Gp7ZRDLo8BAObE8GnsGVwKzNqCvHeWgJsqhjS3C6gvSlV3tEm9XmF6AQDXucIy
+ VfoBwoyMh2h6cSn/ATn5QJb35pgo+ivp3jsMAg==
+Subject: ...
+Content-Type: multipart/encrypted;
+ protocol="application/pgp-encrypted";
+ boundary="------------EOdOT2kJUL5hgCilmIhYyVZg"
+
+This is an OpenPGP/MIME encrypted message (RFC 4880 and 3156)
+--------------EOdOT2kJUL5hgCilmIhYyVZg
+Content-Type: application/pgp-encrypted
+Content-Description: PGP/MIME version identification
+
+Version: 1
+
+--------------EOdOT2kJUL5hgCilmIhYyVZg
+Content-Type: application/octet-stream; name="encrypted.asc"
+Content-Description: OpenPGP encrypted message
+Content-Disposition: inline; filename="encrypted.asc"
+
+-----BEGIN PGP MESSAGE-----
+
+wV4D5tq63hTeebASAQdA1dVUsUjGZCOIfCnYtVdmOvKs/BNovI3sG8w1IH4ymTMwAZzgwVbGS5KL
++e1VTD5mUTeVSEYe1cd3VozH4KbNJa1tBlcO0nzGwCPpsTVDMoxIwcBMA+PY3JvEjuMiAQf/d2yj
+t0+GyaptwX26bgSqo6vj21W8mcWS5vXOi8wjGwRbPaKKjS4kq1xDOz04eHrE8HUPD8otcXoI8CLz
+etJpRbFs0XJP4Cozbsr72dgoWhozRg/iSpBndxWOddTl7Yqo8m/fyhU5uzKZ41m2T8mha6KkKWD8
+QecGdOgieYBucNBjHwWc71p9G6jTnzfy4S4GtGS2gwOSMxpwO7HxpKzsHI4POqFSQbxrl/YRwWSC
+f5WqyYcerasIiR/fnOIw8lnvCeQ5rB90eGEDR70YFGt0t4rFBjfGrSPUiWYaTaC1Zvpd+t5sy7zy
+FpsS2/aTkwP/UpGqmtFaD/brSouRf9hijNLI0QFTaVmSoI3BKzF8B4zwvtEbOLZjyDb+Va/fZJ3w
+nYd2Q/5PPPL+pE4pWKN+jl0TZNzAaqBgvggXomgUqQ7QiksUzym+yuFKrJX0RF2awdrgjQIxjnda
+Qp3UFphnFTyYUJpIU9iewjOfVxgPzv7PyuCHYwoP3kh7MJZ6bgbDmOkeFSnjEDJpdf1m9xC9LlBL
+beC8scmPs6kx9GARBYSHvyPQ025gN3+XEHh4OrTxHZ91U3IlTfd2kACwOOAXEuhItSHmcNOV0K4M
+nI2PH6gW8HgBkWlAPm40K4jUyo3nl1usDiI6ouvYqvW7YUc2hTtPTej1l2/mS57tTt+PFurKs555
+5R9DD/xg9Nx7OuQKy5bIdlXM20UmwuZTOhRJ5kpHFRzLxaHDbSzW+orhRW4llJSevBSAH3cLOjIQ
+gh87j+MxG9j0TD2K2A0rcUcxdrnflw+mxcDVaL4payeqmOa+bJyhlftTqH+vqq5DhR68rX5VW+z7
+riqH3o8VbvO2y0XSpYHf1jowkfJj3vr8pynAUIv1dbylUSF5wtrHvzWOprw4bNrdtwQNRNy+JcVF
+dUKeNmHaL6XOe4LUWpiI11beRyCpAG52khMCEAO3Q6+4e24cEipbu6suSOtv3OpYDZeHjwNrQIhi
+rJg7i9TpMqwOeCvFWK+9UZ+P2n6h9g0/JO2+I82BFGUjVa5IvCTNOgv01GqxWY9ecdtaJjTc+dF2
+OAcRoKwvmtMJlxKEEgveui3BvPA4tuNdSrcoZBrQeo0ZHWVugXPvEZnwfZMcqwwPA+a/sUbZFg0P
+Pr0AR0ZHpytnQE9OXE8wEUgT8H1yofQ+5QoZdgMpeAb8zGs+RuviLxcDkb9NtXUAiQ49ooWuFP3L
+K9wMlaoWFTq7R+n5JVuSEYRCHC0l0bCV1/+awalT7XltXVCupI4lWzjYs52FZGGzuHG7S50Eufad
+m4CQTPVgVaVn8WW2dmpMR8Gj8WbbZdyv21wMGOWjfgT0u3oiDnddGrFOoMNnZHch6rN3FRppoh7h
+0U0fi8xxU1+EhUKq+fSIxZNr2iWN2if3Pipbxi9tyK9M41Y6aVF3HWjD58/OEql3aZjJZ1bqpXcE
+qsPeFoXX78+7mTDvL75olMk2s/mg4mLqAAWQvTuoiOmj+SgMIFuTtFR+4r/TIFNdamz6AQ3RcmWG
+ZcdRii+V27dtMA836vlAwxXRmJyE1LCL1kvUTq+J+AVsZi3xmBLFNlKPTlxswu7vSBrP1DlYOaBq
+AgA0lKnkQdeXyDk/VdbTml7ywMW1g6HkFSqKGW/IIAObmBumBcIyHE6dWEHumRQomlJssIlEFSe+
+XEQ0rwedLetJXi5A0AXT1we1wvaKCEg0Pb0ZUxygwNPDrj6MmdodH7gDfyx0mW/7mEMCtIJb5MB+
+TRGPEa/vqdJb8uGtNXUy9UlwMhJ3tYoT7NXY4+IlNjbDH/yleMdwtWP2H2WH8oC+ysXPYXjlT8eU
+poxRfJzPMVUn5SA3cvdGXDJWdX8U91j5sf9wuoYE5RBVrrJif3D3l0FpMrlWWoGw7wtZbMC2FaeT
+QvdMS5c54IoXBtBTM+/AsTAw7WEE1QSmaQGHnh6xLL5Ns8olsWeKOMlVXdO9jSDbjOGBLr7mWukW
+YzLXkH3TtJPQcbVN79af3YPhaHdMYITVKIwfg+vxZlLFHWLJQnkTl+9Qi7u2gKqkNeU7Zqs4E3CR
+9K4dHrJMyAZLZ2HA1XQEj0/tMnbTpAzZhj02JRcFobLXK9SQfw7dzGZwMRky8cHcBHoK14P5RIEV
+hr+38HSBM6wXtge5gL6DomAACvuORQO4X9x/CTjRt/J8uN3lKK5p+wi3ULeb319CEWiCiqmC1M+C
+TADUhPUhUmTinSAVkTEn+BdbH/97dVaJnvd6HtLmdSlw4xqdWUfVL9Qd7+/5L6iwlOzGLKRv97c/
+gCRw+hzXyAom+5C18slSwanMuyPgIyrrFy/kp9Romk9SQr/c0CUF2am99t8G5qvVi/TiJGHyKEXD
+aUYd4V7lqNlHMiiasvFHeq8blwmFr7rGEvbZzLNplc6sRUVlYhY2unRfyWsq9mqk3NDRW12Fa0J2
+YxQJlnXHQhNE8EyM/zsD9jCVNwsRZJ9/e5KS+ignmu6gKIR+ItDTwRfNI+NG/YmTgENUTyuO+vQC
+CUKS3PCwpP+OEC966ARl7OCMdfn1hEyiAxsZnp1RmFngR6FM+mlGgfUoWNoHvnR1/YyQ4F4dadiA
+QINwuSm5faw75F1EeL8Qi+LHKuqt05Pi/V9GJ6TzIkIsEbyyJ5sKHrp4QsU4C1p7ZhPjddz8De8k
+6ZdwMIeXxi27WKtsFLcr8JKOBe0imIilKdMBOPS31pc1iJe4472WbWM0aBwdEYmnz9+xfOqnjHtO
+0XTMjff7pzV6Y7t/u8J/zm3JS3ykote9HNRQvhZZNeVClVWd0fYFzat5ESnTojZTwHcc/BFTPnhz
+VgLyw1KEIy2r3ZyGHu1b8GSYivzl33MOK/NVBQPZUIEfdcQ5vhkAvj+Yx340IYykRFEChwioprXD
+LrIbTou7TNT5fTFA+beidHFsL+OE002/LMs6C3erSUW5C/LNjAQMS7cAV2yCyjX+/2GBmmDqnC4r
+Ja2x5yik+fbOUPh3kk/md1YvrodlX/JkQeoWRrrVJsX2dr3BgivPJavaN0Jz1eHyxAYKNqlrfd1T
+YWEDIisWerTxAVY/rEruZ6+OqLqOtZtn+4SOajOq8KFusglaMZqoYuM+LhPZck9PlZXwRqX08Vlv
+8jX5V75BFWRhFd5/LYbnQHI6ZW80Wb2sBNngLL2QJT9yXGCDJb5qCdFwGd3i655pvRJXabeyCtDD
+7I2PJcYRDd4stdq07BHyHJmye6vas8mG5QUygyWyUQv78za0m4gLMrRZBgoBDcVpWJUc+cPXzzfG
+7PvLZu/Y0SaD5hqTp0LBB1PFxTpzdVeJ21gzVNQ6D4XGLTtdv4K4fOEYoeKEuzGoBaUDtIqz47gd
+5rwfQ3ps2slkxfbtQcdKEACKvsCwzqHlgwsxD8QNOFzXYLiiiJBX22fIRoiJeSDMKSZyuFtpykCm
+7bOpybPSHv3E7EIr8sIOr9MOe/R5HSthU2IgW1L5Ynr2t9HUnCA8CenkzIQjg0h5sruxcGWCYLx7
+q0f1AQs4Z7SebVbq1SCWVJNX/vc1bVjnjYfri7RX5WMmjJkuSnuIoP6a42cqJcAg7m0STB0elFAy
+oO4vW9/JEmFUqLyQmWnoLJHX3IKtWa9CPvE=
+=OA6b
+-----END PGP MESSAGE-----
+
+--------------EOdOT2kJUL5hgCilmIhYyVZg--

test-data/message/schleuder.eml

@@ -0,0 +1,66 @@
+Return-Path: <mailing-list-bounce@example.org>
+Delivered-To: alice@testrun.org
+Date: Tue, 02 Jan 2024 05:00:00 +0000
+From: mailing-list@example.org
+Sender: mailing-list-bounce@example.org
+To: alice@testrun.org
+Message-ID: <87wmss8juz.fsf@example.org>
+In-Reply-To: 
+References: 
+Subject: [REPOST] Some subject
+Mime-Version: 1.0
+Content-Type: multipart/signed;
+ boundary="--==_mimepart_65938a80866e8_663a2abed9b585c064398";
+ micalg=pgp-sha1;
+ protocol="application/pgp-signature"
+Content-Transfer-Encoding: 7bit
+List-Id: <mailing-list.example.org>
+List-Owner: <mailto:mailing-list-owner@example.org> (Use list's public
+ key)
+List-Help: <https://schleuder.org/>
+List-Post: <mailto:mailing-list@example.org>
+
+This is an OpenPGP/MIME signed message (RFC 4880 and 3156)
+----==_mimepart_65938a80866e8_663a2abed9b585c064398
+Content-Type: multipart/mixed;
+ boundary="--==_mimepart_65938a8086476_663a2abed9b585c0642c7";
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+
+----==_mimepart_65938a8086476_663a2abed9b585c0642c7
+Content-Type: text/plain;
+ charset=UTF-8
+Content-Transfer-Encoding: 7bit
+
+From: bob@example.org
+To: mailing-list@example.org
+Cc: 
+Date: Tue, 02 Jan 2024 05:00:00 +0000
+Sig: Unsigned
+Enc: Unencrypted
+
+----==_mimepart_65938a8086476_663a2abed9b585c0642c7
+Content-Type: text/plain;
+ charset=utf-8
+Content-Transfer-Encoding: quoted-printable
+
+hello,
+bye
+
+----==_mimepart_65938a8086476_663a2abed9b585c0642c7--
+
+----==_mimepart_65938a80866e8_663a2abed9b585c064398
+Content-Type: application/pgp-signature;
+ name=signature.asc
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment;
+ filename=signature.asc
+Content-Description: OpenPGP digital signature
+
+-----BEGIN PGP SIGNATURE-----
+
+REDACTED
+-----END PGP SIGNATURE-----
+
+----==_mimepart_65938a80866e8_663a2abed9b585c064398--