Fingerprint cannot be the primary key

This is necessary to debug the tests.

CHANGELOG.md

@@ -98,6 +98,8 @@
 - [**breaking**] Remove unused `dc_set_chat_protection()`
 - Hide `DcSecretKey` trait from the API.
 - Verified 1:1 chats ([#4315](https://github.com/deltachat/deltachat-core-rust/pull/4315)). Disabled by default, enable with `verified_one_on_one_chats` config.
+- Add api `chat::Chat::is_protection_broken`
+- Add `dc_chat_is_protection_broken()` C API.
 
 ### CI
 

Cargo.lock

@@ -2335,9 +2335,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "human-panic"
-version = "1.2.1"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b82da652938b83f94cfdaaf9ae7aaadb8430d84b0dfda226998416318727eac2"
+checksum = "7a79a67745be0cb8dd2771f03b24c2f25df98d5471fe7a595d668cfa2e6f843d"
 dependencies = [
  "backtrace",
  "os_info",
@@ -4310,9 +4310,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.190"
+version = "1.0.191"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7"
+checksum = "a834c4821019838224821468552240d4d95d14e751986442c816572d39a080c9"
 dependencies = [
  "serde_derive",
 ]
@@ -4337,9 +4337,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.190"
+version = "1.0.191"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3"
+checksum = "46fa52d5646bce91b680189fe5b1c049d2ea38dabb4e2e7c8d00ca12cfbfbcfd"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4995,9 +4995,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.7.8"
+version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257"
+checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -5016,9 +5016,9 @@ dependencies = [
 
 [[package]]
 name = "toml_edit"
-version = "0.19.15"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
+checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03"
 dependencies = [
  "indexmap 2.1.0",
  "serde",

Cargo.toml

@@ -90,7 +90,7 @@ tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 tokio-util = "0.7.9"
-toml = "0.7"
+toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 

deltachat-ffi/deltachat.h

@@ -3743,13 +3743,9 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
  * if `verified_one_on_one_chats` setting is enabled.
  *
  * UI should display a green checkmark
- * in the chat title,
- * in the chat profile title and
+ * in the chat title and
  * in the chatlist item
  * if chat protection is enabled.
- * UI should also display a green checkmark
- * in the contact profile
- * if 1:1 chat with this contact exists and is protected.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -5061,9 +5057,7 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
  *
  * Do not use this function when displaying the contact profile view.
  * Display green checkmark in the title of the contact profile
- * if 1:1 chat with the contact exists and is protected.
- * Use dc_contact_get_verified_id to display the verifier contact
- * in the info section of the contact profile.
+ * if dc_contact_profile_is_verified() returns true.
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5073,6 +5067,23 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
+/**
+ * Check if the contact profile title
+ * should contain a green checkmark.
+ * This indicates whether 1:1 chat with
+ * this contact has a green checkmark
+ * or will have if such chat is created.
+ *
+ * Use dc_contact_get_verified_id to display the verifier contact
+ * in the info section of the contact profile.
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 1=contact profile has a green checkmark in the title,
+ *   0=contact profile has no green checkmark in the title.
+ */
+int             dc_contact_profile_is_verified (dc_contact_t* contact);
+
 
 /**
  * Return the contact ID that verified a contact.
@@ -6249,6 +6260,7 @@ void dc_event_unref(dc_event_t* event);
  * @param data2 (int) The progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
+ *     1000=vg-member-added/vc-contact-confirm received
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 

deltachat-ffi/src/lib.rs

@@ -4119,6 +4119,21 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
         .unwrap_or_default() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_profile_is_verified(contact: *mut dc_contact_t) -> libc::c_int {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_profile_is_verified()");
+        return 0;
+    }
+    let ffi_contact = &*contact;
+    let ctx = &*ffi_contact.context;
+
+    block_on(ffi_contact.contact.is_profile_verified(ctx))
+        .context("is_profile_verified failed")
+        .log_err(ctx)
+        .unwrap_or_default() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) -> u32 {
     if contact.is_null() {

deltachat-jsonrpc/src/api/types/contact.rs

@@ -24,11 +24,16 @@ pub struct ContactObject {
     ///
     /// If this is true
     /// UI should display green checkmark after the contact name
-    /// in the title of the contact profile,
     /// in contact list items and
     /// in chat member list items.
     is_verified: bool,
 
+    /// True if the contact profile title should have a green checkmark.
+    ///
+    /// This indicates whether 1:1 chat has a green checkmark
+    /// or will have a green checkmark if created.
+    is_profile_verified: bool,
+
     /// The ID of the contact that verified this contact.
     ///
     /// If this is present,
@@ -52,6 +57,7 @@ impl ContactObject {
             None => None,
         };
         let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+        let is_profile_verified = contact.is_profile_verified(context).await?;
 
         let verifier_id = contact
             .get_verifier_id(context)
@@ -70,6 +76,7 @@ impl ContactObject {
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
             is_verified,
+            is_profile_verified,
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -21,7 +21,9 @@ class ACFactory:
         self.deltachat = deltachat
 
     def get_unconfigured_account(self) -> Account:
-        return self.deltachat.add_account()
+        account = self.deltachat.add_account()
+        account.set_config("verified_one_on_one_chats", "1")
+        return account
 
     def get_unconfigured_bot(self) -> Bot:
         return Bot(self.get_unconfigured_account())

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,11 +1,12 @@
+import itertools
 import json
 import logging
 import os
 import subprocess
 import sys
 from queue import Queue
-from threading import Event, Lock, Thread
-from typing import Any, Dict, Optional
+from threading import Event, Thread
+from typing import Any, Dict, Iterator, Optional
 
 
 class JsonRpcError(Exception):
@@ -23,8 +24,7 @@ class Rpc:
 
         self._kwargs = kwargs
         self.process: subprocess.Popen
-        self.id: int
-        self.id_lock: Lock
+        self.id_iterator: Iterator[int]
         self.event_queues: Dict[int, Queue]
         # Map from request ID to `threading.Event`.
         self.request_events: Dict[int, Event]
@@ -55,8 +55,7 @@ class Rpc:
                 preexec_fn=os.setpgrp,  # noqa: PLW1509
                 **self._kwargs,
             )
-        self.id = 0
-        self.id_lock = Lock()
+        self.id_iterator = itertools.count(start=1)
         self.event_queues = {}
         self.request_events = {}
         self.request_results = {}
@@ -133,7 +132,9 @@ class Rpc:
                 event = self.get_next_event()
                 account_id = event["contextId"]
                 queue = self.get_queue(account_id)
-                queue.put(event["event"])
+                event = event["event"]
+                logging.debug("account_id=%d got an event %s", account_id, event)
+                queue.put(event)
         except Exception:
             # Log an exception if the event loop dies.
             logging.exception("Exception in the event loop")
@@ -145,11 +146,7 @@ class Rpc:
 
     def __getattr__(self, attr: str):
         def method(*args) -> Any:
-            self.id_lock.acquire()
-            self.id += 1
-            request_id = self.id
-            self.id_lock.release()
-
+            request_id = next(self.id_iterator)
             request = {
                 "jsonrpc": "2.0",
                 "method": attr,

deltachat-rpc-client/tests/test_securejoin.py

@@ -0,0 +1,252 @@
+import logging
+
+import pytest
+from deltachat_rpc_client import Chat, SpecialContactId
+
+
+def test_qr_setup_contact(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+
+    while True:
+        event = alice.wait_for_event()
+        if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+    assert alice_contact_bob_snapshot.is_profile_verified
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+    assert bob_contact_alice_snapshot.is_profile_verified
+
+
+def test_qr_securejoin(acfactory):
+    alice, bob = acfactory.get_online_accounts(2)
+
+    logging.info("Alice creates a verified group")
+    alice_chat = alice.create_group("Verified group", protect=True)
+
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    while True:
+        event = alice.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+    assert alice_contact_bob_snapshot.is_profile_verified
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+    assert bob_contact_alice_snapshot.is_profile_verified
+
+
+@pytest.mark.xfail()
+def test_verified_group_recovery(acfactory, rpc) -> None:
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    while True:
+        event = ac3.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("Received message %s", snapshot.text)
+    assert snapshot.text == "Hi!"
+
+    # ac1 contact cannot be verified by ac2 because ac3 did not gossip ac1 key in the "Hi!" message.
+    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
+    assert not ac1_contact.get_snapshot().is_verified
+
+    ac3_contact_id_ac1 = rpc.lookup_contact_id_by_addr(ac3.id, ac1.get_config("addr"))
+    ac3_chat.remove_contact(ac3_contact_id_ac1)
+    ac3_chat.add_contact(ac3_contact_id_ac1)
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("ac2 got event message: %s", snapshot.text)
+    assert "removed" in snapshot.text
+
+    event = ac2.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    chat_id = event.chat_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("ac2 got event message: %s", snapshot.text)
+    assert "added" in snapshot.text
+
+    assert ac1_contact.get_snapshot().is_verified
+
+    chat = Chat(ac2, chat_id)
+    chat.send_text("Works again!")
+
+    msg_id = ac3.wait_for_incoming_msg_event().msg_id
+    message = ac3.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1.wait_for_incoming_msg_event()  # Hi!
+    ac1.wait_for_incoming_msg_event()  # Member removed
+    ac1.wait_for_incoming_msg_event()  # Member added
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_verified_group_member_added_recovery(acfactory) -> None:
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    while True:
+        event = ac1.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    while True:
+        event = ac3.wait_for_event()
+        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
+            break
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("Received message %s", snapshot.text)
+    assert snapshot.text == "Hi!"
+
+    ac1.wait_for_incoming_msg_event()  # Hi!
+
+    ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
+    ac3_chat.remove_contact(ac3_contact_ac2)
+    ac3_chat.add_contact(ac3_contact_ac2)
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert "removed" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "removed" in snapshot.text
+
+    event = ac2.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    chat_id = event.chat_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("ac2 got event message: %s", snapshot.text)
+    assert "added" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "added" in snapshot.text
+
+    logging.info("ac2 address is %s", ac2.get_config("addr"))
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
+    # assert ac1_contact_ac2_snapshot.is_verified
+    assert ac1_contact_ac2_snapshot.verifier_id == ac1.get_contact_by_addr(ac3.get_config("addr")).id
+
+    chat = Chat(ac2, chat_id)
+    chat.send_text("Works again!")
+
+    msg_id = ac3.wait_for_incoming_msg_event().msg_id
+    message = ac3.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id

deltachat-rpc-client/tests/test_something.py

@@ -1,11 +1,10 @@
 import concurrent.futures
 import json
-import logging
 import subprocess
 from unittest.mock import MagicMock
 
 import pytest
-from deltachat_rpc_client import Chat, EventType, SpecialContactId, events
+from deltachat_rpc_client import EventType, events
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
@@ -378,202 +377,3 @@ def test_provider_info(rpc) -> None:
     rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
-
-
-def test_qr_setup_contact(acfactory) -> None:
-    alice, bob = acfactory.get_online_accounts(2)
-
-    qr_code, _svg = alice.get_qr_code()
-    bob.secure_join(qr_code)
-
-    while True:
-        event = alice.wait_for_event()
-        if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            return
-
-
-@pytest.mark.xfail()
-def test_verified_group_recovery(acfactory, rpc) -> None:
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    logging.info("ac1 creates verified group")
-    chat = ac1.create_group("Verified group", protect=True)
-    assert chat.get_basic_snapshot().is_protected
-
-    logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
-    ac2.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
-
-    # ac1 has ac2 directly verified.
-    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
-
-    logging.info("ac3 joins verified group")
-    ac3_chat = ac3.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
-
-    logging.info("ac2 logs in on a new device")
-    ac2 = acfactory.resetup_account(ac2)
-
-    logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
-    ac2.secure_join(qr_code)
-
-    while True:
-        event = ac3.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
-
-    logging.info("ac3 sends a message to the group")
-    assert len(ac3_chat.get_contacts()) == 3
-    ac3_chat.send_text("Hi!")
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("Received message %s", snapshot.text)
-    assert snapshot.text == "Hi!"
-
-    # ac1 contact cannot be verified by ac2 because ac3 did not gossip ac1 key in the "Hi!" message.
-    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
-    assert not ac1_contact.get_snapshot().is_verified
-
-    ac3_contact_id_ac1 = rpc.lookup_contact_id_by_addr(ac3.id, ac1.get_config("addr"))
-    ac3_chat.remove_contact(ac3_contact_id_ac1)
-    ac3_chat.add_contact(ac3_contact_id_ac1)
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("ac2 got event message: %s", snapshot.text)
-    assert "removed" in snapshot.text
-
-    event = ac2.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    chat_id = event.chat_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("ac2 got event message: %s", snapshot.text)
-    assert "added" in snapshot.text
-
-    assert ac1_contact.get_snapshot().is_verified
-
-    chat = Chat(ac2, chat_id)
-    chat.send_text("Works again!")
-
-    msg_id = ac3.wait_for_incoming_msg_event().msg_id
-    message = ac3.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    ac1.wait_for_incoming_msg_event()  # Hi!
-    ac1.wait_for_incoming_msg_event()  # Member removed
-    ac1.wait_for_incoming_msg_event()  # Member added
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    # ac2 is now verified by ac3 for ac1
-    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
-
-
-def test_verified_group_member_added_recovery(acfactory) -> None:
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    logging.info("ac1 creates verified group")
-    chat = ac1.create_group("Verified group", protect=True)
-    assert chat.get_basic_snapshot().is_protected
-
-    logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
-    ac2.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
-
-    # ac1 has ac2 directly verified.
-    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
-
-    logging.info("ac3 joins verified group")
-    ac3_chat = ac3.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
-
-    logging.info("ac2 logs in on a new device")
-    ac2 = acfactory.resetup_account(ac2)
-
-    logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
-    ac2.secure_join(qr_code)
-
-    while True:
-        event = ac3.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
-
-    logging.info("ac3 sends a message to the group")
-    assert len(ac3_chat.get_contacts()) == 3
-    ac3_chat.send_text("Hi!")
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("Received message %s", snapshot.text)
-    assert snapshot.text == "Hi!"
-
-    ac1.wait_for_incoming_msg_event()  # Hi!
-
-    ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
-    ac3_chat.remove_contact(ac3_contact_ac2)
-    ac3_chat.add_contact(ac3_contact_ac2)
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert "removed" in snapshot.text
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert "removed" in snapshot.text
-
-    event = ac2.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    chat_id = event.chat_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("ac2 got event message: %s", snapshot.text)
-    assert "added" in snapshot.text
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert "added" in snapshot.text
-
-    logging.info("ac2 address is %s", ac2.get_config("addr"))
-    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
-    ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
-    # assert ac1_contact_ac2_snapshot.is_verified
-    assert ac1_contact_ac2_snapshot.verifier_id == ac1.get_contact_by_addr(ac3.get_config("addr")).id
-
-    chat = Chat(ac2, chat_id)
-    chat.send_text("Works again!")
-
-    msg_id = ac3.wait_for_incoming_msg_event().msg_id
-    message = ac3.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    # ac2 is now verified by ac3 for ac1
-    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id

deltachat-rpc-client/tox.ini

@@ -30,4 +30,4 @@ commands =
 [pytest]
 timeout = 300
 log_cli = true
-log_level = info
+log_level = debug

draft/keymanagement.md

@@ -0,0 +1,129 @@
+# New acpeerstates table
+
+A proposal to replace `acpeerstates` table,
+which has columns
+- `id`
+- `addr`
+- `prefer_encrypted`
+- `last_seen`
+- `last_seen_autocrypt`
+- `public_key`
+- `public_key_fingrprint`
+- `gossip_timestamp`
+- `gossip_key`
+- `gossip_key_fingerprint`
+- `verified_key`
+- `verified_key_fingerprint`
+- `verifier`
+- `secondary_verified_key`
+- `secondary_verified_key_fingerprint`
+- `secondary_verifier`
+with a new `public_keys` table with columns
+- `id`
+- `addr` - address of the contact which is supposed to have the private key
+- `public_key` - a blob with the OpenPGP public key
+- `public_key_fingerprint` - public key fingerprint
+- `introduced_by` - address of the contact if received directly in `Autocrypt` header, address of `Autocrypt-Gossip` sender otherwise
+- `is_verified` - boolean flag indicating if the key was received in a verified group or not
+- `timestamp` - timestamp of the most recent row update 
+
+The table has consraints `UNIQUE(addr, introduced_by)`.
+Note that there may be multiple rows with the same fingerprint,
+e.g. if multiple peers gossiped the same key for a contact they all are introducers of the key.
+
+`prefer_encrypted` is moved to the `contacts` table.
+
+
+## Using the table to select the keys
+
+### Sending a message to 1:1 chat
+
+When sending a message to the contact in a 1:1 chat,
+encrypt to the key where `introduced_by` equals `addr`:
+```
+SELECT *
+FROM public_keys
+WHERE addr=? AND introduced_by=addr
+```
+This row is guaranteed to be unique if it exists.
+
+If direct Autocrypt key does not exist,
+use the most recent one according to the timestamp key gossiped for this contact:
+```
+SELECT *
+FROM public_keys
+WHERE addr=?
+ORDER BY timestamp DESC LIMIT 1
+```
+
+If the table contains a row with selected key and `is_verified` flag set,
+display a green checkmark and "Introduced by <introduced_by>" in the 1:1 chat/contact profile.
+Note that row with verification may be older than the most recent gossip.
+
+### Sending a message to unprotected group chat
+
+When sending a message to the contact in a non-verified group chat,
+encrypt to the same key as used in 1:1 chat
+and (NEW!) one or more most recent gossiped keys introduced by chat members:
+```
+SELECT public_key
+FROM public_keys
+WHERE addr=? AND introduced_by IN (<chat-member-list>)
+ORDER BY timestamp DESC
+```
+It does not matter if the gossiped key has `is_verified` flag.
+
+### Sending a message to protected group chat
+
+When sending a message in a protected group chat,
+construct a list of candidate keys for each contact
+the same way as in an unprotected chat,
+but then filter out the keys which do not have `is_verified` flag.
+
+
+## Updating the table
+
+When executing "setup contact" protocol
+with the contact,
+set the row with `addr` and `introduced_by` being equal
+to the contact address and `is_verified` is true.
+In this case the contact becomes directly verified.
+
+When receiving a message,
+first process the `Autocrypt` key,
+then check verification properties to detect if gossip is signed with a verified key,
+then process `Autocrypt-Gossip` key.
+
+### 1. Processing the Autocrypt header
+
+Take the key from the Autocrypt header
+and insert or update the row
+identified by `addr` and `introduced_by` being equal to the `From:` address.
+Notify about setup change and reset `is_verified` flag if the key fingerprint has changed.
+Always update the timestamp.
+
+Update `prefer_encrypted` in the contacts table as needed
+based on the `Autocrypt` header parameters,
+whether the message is signed etc.
+
+### 2. Check verification
+
+Determine whether the green checkmark should now be displayed
+on the 1:1 chat and update it accordingly,
+insert system messages,
+get it into broken state or out of it.
+
+Now we know if the message is signed with a verified key.
+
+### 3. Process `Autocrypt-Gossip`
+
+Insert or update gossip keys
+```
+INSERT INTO public_keys (addr, public_key, public_key_fingerprint, introduced_by, is_verified, timestamp)
+VALUES                  (?,    ?,          ?,                      <introducer>,  ?,           <now>)
+ON CONFLICT
+DO UPDATE SET public_key=excluded.public_key, public_key_fingerprint=excluded.public_key_fingerprint
+```
+
+`is_verified` should be set if the message is signed with a verified key,
+independently of whether the key was gossiped in a protected chat or not.

python/tests/test_1_online.py

@@ -367,7 +367,7 @@ def test_webxdc_download_on_demand(acfactory, data, lp):
 
     lp.sec("ac2 sets download limit")
     ac2.set_config("download_limit", "100")
-    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(50000))}, "some test data")
+    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data")
     ac2_update = ac2._evtracker.wait_next_incoming_message()
     assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE
     assert not msg2.get_status_updates()
@@ -1445,7 +1445,7 @@ def test_reaction_to_partially_fetched_msg(acfactory, lp, tmp_path):
       message, then processes a partially downloaded message.
     - As a result, Bob does not see a reaction
     """
-    download_limit = 32768
+    download_limit = 300000
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_addr = ac1.get_config("addr")
     chat = ac1.create_chat(ac2)

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=3c8f7e846c915a183dc44536fb5480d1f25d7c42
+REV=18f714cf73d0bdfb8b013fa344494ab80c92b477
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

src/chat.rs

@@ -208,9 +208,10 @@ impl ChatId {
         self == DC_CHAT_ID_ALLDONE_HINT
     }
 
-    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id` if it exists.
+    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id`
+    /// if it exists and is not blocked.
     ///
-    /// If it does not exist, `None` is returned.
+    /// If the chat does not exist or is blocked, `None` is returned.
     pub async fn lookup_by_contact(
         context: &Context,
         contact_id: ContactId,
@@ -227,7 +228,7 @@ impl ChatId {
     /// This is an internal API, if **a user action** needs to get a chat
     /// [`ChatId::create_for_contact`] should be used as this also scales up the
     /// [`Contact`]'s origin.
-    pub async fn get_for_contact(context: &Context, contact_id: ContactId) -> Result<Self> {
+    pub(crate) async fn get_for_contact(context: &Context, contact_id: ContactId) -> Result<Self> {
         ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Not)
             .await
             .map(|chat| chat.id)
@@ -1251,6 +1252,16 @@ impl ChatId {
 
         Ok(())
     }
+
+    /// Returns true if the chat is protected.
+    pub async fn is_protected(self, context: &Context) -> Result<ProtectionStatus> {
+        let protection_status = context
+            .sql
+            .query_get_value("SELECT protected FROM chats WHERE id=?", (self,))
+            .await?
+            .unwrap_or_default();
+        Ok(protection_status)
+    }
 }
 
 impl std::fmt::Display for ChatId {
@@ -4462,13 +4473,11 @@ mod tests {
         remove_contact_from_chat(&bob, bob_chat_id, ContactId::SELF).await?;
 
         // Bob receives a msg about Alice adding Claire to the group.
-        let bob_received_add_msg = bob.recv_msg(&alice_sent_add_msg).await;
+        bob.recv_msg(&alice_sent_add_msg).await;
 
         // Test that add message is rewritten.
-        assert_eq!(
-            bob_received_add_msg.get_text(),
-            "Member claire@example.net added by alice@example.org."
-        );
+        bob.golden_test_chat(bob_chat_id, "chat_test_simultaneous_member_remove")
+            .await;
 
         // Bob receives a msg about Alice removing him from the group.
         let bob_received_remove_msg = bob.recv_msg(&alice_sent_remove_msg).await;

src/contact.rs

@@ -19,7 +19,7 @@ use tokio::task;
 use tokio::time::{timeout, Duration};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::ChatId;
+use crate::chat::{ChatId, ProtectionStatus};
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
@@ -1263,7 +1263,7 @@ impl Contact {
     ///
     /// Do not use this function when displaying the contact profile view.
     /// Display green checkmark in the title of the contact profile
-    /// if 1:1 chat with the contact exists and is protected.
+    /// if [Self::is_profile_verified] returns true.
     /// Use [Self::get_verifier_id] to display the verifier contact
     /// in the info section of the contact profile.
     pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
@@ -1317,6 +1317,22 @@ impl Contact {
         }
     }
 
+    /// Returns if the contact profile title should display a green checkmark.
+    ///
+    /// This generally should be consistent with the 1:1 chat with the contact
+    /// so 1:1 chat with the contact and the contact profile
+    /// either both display the green checkmark or both don't display a green checkmark.
+    pub async fn is_profile_verified(&self, context: &Context) -> Result<bool> {
+        let contact_id = self.id;
+
+        if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
+            Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
+        } else {
+            // 1:1 chat does not exist.
+            Ok(self.is_verified(context).await? == VerifiedStatus::BidirectVerified)
+        }
+    }
+
     /// Returns the number of real (i.e. non-special) contacts in the database.
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {

src/download.rs

@@ -18,12 +18,11 @@ use crate::{stock_str, EventType};
 
 /// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
 ///
-/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
-/// need to be downloaded completely to handle them correctly,
-/// eg. to assign them to the correct chat.
-/// As these messages are typically small,
-/// they're caught by `MIN_DOWNLOAD_LIMIT`.
-pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN)
+/// should always be downloaded completely to handle them correctly,
+/// also in larger groups and if group and contact avatar are attached.
+/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`.
+pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840;
 
 /// If a message is downloaded only partially
 /// and `delete_server_after` is set to small timeouts (eg. "at once"),

src/e2ee.rs

@@ -67,13 +67,8 @@ impl EncryptHelper {
                         "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
                     );
                     match peerstate.prefer_encrypt {
-                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::NoPreference | EncryptPreference::Reset => {}
                         EncryptPreference::Mutual => prefer_encrypt_count += 1,
-                        EncryptPreference::Reset => {
-                            if !e2ee_guaranteed {
-                                return Ok(false);
-                            }
-                        }
                     };
                 }
                 None => {

src/events/payload.rs

@@ -250,6 +250,7 @@ pub enum EventType {
         /// Progress as:
         /// 400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
         /// (Bob has verified alice and waits until Alice does the same for him)
+        /// 1000=vg-member-added/vc-contact-confirm received
         progress: usize,
     },
 

src/provider/data.rs

@@ -222,6 +222,99 @@ static P_BUZON_UY: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// c1.testrun.org.md: c1.testrun.org
+static P_C1_TESTRUN_ORG: Provider = Provider {
+    id: "c1.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/c1-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "c1.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "c1.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
+// c2.testrun.org.md: c2.testrun.org
+static P_C2_TESTRUN_ORG: Provider = Provider {
+    id: "c2.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/c2-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "c2.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "c2.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
+// c3.testrun.org.md: c3.testrun.org
+static P_C3_TESTRUN_ORG: Provider = Provider {
+    id: "c3.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/c3-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "c3.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "c3.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
 // chello.at.md: chello.at
 static P_CHELLO_AT: Provider = Provider {
     id: "chello.at",
@@ -867,6 +960,37 @@ static P_NAVER: Provider = Provider {
     oauth2_authorizer: None,
 };
 
+// nine.testrun.org.md: nine.testrun.org
+static P_NINE_TESTRUN_ORG: Provider = Provider {
+    id: "nine.testrun.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/nine-testrun-org",
+    server: &[
+        Server {
+            protocol: Imap,
+            socket: Ssl,
+            hostname: "nine.testrun.org",
+            port: 993,
+            username_pattern: Email,
+        },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "nine.testrun.org",
+            port: 465,
+            username_pattern: Email,
+        },
+    ],
+    opt: ProviderOptions::new(),
+    config_defaults: Some(&[ConfigDefault {
+        key: Config::MvboxMove,
+        value: "0",
+    }]),
+    oauth2_authorizer: None,
+};
+
 // nubo.coop.md: nubo.coop
 static P_NUBO_COOP: Provider = Provider {
     id: "nubo.coop",
@@ -1495,6 +1619,9 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("delta.blindzeln.org", &P_BLINDZELN_ORG),
         ("bluewin.ch", &P_BLUEWIN_CH),
         ("buzon.uy", &P_BUZON_UY),
+        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
+        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
+        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
         ("chello.at", &P_CHELLO_AT),
         ("xfinity.com", &P_COMCAST),
         ("comcast.net", &P_COMCAST),
@@ -1708,6 +1835,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("mailo.com", &P_MAILO_COM),
         ("nauta.cu", &P_NAUTA_CU),
         ("naver.com", &P_NAVER),
+        ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
         ("nubo.coop", &P_NUBO_COOP),
         ("hotmail.com", &P_OUTLOOK_COM),
         ("outlook.com", &P_OUTLOOK_COM),
@@ -1860,6 +1988,9 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("blindzeln.org", &P_BLINDZELN_ORG),
         ("bluewin.ch", &P_BLUEWIN_CH),
         ("buzon.uy", &P_BUZON_UY),
+        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
+        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
+        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
         ("chello.at", &P_CHELLO_AT),
         ("comcast", &P_COMCAST),
         ("dismail.de", &P_DISMAIL_DE),
@@ -1888,6 +2019,7 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("mailo.com", &P_MAILO_COM),
         ("nauta.cu", &P_NAUTA_CU),
         ("naver", &P_NAVER),
+        ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
         ("nubo.coop", &P_NUBO_COOP),
         ("outlook.com", &P_OUTLOOK_COM),
         ("ouvaton.coop", &P_OUVATON_COOP),
@@ -1918,4 +2050,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static _PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 20).unwrap());
+    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 11, 5).unwrap());

src/receive_imf.rs

@@ -457,6 +457,7 @@ async fn add_parts(
     let mut chat_id_blocked = Blocked::Not;
 
     let mut better_msg = None;
+    let mut group_changes_msgs = Vec::new();
     if mime_parser.is_system_message == SystemMessage::LocationStreamingEnabled {
         better_msg = Some(stock_str::msg_location_enabled_by(context, from_id).await);
     }
@@ -649,7 +650,7 @@ async fn add_parts(
                 }
             }
 
-            better_msg = better_msg.or(apply_group_changes(
+            group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
                 sent_timestamp,
@@ -658,7 +659,7 @@ async fn add_parts(
                 to_ids,
                 is_partial_download.is_some(),
             )
-            .await?);
+            .await?;
         }
 
         if chat_id.is_none() {
@@ -892,7 +893,7 @@ async fn add_parts(
         }
 
         if let Some(chat_id) = chat_id {
-            better_msg = better_msg.or(apply_group_changes(
+            group_changes_msgs = apply_group_changes(
                 context,
                 mime_parser,
                 sent_timestamp,
@@ -901,7 +902,7 @@ async fn add_parts(
                 to_ids,
                 is_partial_download.is_some(),
             )
-            .await?);
+            .await?;
         }
 
         if chat_id.is_none() && self_sent {
@@ -1115,7 +1116,9 @@ async fn add_parts(
 
     let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len());
 
-    for part in &mut mime_parser.parts {
+    group_changes_msgs = group_changes_msgs.into_iter().rev().collect();
+    let mut parts = mime_parser.parts.iter_mut().peekable();
+    while let Some(part) = parts.peek() {
         if part.is_reaction {
             let reaction_str = simplify::remove_footers(part.msg.as_str());
             set_msg_reaction(
@@ -1148,7 +1151,10 @@ async fn add_parts(
         }
 
         let mut txt_raw = "".to_string();
-        let (msg, typ): (&str, Viewtype) = if let Some(better_msg) = &better_msg {
+        let group_changes_msg = group_changes_msgs.pop();
+        let (msg, typ): (&str, Viewtype) = if let Some(msg) = &group_changes_msg {
+            (msg, Viewtype::Text)
+        } else if let Some(better_msg) = &better_msg {
             (better_msg, Viewtype::Text)
         } else {
             (&part.msg, part.typ)
@@ -1276,6 +1282,10 @@ RETURNING id
 
         debug_assert!(!row_id.is_special());
         created_db_entries.push(row_id);
+
+        if group_changes_msg.is_none() || (group_changes_msgs.is_empty() && better_msg.is_none()) {
+            parts.next();
+        }
     }
 
     // check all parts whether they contain a new logging webxdc
@@ -1704,15 +1714,15 @@ async fn apply_group_changes(
     from_id: ContactId,
     to_ids: &[ContactId],
     is_partial_download: bool,
-) -> Result<Option<String>> {
+) -> Result<Vec<String>> {
     let mut chat = Chat::load_from_db(context, chat_id).await?;
     if chat.typ != Chattype::Group {
-        return Ok(None);
+        return Ok(Vec::new());
     }
 
     let mut send_event_chat_modified = false;
     let (mut removed_id, mut added_id) = (None, None);
-    let mut better_msg = None;
+    let mut group_changes_msgs = Vec::new();
 
     // True if a Delta Chat client has explicitly added our current primary address.
     let self_added =
@@ -1777,11 +1787,11 @@ async fn apply_group_changes(
     if let Some(removed_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberRemoved) {
         removed_id = Contact::lookup_id_by_addr(context, removed_addr, Origin::Unknown).await?;
 
-        better_msg = if removed_id == Some(from_id) {
-            Some(stock_str::msg_group_left_local(context, from_id).await)
+        group_changes_msgs.push(if removed_id == Some(from_id) {
+            stock_str::msg_group_left_local(context, from_id).await
         } else {
-            Some(stock_str::msg_del_member_local(context, removed_addr, from_id).await)
-        };
+            stock_str::msg_del_member_local(context, removed_addr, from_id).await
+        });
 
         if removed_id.is_some() {
             if !allow_member_list_changes {
@@ -1794,7 +1804,8 @@ async fn apply_group_changes(
             warn!(context, "Removed {removed_addr:?} has no contact id.")
         }
     } else if let Some(added_addr) = mime_parser.get_header(HeaderDef::ChatGroupMemberAdded) {
-        better_msg = Some(stock_str::msg_add_member_local(context, added_addr, from_id).await);
+        group_changes_msgs
+            .push(stock_str::msg_add_member_local(context, added_addr, from_id).await);
 
         if allow_member_list_changes {
             if !recreate_member_list {
@@ -1835,21 +1846,20 @@ async fn apply_group_changes(
                 send_event_chat_modified = true;
             }
 
-            better_msg = Some(stock_str::msg_grp_name(context, old_name, grpname, from_id).await);
+            group_changes_msgs
+                .push(stock_str::msg_grp_name(context, old_name, grpname, from_id).await);
         }
     } else if let Some(value) = mime_parser.get_header(HeaderDef::ChatContent) {
         if value == "group-avatar-changed" {
             if let Some(avatar_action) = &mime_parser.group_avatar {
                 // this is just an explicit message containing the group-avatar,
                 // apart from that, the group-avatar is send along with various other messages
-                better_msg = match avatar_action {
-                    AvatarAction::Delete => {
-                        Some(stock_str::msg_grp_img_deleted(context, from_id).await)
-                    }
+                group_changes_msgs.push(match avatar_action {
+                    AvatarAction::Delete => stock_str::msg_grp_img_deleted(context, from_id).await,
                     AvatarAction::Change(_) => {
-                        Some(stock_str::msg_grp_img_changed(context, from_id).await)
+                        stock_str::msg_grp_img_changed(context, from_id).await
                     }
-                };
+                });
             }
         }
     }
@@ -1883,6 +1893,18 @@ async fn apply_group_changes(
             if !diff.is_empty() {
                 warn!(context, "Implicit addition of {diff:?} to chat {chat_id}.");
             }
+            group_changes_msgs.reserve(diff.len());
+            for contact_id in diff {
+                let contact = Contact::get_by_id(context, contact_id).await?;
+                group_changes_msgs.push(
+                    stock_str::msg_add_member_local(
+                        context,
+                        contact.get_addr(),
+                        ContactId::UNDEFINED,
+                    )
+                    .await,
+                );
+            }
         }
         if let Some(removed_id) = removed_id {
             new_members.remove(&removed_id);
@@ -1949,7 +1971,7 @@ async fn apply_group_changes(
     if send_event_chat_modified {
         context.emit_event(EventType::ChatModified(chat_id));
     }
-    Ok(better_msg)
+    Ok(group_changes_msgs)
 }
 
 static LIST_ID_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^(.+)<(.+)>$").unwrap());

src/securejoin.rs

@@ -8,7 +8,7 @@ use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::{Blocked, Chattype};
+use crate::constants::Blocked;
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::e2ee::ensure_secret_key_exists;
@@ -36,17 +36,15 @@ use crate::token::Namespace;
 /// Set of characters to percent-encode in email addresses and names.
 pub const NON_ALPHANUMERIC_WITHOUT_DOT: &AsciiSet = &NON_ALPHANUMERIC.remove(b'.');
 
-macro_rules! inviter_progress {
-    ($context:tt, $contact_id:expr, $progress:expr) => {
-        assert!(
-            $progress >= 0 && $progress <= 1000,
-            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
-        );
-        $context.emit_event($crate::events::EventType::SecurejoinInviterProgress {
-            contact_id: $contact_id,
-            progress: $progress,
-        });
-    };
+fn inviter_progress(context: &Context, contact_id: ContactId, progress: usize) {
+    debug_assert!(
+        progress <= 1000,
+        "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
+    );
+    context.emit_event(EventType::SecurejoinInviterProgress {
+        contact_id,
+        progress,
+    });
 }
 
 /// Generates a Secure Join QR code.
@@ -318,7 +316,7 @@ pub(crate) async fn handle_securejoin_handshake(
             }
             info!(context, "Secure-join requested.",);
 
-            inviter_progress!(context, contact_id, 300);
+            inviter_progress(context, contact_id, 300);
 
             // for setup-contact, make Alice's one-to-one chat with Bob visible
             // (secure-join-information are shown in the group chat)
@@ -430,7 +428,7 @@ pub(crate) async fn handle_securejoin_handshake(
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await?;
             info!(context, "Auth verified.",);
             context.emit_event(EventType::ContactsChanged(Some(contact_id)));
-            inviter_progress!(context, contact_id, 600);
+            inviter_progress(context, contact_id, 600);
             if join_vg {
                 // the vg-member-added message is special:
                 // this is a normal Chat-Group-Member-Added message
@@ -450,8 +448,8 @@ pub(crate) async fn handle_securejoin_handshake(
                     }
                     None => bail!("Chat {} not found", &field_grpid),
                 }
-                inviter_progress!(context, contact_id, 800);
-                inviter_progress!(context, contact_id, 1000);
+                inviter_progress(context, contact_id, 800);
+                inviter_progress(context, contact_id, 1000);
             } else {
                 // Alice -> Bob
                 secure_connection_established(
@@ -469,7 +467,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await
                 .context("failed sending vc-contact-confirm message")?;
 
-                inviter_progress!(context, contact_id, 1000);
+                inviter_progress(context, contact_id, 1000);
             }
             Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed)
         }
@@ -651,10 +649,10 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 return Ok(HandshakeMessage::Ignore);
             }
             if step.as_str() == "vg-member-added" {
-                inviter_progress!(context, contact_id, 800);
+                inviter_progress(context, contact_id, 800);
             }
             if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" {
-                inviter_progress!(context, contact_id, 1000);
+                inviter_progress(context, contact_id, 1000);
             }
             if step.as_str() == "vg-request-with-auth" || step.as_str() == "vc-request-with-auth" {
                 // This actually reflects what happens on the first device (which does the secure
@@ -684,17 +682,17 @@ async fn secure_connection_established(
         .get_config_bool(Config::VerifiedOneOnOneChats)
         .await?
     {
-        let chat = Chat::load_from_db(context, chat_id).await?;
-        if chat.typ == Chattype::Single {
-            chat_id
-                .set_protection(
-                    context,
-                    ProtectionStatus::Protected,
-                    time(),
-                    Some(contact_id),
-                )
-                .await?;
-        }
+        let private_chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Yes)
+            .await?
+            .id;
+        private_chat_id
+            .set_protection(
+                context,
+                ProtectionStatus::Protected,
+                time(),
+                Some(contact_id),
+            )
+            .await?;
     }
     context.emit_event(EventType::ChatModified(chat_id));
     Ok(())
@@ -767,6 +765,7 @@ mod tests {
     use crate::chat;
     use crate::chat::ProtectionStatus;
     use crate::chatlist::Chatlist;
+    use crate::constants::Chattype;
     use crate::contact::ContactAddress;
     use crate::contact::VerifiedStatus;
     use crate::peerstate::Peerstate;

src/securejoin/bob.rs

@@ -132,6 +132,7 @@ pub(super) async fn handle_contact_confirm(
             // verify both contacts (this could be a bug/security issue, see
             // e.g. https://github.com/deltachat/deltachat-core-rust/issues/1177).
             bobstate.notify_peer_verified(context).await?;
+            bobstate.emit_progress(context, JoinerProgress::Succeeded);
             Ok(retval)
         }
         Some(_) => {
@@ -228,9 +229,8 @@ impl BobState {
         if context
             .get_config_bool(Config::VerifiedOneOnOneChats)
             .await?
-            && chat_id == self.alice_chat()
         {
-            chat_id
+            self.alice_chat()
                 .set_protection(
                     context,
                     ProtectionStatus::Protected,
@@ -256,8 +256,8 @@ enum JoinerProgress {
     ///
     /// Typically shows as "alice@addr verified, introducing myself."
     RequestWithAuthSent,
-    // /// Completed securejoin.
-    // Succeeded,
+    /// Completed securejoin.
+    Succeeded,
 }
 
 impl From<JoinerProgress> for usize {
@@ -265,7 +265,7 @@ impl From<JoinerProgress> for usize {
         match progress {
             JoinerProgress::Error => 0,
             JoinerProgress::RequestWithAuthSent => 400,
-            // JoinerProgress::Succeeded => 1000,
+            JoinerProgress::Succeeded => 1000,
         }
     }
 }

src/stock_str.rs

@@ -416,6 +416,9 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Others will only see this group after you sent a first message."))]
     NewGroupSendFirstMessage = 172,
+
+    #[strum(props(fallback = "Member %1$s added."))]
+    MsgAddMember = 173,
 }
 
 impl StockMessage {
@@ -603,6 +606,7 @@ pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: ContactId
 }
 
 /// Stock string: `I added member %1$s.`.
+/// This one is for sending in group chats.
 ///
 /// The `added_member_addr` parameter should be an email address and is looked up in the
 /// contacts to combine with the authorized display name.
@@ -637,7 +641,11 @@ pub(crate) async fn msg_add_member_local(
             .unwrap_or_else(|_| addr.to_string()),
         _ => addr.to_string(),
     };
-    if by_contact == ContactId::SELF {
+    if by_contact == ContactId::UNDEFINED {
+        translated(context, StockMessage::MsgAddMember)
+            .await
+            .replace1(whom)
+    } else if by_contact == ContactId::SELF {
         translated(context, StockMessage::MsgYouAddMember)
             .await
             .replace1(whom)

src/test_utils.rs

@@ -514,12 +514,7 @@ impl TestContext {
             .await
             .expect("receive_imf() seems not to have added a new message to the db");
 
-        assert_eq!(
-            received.msg_ids.len(),
-            1,
-            "recv_msg() can currently only receive messages with exactly one part"
-        );
-        let msg = Message::load_from_db(self, received.msg_ids[0])
+        let msg = Message::load_from_db(self, *received.msg_ids.last().unwrap())
             .await
             .unwrap();
 

test-data/golden/chat_test_simultaneous_member_remove

@@ -0,0 +1,7 @@
+Group#Chat#10: Group chat [4 member(s)] 
+--------------------------------------------------------------------------------
+Msg#10:  (Contact#Contact#10): Hi! I created a group. [FRESH]
+Msg#11: Me (Contact#Contact#Self): You left the group. [INFO] o
+Msg#12:  (Contact#Contact#10): Member claire@example.net added by alice@example.org. [FRESH][INFO]
+Msg#13:  (Contact#Contact#10): Member Me (bob@example.net) added. [FRESH][INFO]
+--------------------------------------------------------------------------------