feat: pre-messages / next version of download on demand (#7371)

Closes <https://github.com/chatmail/core/issues/7367> Co-authored-by: iequidoo <dgreshilov@gmail.com> Co-authored-by: Hocuri <hocuri@gmx.de>
2026-04-27 18:36:30 +03:00 · 2026-01-08 22:14:32 +00:00
parent 46bbe5f077
commit 2631745a57
43 changed files with 2843 additions and 1393 deletions
--- a/deltachat-rpc-client/tests/test_something.py
+++ b/deltachat-rpc-client/tests/test_something.py
@@ -5,12 +5,11 @@ import logging
 import os
 import socket
 import subprocess
-import time
 from unittest.mock import MagicMock

 import pytest

-from deltachat_rpc_client import Contact, EventType, Message, events
+from deltachat_rpc_client import EventType, events
 from deltachat_rpc_client.const import DownloadState, MessageState
 from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS
 from deltachat_rpc_client.rpc import JsonRpcError
@@ -333,7 +332,7 @@ def test_receive_imf_failure(acfactory) -> None:
    alice_contact_bob = alice.create_contact(bob, "Bob")
    alice_chat_bob = alice_contact_bob.create_chat()

-    bob.set_config("fail_on_receiving_full_msg", "1")
+    bob.set_config("simulate_receive_imf_error", "1")
    alice_chat_bob.send_text("Hello!")
    event = bob.wait_for_event(EventType.MSGS_CHANGED)
    assert event.chat_id == bob.get_device_chat().id
@@ -343,18 +342,17 @@ def test_receive_imf_failure(acfactory) -> None:
    version = bob.get_info()["deltachat_core_version"]
    assert (
        snapshot.text == "❌ Failed to receive a message:"
-        " Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`."
+        " Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`."
        f" Core version {version}."
        " Please report this bug to delta@merlinux.eu or https://support.delta.chat/."
    )

    # The failed message doesn't break the IMAP loop.
-    bob.set_config("fail_on_receiving_full_msg", "0")
+    bob.set_config("simulate_receive_imf_error", "0")
    alice_chat_bob.send_text("Hello again!")
    message = bob.wait_for_incoming_msg()
    snapshot = message.get_snapshot()
    assert snapshot.text == "Hello again!"
-    assert snapshot.download_state == DownloadState.DONE
    assert snapshot.error is None


@@ -687,60 +685,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
    assert snapshot.show_padlock


-def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
-    """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded
-    messages are received out of order".
-
-    If the Inbox contains X small messages followed by Y large messages followed by Z small
-    messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages.
-
-    This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced
-    with online test as follows:
-    - Bob enables download limit and goes offline.
-    - Alice sends a large message to Bob and reacts to this message with a thumbs-up.
-    - Bob goes online
-    - Bob first processes a reaction message and throws it away because there is no corresponding
-      message, then processes a partially downloaded message.
-    - As a result, Bob does not see a reaction
-    """
-    download_limit = 300000
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1_addr = ac1.get_config("addr")
-    chat = ac1.create_chat(ac2)
-    ac2.set_config("download_limit", str(download_limit))
-    ac2.stop_io()
-
-    logging.info("sending small+large messages from ac1 to ac2")
-    msgs = []
-    msgs.append(chat.send_text("hi"))
-    path = tmp_path / "large"
-    path.write_bytes(os.urandom(download_limit + 1))
-    msgs.append(chat.send_file(str(path)))
-    for m in msgs:
-        m.wait_until_delivered()
-
-    logging.info("sending a reaction to the large message from ac1 to ac2")
-    # TODO: Find the reason of an occasional message reordering on the server (so that the reaction
-    # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction
-    # have a later INTERNALDATE.
-    time.sleep(1.1)
-    react_str = "\N{THUMBS UP SIGN}"
-    msgs.append(msgs[-1].send_reaction(react_str))
-    msgs[-1].wait_until_delivered()
-
-    ac2.start_io()
-
-    logging.info("wait for ac2 to receive a reaction")
-    msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
-    assert msg2.get_sender_contact().get_snapshot().address == ac1_addr
-    assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE
-    reactions = msg2.get_reactions()
-    contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
-    assert len(contacts) == 1
-    assert contacts[0].get_snapshot().address == ac1_addr
-    assert list(reactions.reactions_by_contact.values())[0] == [react_str]
-
-
@pytest.mark.parametrize("n_accounts", [3, 2])
 def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
    download_limit = 300000
@@ -767,14 +711,159 @@ def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
    path = tmp_path / "large"
    path.write_bytes(os.urandom(download_limit + 1))

+    n_done = 0
    for i in range(10):
        logging.info("Sending message %s", i)
        alice_group.send_file(str(path))
        snapshot = bob.wait_for_incoming_msg().get_snapshot()
-        assert snapshot.download_state == DownloadState.AVAILABLE
+        if snapshot.download_state == DownloadState.DONE:
+            n_done += 1
+            # Work around lost and reordered pre-messages.
+            assert n_done <= 1
+        else:
+            assert snapshot.download_state == DownloadState.AVAILABLE
        assert snapshot.chat == bob_group


+def test_download_small_msg_first(acfactory, tmp_path):
+    download_limit = 70000
+
+    alice, bob0 = acfactory.get_online_accounts(2)
+    bob1 = bob0.clone()
+    bob1.set_config("download_limit", str(download_limit))
+
+    chat = alice.create_chat(bob0)
+    path = tmp_path / "large_enough"
+    path.write_bytes(os.urandom(download_limit + 1))
+    # Less than 140K, so sent w/o a pre-message.
+    chat.send_file(str(path))
+    chat.send_text("hi")
+    bob0.create_chat(alice)
+    assert bob0.wait_for_incoming_msg().get_snapshot().text == ""
+    assert bob0.wait_for_incoming_msg().get_snapshot().text == "hi"
+
+    bob1.start_io()
+    bob1.create_chat(alice)
+    assert bob1.wait_for_incoming_msg().get_snapshot().text == "hi"
+    assert bob1.wait_for_incoming_msg().get_snapshot().text == ""
+
+
+@pytest.mark.parametrize("delete_chat", [False, True])
+def test_delete_available_msg(acfactory, tmp_path, direct_imap, delete_chat):
+    """
+    Tests `DownloadState.AVAILABLE` message deletion on the receiver side.
+    Also tests pre- and post-message deletion on the sender side.
+    """
+    # Min. UI setting as of v2.35
+    download_limit = 163840
+    alice, bob = acfactory.get_online_accounts(2)
+    bob.set_config("download_limit", str(download_limit))
+    # Avoid immediate deletion from the server
+    alice.set_config("bcc_self", "1")
+    bob.set_config("bcc_self", "1")
+
+    chat_alice = alice.create_chat(bob)
+    path = tmp_path / "large"
+    path.write_bytes(os.urandom(download_limit + 1))
+    msg_alice = chat_alice.send_file(str(path))
+    msg_bob = bob.wait_for_incoming_msg()
+    msg_bob_snapshot = msg_bob.get_snapshot()
+    assert msg_bob_snapshot.download_state == DownloadState.AVAILABLE
+    chat_bob = bob.get_chat_by_id(msg_bob_snapshot.chat_id)
+
+    # Avoid DeleteMessages sync message
+    bob.set_config("bcc_self", "0")
+    if delete_chat:
+        chat_bob.delete()
+    else:
+        bob.delete_messages([msg_bob])
+    alice.wait_for_event(EventType.SMTP_MESSAGE_SENT)
+    alice.wait_for_event(EventType.SMTP_MESSAGE_SENT)
+    alice.set_config("bcc_self", "0")
+    if delete_chat:
+        chat_alice.delete()
+    else:
+        alice.delete_messages([msg_alice])
+    for acc in [bob, alice]:
+        if not delete_chat:
+            acc.wait_for_event(EventType.MSG_DELETED)
+        acc_direct_imap = direct_imap(acc)
+        # Messages may be deleted separately
+        while True:
+            acc.wait_for_event(EventType.IMAP_MESSAGE_DELETED)
+            while True:
+                event = acc.wait_for_event()
+                if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:
+                    break
+            if len(acc_direct_imap.get_all_messages()) == 0:
+                break
+
+
+def test_delete_fully_downloaded_msg(acfactory, tmp_path, direct_imap):
+    alice, bob = acfactory.get_online_accounts(2)
+    # Avoid immediate deletion from the server
+    bob.set_config("bcc_self", "1")
+
+    chat_alice = alice.create_chat(bob)
+    path = tmp_path / "large"
+    # Big enough to be sent with a pre-message
+    path.write_bytes(os.urandom(300000))
+    chat_alice.send_file(str(path))
+
+    msg = bob.wait_for_incoming_msg()
+    msg_snapshot = msg.get_snapshot()
+    assert msg_snapshot.download_state == DownloadState.AVAILABLE
+    msgs_changed_event = bob.wait_for_msgs_changed_event()
+    assert msgs_changed_event.msg_id == msg.id
+    msg_snapshot = msg.get_snapshot()
+    assert msg_snapshot.download_state == DownloadState.DONE
+
+    bob_direct_imap = direct_imap(bob)
+    assert len(bob_direct_imap.get_all_messages()) == 2
+    # Avoid DeleteMessages sync message
+    bob.set_config("bcc_self", "0")
+    bob.delete_messages([msg])
+    bob.wait_for_event(EventType.MSG_DELETED)
+    # Messages may be deleted separately
+    while True:
+        bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED)
+        while True:
+            event = bob.wait_for_event()
+            if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:
+                break
+        if len(bob_direct_imap.get_all_messages()) == 0:
+            break
+
+
+def test_imap_autodelete_fully_downloaded_msg(acfactory, tmp_path, direct_imap):
+    alice, bob = acfactory.get_online_accounts(2)
+
+    chat_alice = alice.create_chat(bob)
+    path = tmp_path / "large"
+    # Big enough to be sent with a pre-message
+    path.write_bytes(os.urandom(300000))
+    chat_alice.send_file(str(path))
+
+    msg = bob.wait_for_incoming_msg()
+    msg_snapshot = msg.get_snapshot()
+    assert msg_snapshot.download_state == DownloadState.AVAILABLE
+    msgs_changed_event = bob.wait_for_msgs_changed_event()
+    assert msgs_changed_event.msg_id == msg.id
+    msg_snapshot = msg.get_snapshot()
+    assert msg_snapshot.download_state == DownloadState.DONE
+
+    bob_direct_imap = direct_imap(bob)
+    # Messages may be deleted separately
+    while True:
+        if len(bob_direct_imap.get_all_messages()) == 0:
+            break
+        bob.wait_for_event(EventType.IMAP_MESSAGE_DELETED)
+        while True:
+            event = bob.wait_for_event()
+            if event.kind == EventType.INFO and "Close/expunge succeeded." in event.msg:
+                break
+
+
 def test_markseen_contact_request(acfactory):
    """
    Test that seen status is synchronized for contact request messages
@@ -1152,3 +1241,23 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log):

    assert chat.num_contacts() == 2
    assert msg.get_snapshot().chat.num_contacts() == 2
+
+
+def test_large_message(acfactory) -> None:
+    """
+    Test sending large message without download limit set,
+    so it is sent with pre-message but downloaded without user interaction.
+    """
+    alice, bob = acfactory.get_online_accounts(2)
+
+    alice_chat_bob = alice.create_chat(bob)
+    alice_chat_bob.send_message(
+        "Hello World, this message is bigger than 5 bytes",
+        file="../test-data/image/screenshot.jpg",
+    )
+
+    msg = bob.wait_for_incoming_msg()
+    msgs_changed_event = bob.wait_for_msgs_changed_event()
+    assert msg.id == msgs_changed_event.msg_id
+    snapshot = msg.get_snapshot()
+    assert snapshot.text == "Hello World, this message is bigger than 5 bytes"