fix!: Download outgoing messages only from Sentbox and if SentboxWatch is set (#7169)

Download outgoing messages only from Inbox and Mvbox (and Sent -- until `Config::SentboxWatch` is
dropped) to ensure they are sorted correctly with incoming messages. This way we actually do
server-side sorting. Otherwise it's possible that outgoing replies from other devices appear before
incoming messages. Users should CC/BCC/To themselves to see sent messages in Delta Chat.

python/tests/test_1_online.py

@@ -846,6 +846,7 @@ def test_dont_show_emails(acfactory, lp):
     ac1.direct_imap.create_folder("Sent")
     ac1.direct_imap.create_folder("Spam")
     ac1.direct_imap.create_folder("Junk")
+    ac1.set_config("sentbox_watch", "1")
 
     acfactory.bring_accounts_online()
     ac1.stop_io()
@@ -955,7 +956,10 @@ def test_dont_show_emails(acfactory, lp):
 
     msg = ac1._evtracker.wait_next_messages_changed()
 
-    # Wait until each folder was scanned, this is necessary for this test to test what it should test:
+    # Wait until each folder was scanned, this is necessary for this test to test what it should
+    # test. Need to wait two times because of "sentbox_watch" set (the same event is emitted for
+    # Sentbox currently).
+    ac1._evtracker.wait_idle_inbox_ready()
     ac1._evtracker.wait_idle_inbox_ready()
 
     assert msg.text == "subj – message in Sent"

src/events/payload.rs

@@ -35,7 +35,7 @@ pub enum EventType {
     /// Emitted when an IMAP message has been moved
     ImapMessageMoved(String),
 
-    /// Emitted before going into IDLE on the Inbox folder.
+    /// Emitted before going into IDLE on any folder.
     ImapInboxIdle,
 
     /// Emitted when an new file in the $BLOBDIR was created

src/imap.rs

@@ -665,18 +665,15 @@ impl Imap {
             // message, move it to the movebox and then download the second message before
             // downloading the first one, if downloading from inbox before moving is allowed.
             if folder == target
-                // Never download messages directly from the spam folder.
-                // If the sender is known, the message will be moved to the Inbox or Mvbox
-                // and then we download the message from there.
-                // Also see `spam_target_folder_cfg()`.
-                && folder_meaning != FolderMeaning::Spam
                 && prefetch_should_download(
                     context,
+                    folder_meaning,
                     &headers,
                     &message_id,
                     fetch_response.flags(),
                 )
-                .await.context("prefetch_should_download")?
+                .await
+                .context("prefetch_should_download")?
             {
                 match download_limit {
                     Some(download_limit) => uids_fetch.push((
@@ -2244,10 +2241,34 @@ async fn prefetch_get_chat(
 /// Determines whether the message should be downloaded based on prefetched headers.
 pub(crate) async fn prefetch_should_download(
     context: &Context,
+    folder_meaning: FolderMeaning,
     headers: &[mailparse::MailHeader<'_>],
     message_id: &str,
     mut flags: impl Iterator<Item = Flag<'_>>,
 ) -> Result<bool> {
+    // Never download messages directly from the spam folder.
+    // If the sender is known, the message will be moved to the Inbox or Mvbox
+    // and then we download the message from there.
+    // Also see `spam_target_folder_cfg()`.
+    if folder_meaning == FolderMeaning::Spam {
+        return Ok(false);
+    }
+
+    // Download outgoing messages only from Inbox and Mvbox (and Sent -- until
+    // `Config::SentboxWatch` is dropped) to ensure they are sorted correctly with incoming
+    // messages. This way we actually do server-side sorting. Otherwise it's possible that outgoing
+    // replies from other devices appear before incoming messages. Users should CC/BCC/To themselves
+    // to see sent messages in Delta Chat.
+    if matches!(
+        folder_meaning,
+        FolderMeaning::Inbox | FolderMeaning::Mvbox | FolderMeaning::Sent
+    ) {
+    } else if let Some(from) = mimeparser::get_from(headers) {
+        if context.is_self_addr(&from.addr).await? {
+            return Ok(false);
+        }
+    }
+
     if message::rfc724_mid_exists(context, message_id)
         .await?
         .is_some()

src/imap/scan_folders.rs

@@ -71,10 +71,14 @@ impl Imap {
                 _ => folder_meaning,
             };
 
+            // Scan folders for incoming messages. As for outgoing ones, users should CC/BCC/To
+            // themselves so that Delta Chat sees such messages in Inbox.
+            //
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string())
                 && folder_meaning != FolderMeaning::Drafts
                 && folder_meaning != FolderMeaning::Trash
+                && folder_meaning != FolderMeaning::Sent
             {
                 self.fetch_move_delete(context, session, folder.name(), folder_meaning)
                     .await

src/receive_imf/receive_imf_tests.rs

@@ -12,7 +12,7 @@ use crate::chatlist::Chatlist;
 use crate::constants::DC_GCL_FOR_FORWARDING;
 use crate::contact;
 use crate::download::MIN_DOWNLOAD_LIMIT;
-use crate::imap::prefetch_should_download;
+use crate::imap::{FolderMeaning, prefetch_should_download};
 use crate::imex::{ImexMode, imex};
 use crate::securejoin::get_securejoin_qr;
 use crate::test_utils::{
@@ -672,9 +672,15 @@ async fn test_parse_ndn(
     // Check that the ndn would be downloaded:
     let headers = mailparse::parse_mail(raw_ndn).unwrap().headers;
     assert!(
-        prefetch_should_download(&t, &headers, "some-other-message-id", std::iter::empty(),)
-            .await
-            .unwrap()
+        prefetch_should_download(
+            &t,
+            FolderMeaning::Inbox,
+            &headers,
+            "some-other-message-id",
+            std::iter::empty(),
+        )
+        .await
+        .unwrap()
     );
 
     receive_imf(&t, raw_ndn, false).await.unwrap();

src/scheduler.rs

@@ -661,6 +661,7 @@ async fn fetch_idle(
 
     connection.connectivity.set_idle(ctx);
 
+    // Maybe we'll remove watching other folders soon, so use this event for all folders for now.
     ctx.emit_event(EventType::ImapInboxIdle);
 
     if !session.can_idle() {

src/tests/verified_chats.rs

@@ -316,9 +316,8 @@ async fn test_old_message_4() -> Result<()> {
     Ok(())
 }
 
-/// Alice is offline for some time.
-/// When they come online, first their sentbox is synced and then their inbox.
-/// This test tests that the messages are still in the right order.
+/// Alice's device#0 is offline for some time.
+/// When it comes online, it sees a message from another device and an incoming message.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_old_message_5() -> Result<()> {
     let alice = TestContext::new_alice().await;
@@ -348,11 +347,11 @@ async fn test_old_message_5() -> Result<()> {
     .await?
     .unwrap();
 
+    // It's not clear what we want here: if `msg_sent` is sent by Alice, it better to sort
+    // `msg_incoming` after it so that it's more visible (current behavior), but if Alice shares her
+    // account with someone else or has some auto-reply bot, messages should be sorted just by
+    // "Date".
     assert!(msg_sent.sort_timestamp == msg_incoming.sort_timestamp);
-    alice
-        .golden_test_chat(msg_sent.chat_id, "test_old_message_5")
-        .await;
-
     Ok(())
 }
 

test-data/golden/test_old_message_5

@@ -1,5 +0,0 @@
-Single#Chat#10: Bob [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png 
---------------------------------------------------------------------------------
-Msg#10: Me (Contact#Contact#Self): Happy birthday, Bob!  √
-Msg#11:  (Contact#Contact#10): Happy birthday to me, Alice! [FRESH]
---------------------------------------------------------------------------------