fix: Send pre-message after successful sending of post-message (#8063)

Also, send the message text in the post-message too, we can't support the workaround for duplicated
message text shown in ancient versions anymore, otherwise newer versions won't show the message text
at all.

On the receiving side, download post messages w/o known pre-message after 30s to protect from lost
messages.

This isn't easy to test in Rust though because the `smtp` sending code requires an SMTP connection,
so it's only tested (by already existing tests) that messages are queued in the right order.

Another problem is that the `smtp` sending logic doesn't try to send any messages following a
failed-to-send message until the retry limit is reached, so if there's smth wrong with a
post-message, other unrelated messages are delayed, but this problem has already existed.

.github/workflows/ci.yml

@@ -41,6 +41,8 @@ jobs:
         shell: bash
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Run rustfmt
         run: cargo fmt --all -- --check
       - name: Run clippy
@@ -92,6 +94,8 @@ jobs:
           persist-credentials: false
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Rustdoc
         run: cargo doc --document-private-items --no-deps
 
@@ -135,6 +139,8 @@ jobs:
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Install nextest
         uses: taiki-e/install-action@5f57d6cb7cd20b14a8a27f522884c4bc8a187458
@@ -169,6 +175,8 @@ jobs:
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Build C library
         run: cargo build -p deltachat_ffi
@@ -195,6 +203,8 @@ jobs:
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4
+        with:
+          save-if: ${{ github.ref == 'refs/heads/main' }}
 
       - name: Build deltachat-rpc-server
         run: cargo build -p deltachat-rpc-server

Cargo.lock

@@ -934,9 +934,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
 
 [[package]]
 name = "colorutils-rs"
-version = "0.7.6"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e2fc25857fa523662de5cae84225b0e7bfb24a2a3f9ed8802fecf03df7252b1"
+checksum = "69abc9a8ed011e2b7946769f460b9e76e8b659ece9ef4001b9d8bba3489f796d"
 dependencies = [
  "erydanos",
  "half",
@@ -1301,9 +1301,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.10.0"
+version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
+checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
 
 [[package]]
 name = "dbl"
@@ -2662,7 +2662,7 @@ dependencies = [
  "hyper",
  "libc",
  "pin-project-lite",
- "socket2 0.5.9",
+ "socket2 0.6.3",
  "tokio",
  "tower-service",
  "tracing",
@@ -3268,9 +3268,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.184"
+version = "0.2.186"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
 [[package]]
 name = "libm"
@@ -3941,15 +3941,14 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
 [[package]]
 name = "openssl"
-version = "0.10.78"
+version = "0.10.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222"
+checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"
 dependencies = [
  "bitflags 2.11.0",
  "cfg-if",
  "foreign-types",
  "libc",
- "once_cell",
  "openssl-macros",
  "openssl-sys",
 ]
@@ -3982,9 +3981,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.114"
+version = "0.9.115"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6"
+checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"
 dependencies = [
  "cc",
  "libc",

Cargo.toml

@@ -53,7 +53,7 @@ blake3 = "1.8.2"
 brotli = { version = "8", default-features=false, features = ["std"] }
 bytes = "1"
 chrono = { workspace = true, features = ["alloc", "clock", "std"] }
-colorutils-rs = { version = "0.7.5", default-features = false }
+colorutils-rs = { version = "0.8.0", default-features = false }
 data-encoding = "2.9.0"
 escaper = "0.1"
 fast-socks5 = "1"

deltachat-ffi/deltachat.h

@@ -4003,8 +4003,6 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  *   Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
  *
  * Outgoing message states:
- * - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent,
- *   the message enters this state before @ref DC_STATE_OUT_PENDING. Deprecated.
  * - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft()
  * - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the
  *   message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
@@ -5589,13 +5587,6 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define         DC_STATE_IN_SEEN             16
 
-/**
- * Outgoing message being prepared. See dc_msg_get_state() for details.
- *
- * @deprecated 2024-12-07
- */
-#define         DC_STATE_OUT_PREPARING       18
-
 /**
  * Outgoing message drafted. See dc_msg_get_state() for details.
  */

deltachat-ffi/src/lot.rs

@@ -230,7 +230,6 @@ pub enum LotState {
     MsgInFresh = 10,
     MsgInNoticed = 13,
     MsgInSeen = 16,
-    MsgOutPreparing = 18,
     MsgOutDraft = 19,
     MsgOutPending = 20,
     MsgOutFailed = 24,
@@ -246,7 +245,6 @@ impl From<MessageState> for LotState {
             InFresh => LotState::MsgInFresh,
             InNoticed => LotState::MsgInNoticed,
             InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
             OutDraft => LotState::MsgOutDraft,
             OutPending => LotState::MsgOutPending,
             OutFailed => LotState::MsgOutFailed,

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -190,7 +190,6 @@ class MessageState(IntEnum):
     IN_FRESH = 10
     IN_NOTICED = 13
     IN_SEEN = 16
-    OUT_PREPARING = 18
     OUT_DRAFT = 19
     OUT_PENDING = 20
     OUT_FAILED = 24

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -251,12 +251,7 @@ def test_realtime_large_webxdc(acfactory, path_to_large_webxdc):
     ac1_ac2_chat = ac1.create_chat(ac2)
     ac1_webxdc_msg = ac1_ac2_chat.send_message(text="realtime check", file=path_to_large_webxdc)
 
-    # Receive pre-message.
     ac2_webxdc_msg = ac2.wait_for_incoming_msg()
-
-    # Receive post-message.
-    ac2_webxdc_msg = ac2.wait_for_msg(EventType.MSGS_CHANGED)
-
     ac2_webxdc_msg.send_webxdc_realtime_advertisement()
     event = ac1.wait_for_event(EventType.WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED)
     assert event.msg_id == ac1_webxdc_msg.id

deltachat-rpc-client/tests/test_something.py

@@ -865,10 +865,6 @@ def test_delete_fully_downloaded_msg(acfactory, tmp_path, direct_imap):
 
     msg = bob.wait_for_incoming_msg()
     msg_snapshot = msg.get_snapshot()
-    assert msg_snapshot.download_state == DownloadState.AVAILABLE
-    msgs_changed_event = bob.wait_for_msgs_changed_event()
-    assert msgs_changed_event.msg_id == msg.id
-    msg_snapshot = msg.get_snapshot()
     assert msg_snapshot.download_state == DownloadState.DONE
 
     bob_direct_imap = direct_imap(bob)
@@ -899,10 +895,6 @@ def test_imap_autodelete_fully_downloaded_msg(acfactory, tmp_path, direct_imap):
 
     msg = bob.wait_for_incoming_msg()
     msg_snapshot = msg.get_snapshot()
-    assert msg_snapshot.download_state == DownloadState.AVAILABLE
-    msgs_changed_event = bob.wait_for_msgs_changed_event()
-    assert msgs_changed_event.msg_id == msg.id
-    msg_snapshot = msg.get_snapshot()
     assert msg_snapshot.download_state == DownloadState.DONE
 
     bob_direct_imap = direct_imap(bob)
@@ -1379,7 +1371,5 @@ def test_large_message(acfactory) -> None:
     )
 
     msg = bob.wait_for_incoming_msg()
-    msgs_changed_event = bob.wait_for_msgs_changed_event()
-    assert msg.id == msgs_changed_event.msg_id
     snapshot = msg.get_snapshot()
     assert snapshot.text == "Hello World, this message is bigger than 5 bytes"

python/src/deltachat/chat.py

@@ -271,15 +271,6 @@ class Chat:
            sent out.  This is the same object as was passed in, which
            has been modified with the new state of the core.
         """
-        if msg.is_out_preparing():
-            assert msg.id != 0
-            # get a fresh copy of dc_msg, the core needs it
-            maybe_msg = Message.from_db(self.account, msg.id)
-            if maybe_msg is not None:
-                msg = maybe_msg
-            else:
-                raise ValueError("message does not exist")
-
         sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
@@ -333,26 +324,6 @@ class Chat:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
 
-    def send_prepared(self, message):
-        """send a previously prepared message.
-
-        :param message: a :class:`Message` instance previously returned by
-                        :meth:`prepare_file`.
-        :raises ValueError: if message can not be sent.
-        :returns: a :class:`deltachat.message.Message` instance as sent out.
-        """
-        assert message.id != 0 and message.is_out_preparing()
-        # get a fresh copy of dc_msg, the core needs it
-        msg = Message.from_db(self.account, message.id)
-
-        # pass 0 as chat-id because core-docs say it's ok when out-preparing
-        sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
-        if sent_id == 0:
-            raise ValueError("message could not be sent")
-        assert sent_id == msg.id
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
-
     def set_draft(self, message):
         """set message as draft.
 

python/src/deltachat/message.py

@@ -351,17 +351,12 @@ class Message:
     def is_outgoing(self):
         """Return True if Message is outgoing."""
         return lib.dc_msg_get_state(self._dc_msg) in (
-            const.DC_STATE_OUT_PREPARING,
             const.DC_STATE_OUT_PENDING,
             const.DC_STATE_OUT_FAILED,
             const.DC_STATE_OUT_MDN_RCVD,
             const.DC_STATE_OUT_DELIVERED,
         )
 
-    def is_out_preparing(self):
-        """Return True if Message is outgoing, but its file is being prepared."""
-        return self._msgstate == const.DC_STATE_OUT_PREPARING
-
     def is_out_pending(self):
         """Return True if Message is outgoing, but is pending (no single checkmark)."""
         return self._msgstate == const.DC_STATE_OUT_PENDING

src/chat.rs

@@ -2613,7 +2613,7 @@ pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) ->
         "chat_id cannot be a special chat: {chat_id}"
     );
 
-    if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing {
+    if msg.state != MessageState::Undefined {
         msg.param.remove(Param::GuaranteeE2ee);
         msg.param.remove(Param::ForcePlaintext);
         // create_send_msg_jobs() will update `param` in the db.
@@ -2721,10 +2721,7 @@ async fn prepare_send_msg(
         None
     };
 
-    if matches!(
-        msg.state,
-        MessageState::Undefined | MessageState::OutPreparing
-    )
+    if msg.state == MessageState::Undefined
         // Legacy SecureJoin "v*-request" messages are unencrypted.
         && msg.param.get_cmd() != SystemMessage::SecurejoinMessage
         && chat.is_encrypted(context).await?
@@ -2937,8 +2934,8 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
 UPDATE msgs SET
     timestamp=(
         SELECT MAX(timestamp) FROM msgs INDEXED BY msgs_index7 WHERE
-            -- From `InFresh` to `OutMdnRcvd` inclusive except `OutDraft`.
-            state IN(10,13,16,18,20,24,26,28) AND
+            -- From `InFresh` to `OutDelivered` inclusive, except `OutDraft`.
+            state IN(10,13,16,18,20,24,26) AND
             hidden IN(0,1) AND
             chat_id=? AND
             id<=?
@@ -2973,15 +2970,6 @@ WHERE id=?
         )?;
         for recipients_chunk in recipients.chunks(chunk_size) {
             let recipients_chunk = recipients_chunk.join(" ");
-            if let Some(pre_msg) = &rendered_pre_msg {
-                let row_id = stmt.execute((
-                    &pre_msg.rfc724_mid,
-                    &recipients_chunk,
-                    &pre_msg.message,
-                    msg.id,
-                ))?;
-                row_ids.push(row_id.try_into()?);
-            }
             let row_id = stmt.execute((
                 &rendered_msg.rfc724_mid,
                 &recipients_chunk,
@@ -2989,6 +2977,16 @@ WHERE id=?
                 msg.id,
             ))?;
             row_ids.push(row_id.try_into()?);
+            let Some(pre_msg) = &rendered_pre_msg else {
+                continue;
+            };
+            let row_id = stmt.execute((
+                &pre_msg.rfc724_mid,
+                &recipients_chunk,
+                &pre_msg.message,
+                msg.id,
+            ))?;
+            row_ids.push(row_id.try_into()?);
         }
         Ok(row_ids)
     };
@@ -4539,6 +4537,7 @@ pub async fn forward_msgs_2ctx(
 
         msg.state = MessageState::OutPending;
         msg.rfc724_mid = create_outgoing_rfc724_mid();
+        msg.pre_rfc724_mid.clear();
         msg.timestamp_sort = curr_timestamp;
         chat.prepare_msg_raw(ctx_dst, &mut msg, None).await?;
 

src/download.rs

@@ -10,6 +10,7 @@ use crate::context::Context;
 use crate::imap::session::Session;
 use crate::log::warn;
 use crate::message::{self, Message, MsgId, rfc724_mid_exists};
+use crate::tools;
 use crate::{EventType, chatlist_events};
 
 pub(crate) mod post_msg_metadata;
@@ -320,12 +321,22 @@ pub(crate) async fn download_known_post_messages_without_pre_message(
     context: &Context,
     session: &mut Session,
 ) -> Result<()> {
+    const PRE_MSG_WAIT_TIME: i64 = 30;
+    let now = tools::time();
     let rfc724_mids = context
         .sql
-        .query_map_vec("SELECT rfc724_mid FROM available_post_msgs", (), |row| {
-            let rfc724_mid: String = row.get(0)?;
-            Ok(rfc724_mid)
-        })
+        .query_map_vec(
+            "
+SELECT rfc724_mid FROM available_post_msgs
+WHERE timestamp<=? OR timestamp>?
+ORDER BY timestamp, rowid
+            ",
+            (now.saturating_sub(PRE_MSG_WAIT_TIME), now),
+            |row| {
+                let rfc724_mid: String = row.get(0)?;
+                Ok(rfc724_mid)
+            },
+        )
         .await?;
     for rfc724_mid in &rfc724_mids {
         if msg_is_downloaded_for(context, rfc724_mid).await? {

src/imap.rs

@@ -601,7 +601,7 @@ impl Imap {
         let _fetch_msgs_lock_guard = context.fetch_msgs_mutex.lock().await;
 
         let mut uids_fetch: Vec<u32> = Vec::new();
-        let mut available_post_msgs: Vec<String> = Vec::new();
+        let mut available_post_msgs: Vec<(String, i64)> = Vec::new();
         let mut download_later: Vec<String> = Vec::new();
         let mut uid_message_ids = BTreeMap::new();
         let mut largest_uid_skipped = None;
@@ -689,7 +689,7 @@ impl Imap {
                     .is_some()
                 {
                     info!(context, "{message_id:?} is a post-message.");
-                    available_post_msgs.push(message_id.clone());
+                    available_post_msgs.push((message_id.clone(), time()));
 
                     let is_bot = context.get_config_bool(Config::Bot).await?;
                     if is_bot && download_limit.is_none_or(|download_limit| size <= download_limit)
@@ -793,9 +793,10 @@ impl Imap {
                 download_later.len(),
             );
             let trans_fn = |t: &mut rusqlite::Transaction| {
-                let mut stmt = t.prepare("INSERT OR IGNORE INTO available_post_msgs VALUES (?)")?;
-                for rfc724_mid in available_post_msgs {
-                    stmt.execute((rfc724_mid,))
+                let mut stmt =
+                    t.prepare("INSERT OR IGNORE INTO available_post_msgs VALUES (?,?)")?;
+                for entry in available_post_msgs {
+                    stmt.execute(entry)
                         .context("INSERT OR IGNORE INTO available_post_msgs")?;
                 }
                 let mut stmt =

src/message.rs

@@ -1381,13 +1381,8 @@ pub enum MessageState {
     /// IMAP and MDN may be sent.
     InSeen = 16,
 
-    /// For files which need time to be prepared before they can be
-    /// sent, the message enters this state before
-    /// OutPending.
-    ///
-    /// Deprecated 2024-12-07.
-    OutPreparing = 18,
-
+    // Deprecated 2024-12-07. Removed 2026-04.
+    // OutPreparing = 18,
     /// Message saved as draft.
     OutDraft = 19,
 
@@ -1420,7 +1415,6 @@ impl std::fmt::Display for MessageState {
                 Self::InFresh => "Fresh",
                 Self::InNoticed => "Noticed",
                 Self::InSeen => "Seen",
-                Self::OutPreparing => "Preparing",
                 Self::OutDraft => "Draft",
                 Self::OutPending => "Pending",
                 Self::OutFailed => "Failed",
@@ -1437,7 +1431,7 @@ impl MessageState {
         use MessageState::*;
         matches!(
             self,
-            OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
+            OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
         )
     }
 
@@ -1446,7 +1440,7 @@ impl MessageState {
         use MessageState::*;
         matches!(
             self,
-            OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
+            OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
         )
     }
 

src/mimefactory.rs

@@ -1723,23 +1723,19 @@ impl MimeFactory {
 
         let footer = if is_reaction { "" } else { &self.selfstatus };
 
-        let message_text = if self.pre_message_mode == PreMessageMode::Post {
-            "".to_string()
-        } else {
-            format!(
-                "{}{}{}{}{}{}",
-                fwdhint.unwrap_or_default(),
-                quoted_text.unwrap_or_default(),
-                escape_message_footer_marks(final_text),
-                if !final_text.is_empty() && !footer.is_empty() {
-                    "\r\n\r\n"
-                } else {
-                    ""
-                },
-                if !footer.is_empty() { "-- \r\n" } else { "" },
-                footer
-            )
-        };
+        let message_text = format!(
+            "{}{}{}{}{}{}",
+            fwdhint.unwrap_or_default(),
+            quoted_text.unwrap_or_default(),
+            escape_message_footer_marks(final_text),
+            if !final_text.is_empty() && !footer.is_empty() {
+                "\r\n\r\n"
+            } else {
+                ""
+            },
+            if !footer.is_empty() { "-- \r\n" } else { "" },
+            footer
+        );
 
         let mut main_part = MimePart::new("text/plain", message_text);
         if is_reaction {

src/net/dns.rs

@@ -231,10 +231,7 @@ static DNS_PRELOAD: LazyLock<HashMap<&'static str, Vec<IpAddr>>> = LazyLock::new
     HashMap::from([
         (
             "imap.163.com",
-            vec![
-                IpAddr::V4(Ipv4Addr::new(111, 124, 203, 45)),
-                IpAddr::V4(Ipv4Addr::new(111, 124, 203, 50)),
-            ],
+            vec![IpAddr::V4(Ipv4Addr::new(111, 124, 203, 45))],
         ),
         (
             "smtp.163.com",
@@ -425,12 +422,12 @@ static DNS_PRELOAD: LazyLock<HashMap<&'static str, Vec<IpAddr>>> = LazyLock::new
             "nine.testrun.org",
             vec![
                 IpAddr::V4(Ipv4Addr::new(128, 140, 126, 197)),
+                IpAddr::V4(Ipv4Addr::new(116, 202, 233, 236)),
                 IpAddr::V4(Ipv4Addr::new(216, 144, 228, 100)),
-                IpAddr::V4(Ipv4Addr::new(77, 42, 49, 41)),
+                IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f8, 0x241, 0x4ce8, 0, 0, 0, 2)),
                 IpAddr::V6(Ipv6Addr::new(
                     0x2001, 0x41d0, 0x701, 0x1100, 0, 0, 0, 0x8ab1,
                 )),
-                IpAddr::V6(Ipv6Addr::new(0x2a01, 0x4f9, 0xfff1, 0x59, 0, 0, 0, 1)),
             ],
         ),
         (
@@ -700,10 +697,6 @@ static DNS_PRELOAD: LazyLock<HashMap<&'static str, Vec<IpAddr>>> = LazyLock::new
             "chatmail.hackea.org",
             vec![IpAddr::V4(Ipv4Addr::new(82, 165, 11, 85))],
         ),
-        (
-            "chat.adminforge.de",
-            vec![IpAddr::V4(Ipv4Addr::new(94, 130, 17, 142))],
-        ),
         (
             "chika.aangat.lahat.computer",
             vec![IpAddr::V4(Ipv4Addr::new(71, 19, 150, 113))],
@@ -745,46 +738,6 @@ static DNS_PRELOAD: LazyLock<HashMap<&'static str, Vec<IpAddr>>> = LazyLock::new
             "danneskjold.de",
             vec![IpAddr::V4(Ipv4Addr::new(46, 62, 216, 132))],
         ),
-        (
-            "chat.in-the.eu",
-            vec![IpAddr::V4(Ipv4Addr::new(78, 46, 190, 129))],
-        ),
-        (
-            "chat.nuvon.app",
-            vec![IpAddr::V4(Ipv4Addr::new(178, 238, 38, 165))],
-        ),
-        (
-            "nibblehole.com",
-            vec![IpAddr::V4(Ipv4Addr::new(94, 247, 42, 209))],
-        ),
-        (
-            "chat.zashm.org",
-            vec![IpAddr::V4(Ipv4Addr::new(91, 245, 76, 39))],
-        ),
-        (
-            "chat.sus.fr",
-            vec![IpAddr::V4(Ipv4Addr::new(152, 67, 76, 190))],
-        ),
-        (
-            "delta.thelab.uno",
-            vec![IpAddr::V4(Ipv4Addr::new(146, 59, 228, 39))],
-        ),
-        (
-            "chat.vim.wtf",
-            vec![IpAddr::V4(Ipv4Addr::new(116, 203, 206, 170))],
-        ),
-        (
-            "uninterest.ing",
-            vec![IpAddr::V4(Ipv4Addr::new(172, 245, 70, 237))],
-        ),
-        (
-            "sweetfern.net",
-            vec![IpAddr::V4(Ipv4Addr::new(178, 156, 228, 133))],
-        ),
-        (
-            "delta.disobey.net",
-            vec![IpAddr::V4(Ipv4Addr::new(37, 74, 102, 44))],
-        ),
         (
             "darkrun.dev",
             vec![IpAddr::V4(Ipv4Addr::new(72, 11, 149, 146))],

src/quota.rs

@@ -66,12 +66,6 @@ impl Context {
 
     /// Updates `quota.recent`, sets `quota.modified` to the current time
     /// and emits an event to let the UIs update connectivity view.
-    ///
-    /// Moreover, once each time quota gets larger than `QUOTA_WARN_THRESHOLD_PERCENTAGE`,
-    /// a device message is added.
-    /// As the message is added only once, the user is not spammed
-    /// in case for some providers the quota is always at ~100%
-    /// and new space is allocated as needed.
     pub(crate) async fn update_recent_quota(
         &self,
         session: &mut ImapSession,

src/receive_imf.rs

@@ -525,18 +525,10 @@ pub(crate) async fn receive_imf_inner(
         "Receiving message {rfc724_mid_orig:?}, seen={seen}...",
     );
 
-    // These checks must be done before processing of SecureJoin and other special messages.
-    if mime_parser.pre_message == mimeparser::PreMessageMode::Post {
-        // Post-Message just replaces the attachment and modifies Params, not the whole message.
-        // This is done in the `handle_post_message` method.
-    } else if let Some(msg_id) = message::rfc724_mid_exists(context, rfc724_mid_orig).await? {
-        info!(
-            context,
-            "Message {rfc724_mid} is already in some chat or deleted."
-        );
-        if mime_parser.incoming {
-            return Ok(None);
-        }
+    let msg_id = message::rfc724_mid_exists(context, rfc724_mid_orig).await?;
+    if let Some(msg_id) = msg_id
+        && !mime_parser.incoming
+    {
         // For the case if we missed a successful SMTP response. Be optimistic that the message is
         // delivered also.
         let self_addr = context.get_primary_self_addr().await?;
@@ -551,6 +543,16 @@ pub(crate) async fn receive_imf_inner(
         if !msg_has_pending_smtp_job(context, msg_id).await? {
             msg_id.set_delivered(context).await?;
         }
+    }
+    // These checks must be done before processing of SecureJoin and other special messages.
+    if mime_parser.pre_message == mimeparser::PreMessageMode::Post {
+        // Post-Message just replaces the attachment and modifies Params, not the whole message.
+        // This is done in the `update_from_post_msg` method.
+    } else if msg_id.is_some() {
+        info!(
+            context,
+            "Message {rfc724_mid} is already in some chat or deleted."
+        );
         return Ok(None);
     }
 
@@ -806,8 +808,6 @@ UPDATE config SET value=? WHERE keyname='configured_addr' AND value!=?1
                 if transport_changed {
                     info!(context, "Primary transport changed to {from_addr:?}.");
                     context.sql.uncache_raw_config("configured_addr").await;
-
-                    // Regenerate User ID in V4 keys.
                     context.self_public_key.lock().await.take();
 
                     context.emit_event(EventType::TransportsModified);
@@ -2081,7 +2081,7 @@ async fn add_parts(
     }
 
     handle_edit_delete(context, mime_parser, from_id).await?;
-    handle_post_message(context, mime_parser, from_id, state).await?;
+    update_from_post_msg(context, mime_parser, from_id, state).await?;
 
     if mime_parser.is_system_message == SystemMessage::CallAccepted
         || mime_parser.is_system_message == SystemMessage::CallEnded
@@ -2275,8 +2275,7 @@ INSERT INTO msgs
 
     // Maybe set logging xdc and add gossip topics for webxdcs.
     for (part, msg_id) in mime_parser.parts.iter().zip(&created_db_entries) {
-        if mime_parser.pre_message != PreMessageMode::Post
-            && part.typ == Viewtype::Webxdc
+        if part.typ == Viewtype::Webxdc
             && let Some(topic) = mime_parser.get_header(HeaderDef::IrohGossipTopic)
         {
             let topic = iroh_topic_from_str(topic)?;
@@ -2423,7 +2422,7 @@ async fn handle_edit_delete(
     Ok(())
 }
 
-async fn handle_post_message(
+async fn update_from_post_msg(
     context: &Context,
     mime_parser: &MimeMessage,
     from_id: ContactId,
@@ -2441,7 +2440,7 @@ async fn handle_post_message(
     let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else {
         warn!(
             context,
-            "handle_post_message: {rfc724_mid}: Database entry does not exist."
+            "update_from_post_msg: {rfc724_mid}: Database entry does not exist."
         );
         return Ok(());
     };
@@ -2449,7 +2448,7 @@ async fn handle_post_message(
         // else: message is processed like a normal message
         warn!(
             context,
-            "handle_post_message: {rfc724_mid}: Pre-message was not downloaded yet so treat as normal message."
+            "update_from_post_msg: {rfc724_mid}: Pre-message was not downloaded yet so treat as normal message."
         );
         return Ok(());
     };
@@ -2460,7 +2459,7 @@ async fn handle_post_message(
     // Do nothing if safety checks fail, the worst case is the message modifies the chat if the
     // sender is a member.
     if from_id != original_msg.from_id {
-        warn!(context, "handle_post_message: {rfc724_mid}: Bad sender.");
+        warn!(context, "update_from_post_msg: {rfc724_mid}: Bad sender.");
         return Ok(());
     }
     let post_msg_showpadlock = part
@@ -2468,14 +2467,17 @@ async fn handle_post_message(
         .get_bool(Param::GuaranteeE2ee)
         .unwrap_or_default();
     if !post_msg_showpadlock && original_msg.get_showpadlock() {
-        warn!(context, "handle_post_message: {rfc724_mid}: Not encrypted.");
+        warn!(
+            context,
+            "update_from_post_msg: {rfc724_mid}: Not encrypted."
+        );
         return Ok(());
     }
 
     if !part.typ.has_file() {
         warn!(
             context,
-            "handle_post_message: {rfc724_mid}: First mime part's message-viewtype has no file."
+            "update_from_post_msg: {rfc724_mid}: First mime part's message-viewtype has no file."
         );
         return Ok(());
     }
@@ -2506,7 +2508,10 @@ WHERE id=?
                 part.typ,
                 part.bytes as isize,
                 part.error.as_deref().unwrap_or_default(),
-                state,
+                match mime_parser.incoming {
+                    true => state,
+                    false => MessageState::Undefined,
+                },
                 DownloadState::Done as u32,
                 original_msg.id,
             ),

src/receive_imf/receive_imf_tests.rs

@@ -5592,27 +5592,27 @@ async fn test_mark_message_as_delivered_only_after_sent_out_fully() -> Result<()
         .await
         .unwrap();
 
-    let (pre_msg_id, pre_msg_payload) = first_row_in_smtp_queue(alice).await;
-    assert_eq!(msg_id, pre_msg_id);
-    assert!(pre_msg_payload.len() < file_bytes.len());
-
-    assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
-    // Alice receives her own pre-message because of bcc_self
-    // This should not yet mark the message as delivered,
-    // because not everything was sent,
-    // but it does remove the pre-message from the SMTP queue
-    receive_imf(alice, pre_msg_payload.as_bytes(), false).await?;
-    assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
-
     let (post_msg_id, post_msg_payload) = first_row_in_smtp_queue(alice).await;
     assert_eq!(msg_id, post_msg_id);
     assert!(post_msg_payload.len() > file_bytes.len());
 
     assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
     // Alice receives her own post-message because of bcc_self
+    // This should not yet mark the message as delivered,
+    // because not everything was sent,
+    // but it does remove the post-message from the SMTP queue.
+    receive_imf(alice, post_msg_payload.as_bytes(), false).await?;
+    assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
+
+    let (pre_msg_id, pre_msg_payload) = first_row_in_smtp_queue(alice).await;
+    assert_eq!(msg_id, pre_msg_id);
+    assert!(pre_msg_payload.len() < file_bytes.len());
+
+    assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
+    // Alice receives her own pre-message because of bcc_self
     // This should now mark the message as delivered,
     // because everything was sent by now.
-    receive_imf(alice, post_msg_payload.as_bytes(), false).await?;
+    receive_imf(alice, pre_msg_payload.as_bytes(), false).await?;
     assert_eq!(msg_id.get_state(alice).await?, MessageState::OutDelivered);
 
     Ok(())

src/smtp.rs

@@ -379,7 +379,7 @@ pub(crate) async fn send_msg_to_smtp(
     if retries > 6 {
         context
             .sql
-            .execute("DELETE FROM smtp WHERE id=?", (rowid,))
+            .execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))
             .await
             .context("Failed to remove message with exceeded retry limit from smtp table")?;
         if let Some(mut msg) = Message::load_from_db_optional(context, msg_id).await? {

src/sql/migrations.rs

@@ -2373,6 +2373,31 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT;
         .await?;
     }
 
+    inc_and_check(&mut migration_version, 152)?;
+    if dbversion < migration_version {
+        sql.execute_migration(
+            "
+UPDATE msgs SET state=26 WHERE state=28; -- Change OutMdnRcvd to OutDelivered.
+UPDATE msgs SET state=19 WHERE state=24; -- Change OutPreparing to OutFailed.
+            ",
+            migration_version,
+        )
+        .await?;
+    }
+
+    inc_and_check(&mut migration_version, 153)?;
+    if dbversion < migration_version {
+        sql.execute_migration(
+            "
+CREATE INDEX smtp_index_msg_id ON smtp (msg_id, id);
+ALTER TABLE available_post_msgs ADD COLUMN timestamp INTEGER DEFAULT 0 NOT NULL;
+CREATE INDEX available_post_msgs_timestamp ON available_post_msgs (timestamp);
+            ",
+            migration_version,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?

src/test_utils.rs

@@ -722,12 +722,14 @@ ORDER BY id"
         })
     }
 
+    /// Returns `SentMessage` instances representing `smtp` rows for the given message. Returned
+    /// items go in reverse order for historical reasons.
     pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec<SentMessage<'a>> {
         let sent_msgs = self
             .ctx
             .sql
             .query_map_vec(
-                "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?",
+                "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=? ORDER BY id DESC",
                 (msg_id,),
                 |row| {
                     let _id: MsgId = row.get(0)?;
@@ -1055,9 +1057,17 @@ ORDER BY id"
     /// This is not hooked up to any SMTP-IMAP pipeline, so the other account must call
     /// [`TestContext::recv_msg`] with the returned [`SentMessage`] if it wants to receive
     /// the message.
+    ///
+    /// Removes SMTP jobs existed before and marks the corresponding messages as delivered, as
+    /// tracking of these jobs is probably already lost by the test code.
     pub async fn send_msg(&self, chat_id: ChatId, msg: &mut Message) -> SentMessage<'_> {
+        while self.pop_sent_msg_opt(Duration::ZERO).await.is_some() {}
         let msg_id = chat::send_msg(self, chat_id, msg).await.unwrap();
-        let res = self.pop_sent_msg().await;
+        let rev_order = false;
+        let res = self
+            .pop_sent_msg_ex(rev_order, Duration::ZERO)
+            .await
+            .unwrap();
         assert_eq!(
             res.sender_msg_id, msg_id,
             "Apparently the message was not actually sent out"

src/tests/pre_messages/forward_and_save.rs

@@ -1,4 +1,6 @@
 //! Tests about forwarding and saving Pre-Messages
+use std::time::Duration;
+
 use anyhow::Result;
 use pretty_assertions::assert_eq;
 
@@ -8,6 +10,7 @@ use crate::chatlist::get_last_message_for_chat;
 use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD};
 use crate::message::{Message, Viewtype};
 use crate::test_utils::TestContextManager;
+use crate::tests::pre_messages::util::send_large_file_message;
 
 /// Test that forwarding Pre-Message should forward additional text to not be empty
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -86,6 +89,43 @@ async fn test_forwarding_pre_message_empty_text() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_both() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_chat_id = alice.create_group_with_members("", &[bob]).await;
+
+    let (pre_message, post_message, alice_msg_id) =
+        send_large_file_message(alice, alice_chat_id, Viewtype::File, &vec![0u8; 200_000]).await?;
+
+    let msg = bob.recv_msg(&pre_message).await;
+    let _ = bob.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(bob, msg.id).await?;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.text, "test".to_owned());
+
+    forward_msgs(alice, &[alice_msg_id], alice_chat_id).await?;
+    let rev_order = true;
+    let msg = bob
+        .recv_msg(
+            &alice
+                .pop_sent_msg_ex(rev_order, Duration::ZERO)
+                .await
+                .unwrap(),
+        )
+        .await;
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.is_forwarded(), true);
+    assert_eq!(msg.text, "test".to_owned());
+    let _ = bob.recv_msg_trash(&alice.pop_sent_msg().await).await;
+    let msg = Message::load_from_db(bob, msg.id).await?;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.is_forwarded(), true);
+    assert_eq!(msg.text, "test".to_owned());
+    Ok(())
+}
+
 /// Test that forwarding Pre-Message should forward additional text to not be empty
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_saving_pre_message_empty_text() -> Result<()> {

src/tests/pre_messages/receiving.rs

@@ -257,7 +257,7 @@ async fn test_lost_pre_msg() -> Result<()> {
     let _pre_msg = alice.pop_sent_msg().await;
     let msg = bob.recv_msg(&full_msg).await;
     assert_eq!(msg.download_state, DownloadState::Done);
-    assert_eq!(msg.text, "");
+    assert_eq!(msg.text, "populate");
     Ok(())
 }
 
@@ -547,8 +547,8 @@ async fn test_webxdc_updates_in_post_message_after_pre_message() -> Result<()> {
         .await?;
 
     send_msg(alice, alice_chat_id, &mut alice_instance).await?;
-    let post_message = alice.pop_sent_msg().await;
     let pre_message = alice.pop_sent_msg().await;
+    let post_message = alice.pop_sent_msg().await;
 
     let bob_instance = bob.recv_msg(&pre_message).await;
     assert_eq!(bob_instance.download_state, DownloadState::Available);
@@ -588,8 +588,8 @@ async fn test_webxdc_updates_in_post_message_without_pre_message() -> Result<()>
         .await?;
 
     send_msg(alice, alice_chat_id, &mut alice_instance).await?;
-    let post_message = alice.pop_sent_msg().await;
     let pre_message = alice.pop_sent_msg().await;
+    let post_message = alice.pop_sent_msg().await;
 
     // Bob receives post-message first.
     let bob_instance = bob.recv_msg(&post_message).await;

src/transport.rs

@@ -791,7 +791,18 @@ pub(crate) async fn sync_transports(
     context
         .sql
         .transaction(|transaction| {
+            let configured_addr = transaction.query_row(
+                "SELECT value FROM config WHERE keyname='configured_addr'",
+                (),
+                |row| {
+                    let addr: String = row.get(0)?;
+                    Ok(addr)
+                },
+            )?;
             for RemovedTransportData { addr, timestamp } in removed_transports {
+                if *addr == configured_addr {
+                    continue;
+                }
                 modified |= transaction.execute(
                     "DELETE FROM transports
                      WHERE addr=? AND add_timestamp<=?",

src/webxdc.rs

@@ -550,7 +550,7 @@ impl Context {
 
         let send_now = !matches!(
             instance.state,
-            MessageState::Undefined | MessageState::OutPreparing | MessageState::OutDraft
+            MessageState::Undefined | MessageState::OutDraft
         );
 
         status_update.uid = Some(create_id());