fix: Manipulate sort_timestamp to not be 0

This is to avoid sorting incoming messages that
are slightly in the past above system messages
about SecureJoin. SecureJoin messages are
timed according to smeared timestamp,
so even in the local tests they are in the future
by a few seconds.

.github/workflows/dependabot.yml

@@ -10,7 +10,7 @@ permissions:
 jobs:
   dependabot:
     runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
+    if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == github.event.pull_request.head.repo.full_name
     steps:
       - name: Dependabot metadata
         id: metadata

.github/workflows/upload-docs.yml

@@ -1,16 +1,18 @@
-name: Build & deploy documentation on rs.delta.chat, c.delta.chat, and py.delta.chat
+name: Build & deploy documentation on rs.delta.chat, c.delta.chat, py.delta.chat and cffi.delta.chat
 
 on:
   push:
     branches:
       - main
-      - build_jsonrpc_docs_ci
 
 permissions: {}
 
 jobs:
   build-rs:
     runs-on: ubuntu-latest
+    environment:
+      name: rs.delta.chat
+      url: https://rs.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -23,12 +25,15 @@ jobs:
       - name: Upload to rs.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.RS_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.USERNAME }}@rs.delta.chat:/var/www/html/rs/"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.RS_DOCS_SSH_USER }}@rs.delta.chat:/var/www/html/rs.delta.chat/"
 
   build-python:
     runs-on: ubuntu-latest
+    environment:
+      name: py.delta.chat
+      url: https://py.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -42,12 +47,15 @@ jobs:
       - name: Upload to py.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.PY_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "${{ secrets.PY_DOCS_SSH_USER }}@py.delta.chat:/var/www/html/py.delta.chat"
 
   build-c:
     runs-on: ubuntu-latest
+    environment:
+      name: c.delta.chat
+      url: https://c.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -61,12 +69,16 @@ jobs:
       - name: Upload to c.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.C_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "${{ secrets.C_DOCS_SSH_USER }}@c.delta.chat:/var/www/html/c.delta.chat"
 
   build-ts:
     runs-on: ubuntu-latest
+    environment:
+      name: js.jsonrpc.delta.chat
+      url: https://js.jsonrpc.delta.chat/
+
     defaults:
       run:
         working-directory: ./deltachat-jsonrpc/typescript
@@ -90,6 +102,27 @@ jobs:
       - name: Upload to js.jsonrpc.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.JS_JSONRPC_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.USERNAME }}@js.jsonrpc.delta.chat:/var/www/html/js-jsonrpc/"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.JS_JSONRPC_DOCS_SSH_USER }}@js.jsonrpc.delta.chat:/var/www/html/js.jsonrpc.delta.chat/"
+
+  build-cffi:
+    runs-on: ubuntu-latest
+    environment:
+      name: cffi.delta.chat
+      url: https://cffi.delta.chat/
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          show-progress: false
+          persist-credentials: false
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat_ffi --no-deps
+      - name: Upload to cffi.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CFFI_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.CFFI_DOCS_SSH_USER }}@delta.chat:/var/www/html/cffi.delta.chat/"

.github/workflows/upload-ffi-docs.yml

@@ -1,31 +0,0 @@
-# GitHub Actions workflow
-# to build `deltachat_ffi` crate documentation
-# and upload it to <https://cffi.delta.chat/>
-
-name: Build & Deploy Documentation on cffi.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-permissions: {}
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          show-progress: false
-          persist-credentials: false
-      - name: Build the documentation with cargo
-        run: |
-          cargo doc --package deltachat_ffi --no-deps
-      - name: Upload to cffi.delta.chat
-        run: |
-          mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
-          chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"

src/chat.rs

@@ -477,12 +477,17 @@ impl ChatId {
     /// Adds message "Messages are end-to-end encrypted".
     pub(crate) async fn add_e2ee_notice(self, context: &Context, timestamp: i64) -> Result<()> {
         let text = stock_str::messages_e2ee_info_msg(context);
+
+        // Sort this notice to the very beginning of the chat.
+        // We don't want any message to appear before this notice
+        // which is normally added when encrypted chat is created.
+        let sort_timestamp = 0;
         add_info_msg_with_cmd(
             context,
             self,
             &text,
             SystemMessage::ChatE2ee,
-            Some(timestamp),
+            Some(sort_timestamp),
             timestamp,
             None,
             None,
@@ -925,6 +930,17 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
             .unwrap_or(0))
     }
 
+    /// Returns timestamp of us joining the chat if we are the member of the chat.
+    pub(crate) async fn join_timestamp(self, context: &Context) -> Result<Option<i64>> {
+        context
+            .sql
+            .query_get_value(
+                "SELECT add_timestamp FROM chats_contacts WHERE chat_id=? AND contact_id=?",
+                (self, ContactId::SELF),
+            )
+            .await
+    }
+
     /// Returns timestamp of the latest message in the chat,
     /// including hidden messages or a draft if there is one.
     pub(crate) async fn get_timestamp(self, context: &Context) -> Result<Option<i64>> {
@@ -1212,15 +1228,11 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
     /// corresponding event in case of a system message (usually the current system time).
     /// `always_sort_to_bottom` makes this adjust the returned timestamp up so that the message goes
     /// to the chat bottom.
-    /// `received` -- whether the message is received. Otherwise being sent.
-    /// `incoming` -- whether the message is incoming.
     pub(crate) async fn calc_sort_timestamp(
         self,
         context: &Context,
         message_timestamp: i64,
         always_sort_to_bottom: bool,
-        received: bool,
-        incoming: bool,
     ) -> Result<i64> {
         let mut sort_timestamp = cmp::min(message_timestamp, smeared_time(context));
 
@@ -1240,38 +1252,6 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
                     (self, MessageState::OutDraft),
                 )
                 .await?
-        } else if received {
-            // Received messages shouldn't mingle with just sent ones and appear somewhere in the
-            // middle of the chat, so we go after the newest non fresh message.
-            //
-            // But if a received outgoing message is older than some seen message, better sort the
-            // received message purely by timestamp. We could place it just before that seen
-            // message, but anyway the user may not notice it.
-            //
-            // NB: Received outgoing messages may break sorting of fresh incoming ones, but this
-            // shouldn't happen frequently. Seen incoming messages don't really break sorting of
-            // fresh ones, they rather mean that older incoming messages are actually seen as well.
-            context
-                .sql
-                .query_row_optional(
-                    "SELECT MAX(timestamp), MAX(IIF(state=?,timestamp_sent,0))
-                     FROM msgs
-                     WHERE chat_id=? AND hidden=0 AND state>?
-                     HAVING COUNT(*) > 0",
-                    (MessageState::InSeen, self, MessageState::InFresh),
-                    |row| {
-                        let ts: i64 = row.get(0)?;
-                        let ts_sent_seen: i64 = row.get(1)?;
-                        Ok((ts, ts_sent_seen))
-                    },
-                )
-                .await?
-                .and_then(|(ts, ts_sent_seen)| {
-                    match incoming || ts_sent_seen <= message_timestamp {
-                        true => Some(ts),
-                        false => None,
-                    }
-                })
         } else {
             None
         };
@@ -1282,7 +1262,16 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
             sort_timestamp = last_msg_time;
         }
 
-        Ok(sort_timestamp)
+        if let Some(join_timestamp) = self.join_timestamp(context).await? {
+            // If we are the member of the chat, don't add messages
+            // before the timestamp of us joining it.
+            // This is needed to avoid sorting "Member added"
+            // or automatically sent bot welcome messages
+            // above SecureJoin system messages.
+            Ok(std::cmp::max(sort_timestamp, join_timestamp))
+        } else {
+            Ok(sort_timestamp)
+        }
     }
 }
 
@@ -4938,15 +4927,8 @@ pub(crate) async fn add_info_msg_with_cmd(
         ts
     } else {
         let sort_to_bottom = true;
-        let (received, incoming) = (false, false);
         chat_id
-            .calc_sort_timestamp(
-                context,
-                smeared_time(context),
-                sort_to_bottom,
-                received,
-                incoming,
-            )
+            .calc_sort_timestamp(context, smeared_time(context), sort_to_bottom)
             .await?
     };
 

src/chatlist.rs

@@ -142,7 +142,7 @@ impl Chatlist {
                    AND c.blocked!=1
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2 AND add_timestamp >= remove_timestamp)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY c.archived=?3 DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                 (MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned),
                 process_row,
             ).await?
@@ -168,7 +168,7 @@ impl Chatlist {
                    AND c.blocked!=1
                    AND c.archived=1
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft,),
                     process_row,
                 )
@@ -204,7 +204,7 @@ impl Chatlist {
                    AND IFNULL(c.name_normalized,c.name) LIKE ?3
                    AND (NOT ?4 OR EXISTS (SELECT 1 FROM msgs m WHERE m.chat_id = c.id AND m.state == ?5 AND hidden=0))
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, str_like_cmd, only_unread, MessageState::InFresh),
                     process_row,
                 )
@@ -253,7 +253,7 @@ impl Chatlist {
                        AND NOT c.archived=?
                        AND (c.type!=? OR c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=? AND add_timestamp >= remove_timestamp))
                      GROUP BY c.id
-                     ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                     ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (
                         MessageState::OutDraft, skip_id, ChatVisibility::Archived,
                         Chattype::Group, ContactId::SELF,
@@ -279,7 +279,7 @@ impl Chatlist {
                        AND (c.blocked=0 OR c.blocked=2)
                        AND NOT c.archived=?
                      GROUP BY c.id
-                     ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                     ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, ChatVisibility::Archived, ChatVisibility::Pinned),
                     process_row,
                 ).await?

src/configure.rs

@@ -243,11 +243,6 @@ impl Context {
                         Ok((id, add_timestamp))
                     },
                 )?;
-                transaction.execute("DELETE FROM imap WHERE transport_id=?", (transport_id,))?;
-                transaction.execute(
-                    "DELETE FROM imap_sync WHERE transport_id=?",
-                    (transport_id,),
-                )?;
 
                 // Removal timestamp should not be lower than addition timestamp
                 // to be accepted by other devices when synced.

src/imap.rs

@@ -323,8 +323,7 @@ impl Imap {
         if !ratelimit_duration.is_zero() {
             warn!(
                 context,
-                "Transport {}: IMAP got rate limited, waiting for {} until can connect.",
-                self.transport_id,
+                "IMAP got rate limited, waiting for {} until can connect.",
                 duration_to_str(ratelimit_duration),
             );
             let interrupted = async {
@@ -336,16 +335,12 @@ impl Imap {
             if interrupted {
                 info!(
                     context,
-                    "Transport {}: Connecting to IMAP without waiting for ratelimit due to interrupt.",
-                    self.transport_id
+                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
                 );
             }
         }
 
-        info!(
-            context,
-            "Transport {}: Connecting to IMAP server.", self.transport_id
-        );
+        info!(context, "Connecting to IMAP server.");
         self.connectivity.set_connecting(context);
 
         self.conn_last_try = tools::Time::now();
@@ -360,10 +355,7 @@ impl Imap {
         let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
         let mut first_error = None;
         for lp in login_params {
-            info!(
-                context,
-                "Transport {}: IMAP trying to connect to {}.", self.transport_id, &lp.connection
-            );
+            info!(context, "IMAP trying to connect to {}.", &lp.connection);
             let connection_candidate = lp.connection.clone();
             let client = match Client::connect(
                 context,
@@ -411,10 +403,7 @@ impl Imap {
                     let resync_request_sender = self.resync_request_sender.clone();
 
                     let session = if capabilities.can_compress {
-                        info!(
-                            context,
-                            "Transport {}: Enabling IMAP compression.", self.transport_id
-                        );
+                        info!(context, "Enabling IMAP compression.");
                         let compressed_session = session
                             .compress(|s| {
                                 let session_stream: Box<dyn SessionStream> = Box::new(s);
@@ -447,10 +436,7 @@ impl Imap {
                         lp.user
                     )));
                     self.connectivity.set_preparing(context);
-                    info!(
-                        context,
-                        "Transport {}: Successfully logged into IMAP server.", self.transport_id
-                    );
+                    info!(context, "Successfully logged into IMAP server.");
                     return Ok(session);
                 }
 
@@ -458,10 +444,7 @@ impl Imap {
                     let imap_user = lp.user.to_owned();
                     let message = stock_str::cannot_login(context, &imap_user);
 
-                    warn!(
-                        context,
-                        "Transport {}: IMAP failed to login: {err:#}.", self.transport_id
-                    );
+                    warn!(context, "IMAP failed to login: {err:#}.");
                     first_error.get_or_insert(format_err!("{message} ({err:#})"));
 
                     // If it looks like the password is wrong, send a notification:
@@ -480,11 +463,7 @@ impl Imap {
                             )
                             .await
                             {
-                                warn!(
-                                    context,
-                                    "Transport {}: Failed to add device message: {e:#}.",
-                                    self.transport_id
-                                );
+                                warn!(context, "Failed to add device message: {e:#}.");
                             } else {
                                 context
                                     .set_config_internal(Config::NotifyAboutWrongPw, None)
@@ -546,21 +525,10 @@ impl Imap {
             bail!("IMAP operation attempted while it is torn down");
         }
 
-        let transport_id = session.transport_id();
-        info!(
-            context,
-            "Transport {transport_id}: fetch_move_delete start."
-        );
-
         let msgs_fetched = self
             .fetch_new_messages(context, session, watch_folder, folder_meaning)
             .await
             .context("fetch_new_messages")?;
-
-        info!(
-            context,
-            "Transport {transport_id}: fetch_move_delete finished fetch_new_messages."
-        );
         if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
             // New messages were fetched and shall be deleted later, restart ephemeral loop.
             // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
@@ -599,18 +567,10 @@ impl Imap {
             return Ok(false);
         }
 
-        info!(
-            context,
-            "Transport {transport_id}: fetch_new_messages selects folder {folder:?}."
-        );
         let folder_exists = session
             .select_with_uidvalidity(context, folder)
             .await
             .with_context(|| format!("Failed to select folder {folder:?}"))?;
-        info!(
-            context,
-            "Transport {transport_id}: fetch_new_messages selected folder {folder:?}."
-        );
 
         if !session.new_mail {
             info!(
@@ -1143,16 +1103,22 @@ impl Session {
             return Ok(());
         }
 
+        context
+            .sql
+            .execute(
+                "DELETE FROM imap_markseen WHERE id NOT IN (SELECT imap.id FROM imap)",
+                (),
+            )
+            .await?;
+
         let transport_id = self.transport_id();
-        info!(context, "Transport {transport_id}: Storing seen flags.");
-        let rows = context
+        let mut rows = context
             .sql
             .query_map_vec(
                 "SELECT imap.id, uid, folder FROM imap, imap_markseen
                  WHERE imap.id = imap_markseen.id
                  AND imap.transport_id=?
-                 AND target = folder
-                 ORDER BY folder, uid",
+                 AND target = folder",
                 (transport_id,),
                 |row| {
                     let rowid: i64 = row.get(0)?;
@@ -1163,6 +1129,16 @@ impl Session {
             )
             .await?;
 
+        // Number of SQL results is expected to be low as
+        // we usually don't have many messages to mark on IMAP at once.
+        // We are sorting outside of SQL to avoid SQLite constructing a query plan
+        // that scans the whole `imap` table. Scanning `imap_markseen` is fine
+        // as it should not have many items.
+        // If you change the SQL query, test it with `EXPLAIN QUERY PLAN`.
+        rows.sort_unstable_by(|(_rowid1, uid1, folder1), (_rowid2, uid2, folder2)| {
+            (folder1, uid1).cmp(&(folder2, uid2))
+        });
+
         for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
             let folder_exists = match self.select_with_uidvalidity(context, &folder).await {
                 Err(err) => {
@@ -1179,15 +1155,13 @@ impl Session {
             } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
                 warn!(
                     context,
-                    "Transport {transport_id}: Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
+                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
                 );
                 continue;
             } else {
                 info!(
                     context,
-                    "Transport {transport_id}: Marked messages {} in folder {} as seen.",
-                    uid_set,
-                    folder
+                    "Marked messages {} in folder {} as seen.", uid_set, folder
                 );
             }
             context
@@ -1202,10 +1176,6 @@ impl Session {
                 .await
                 .context("Cannot remove messages marked as seen from imap_markseen table")?;
         }
-        info!(
-            context,
-            "Transport {transport_id}: Finished storing seen flags."
-        );
 
         Ok(())
     }
@@ -1546,10 +1516,9 @@ impl Session {
             return Ok(());
         }
 
-        let transport_id = self.transport_id();
         info!(
             context,
-            "Transport {transport_id}: Server supports metadata, retrieving server comment and admin contact."
+            "Server supports metadata, retrieving server comment and admin contact."
         );
 
         let mut comment = None;
@@ -1582,8 +1551,7 @@ impl Session {
                         } else {
                             warn!(
                                 context,
-                                "Transport {transport_id}: Got invalid URL from iroh relay metadata: {:?}.",
-                                value
+                                "Got invalid URL from iroh relay metadata: {:?}.", value
                             );
                         }
                     }
@@ -1612,7 +1580,6 @@ impl Session {
             create_fallback_ice_servers()
         };
 
-        info!(context, "Transport {transport_id}: Got IMAP metadata.");
         *lock = Some(ServerMetadata {
             comment,
             admin,

src/message.rs

@@ -815,7 +815,11 @@ impl Message {
 
     /// Returns the timestamp of the message for sorting.
     pub fn get_sort_timestamp(&self) -> i64 {
-        self.timestamp_sort
+        if self.timestamp_sort != 0 {
+            self.timestamp_sort
+        } else {
+            self.timestamp_sent
+        }
     }
 
     /// Returns the text of the message.

src/mimefactory.rs

@@ -852,7 +852,13 @@ impl MimeFactory {
 
         let rfc724_mid = match &self.loaded {
             Loaded::Message { msg, .. } => match &self.pre_message_mode {
-                PreMessageMode::Pre { .. } => create_outgoing_rfc724_mid(),
+                PreMessageMode::Pre { .. } => {
+                    if msg.pre_rfc724_mid.is_empty() {
+                        create_outgoing_rfc724_mid()
+                    } else {
+                        msg.pre_rfc724_mid.clone()
+                    }
+                }
                 _ => msg.rfc724_mid.clone(),
             },
             Loaded::Mdn { .. } => create_outgoing_rfc724_mid(),

src/receive_imf.rs

@@ -1835,6 +1835,16 @@ async fn add_parts(
         }
     }
 
+    // Sort message to the bottom if we are not in the chat
+    // so if we are added via QR code scan
+    // the message about our addition goes after all the info messages.
+    // Info messages are sorted by local smeared_timestamp()
+    // which advances quickly during SecureJoin,
+    // while "member added" message may have older timestamp
+    // corresponding to the sender clock.
+    // In practice inviter clock may even be slightly in the past.
+    let sort_to_bottom = !chat.is_self_in_chat(context).await?;
+
     let is_location_kml = mime_parser.location_kml.is_some();
     let mut group_changes = match chat.typ {
         _ if chat.id.is_special() => GroupChangesInfo::default(),
@@ -1885,16 +1895,8 @@ async fn add_parts(
     };
     let in_fresh = state == MessageState::InFresh;
 
-    let sort_to_bottom = false;
-    let received = true;
     let sort_timestamp = chat_id
-        .calc_sort_timestamp(
-            context,
-            mime_parser.timestamp_sent,
-            sort_to_bottom,
-            received,
-            mime_parser.incoming,
-        )
+        .calc_sort_timestamp(context, mime_parser.timestamp_sent, sort_to_bottom)
         .await?;
 
     // Apply ephemeral timer changes to the chat.

src/receive_imf/receive_imf_tests.rs

@@ -819,9 +819,12 @@ async fn load_imf_email(context: &Context, imf_raw: &[u8]) -> Message {
         .set_config(Config::ShowEmails, Some("2"))
         .await
         .unwrap();
-    receive_imf(context, imf_raw, false).await.unwrap();
-    let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
-    let msg_id = chats.get_msg_id(0).unwrap().unwrap();
+    let received_msg = receive_imf(context, imf_raw, false)
+        .await
+        .expect("receive_imf failure")
+        .expect("No message received");
+    assert_eq!(received_msg.msg_ids.len(), 1);
+    let msg_id = received_msg.msg_ids[0];
     Message::load_from_db(context, msg_id).await.unwrap()
 }
 
@@ -2872,9 +2875,8 @@ async fn test_rfc1847_encapsulation() -> Result<()> {
 
     // Alice sends a message to Bob using Thunderbird.
     let raw = include_bytes!("../../test-data/message/rfc1847_encapsulation.eml");
-    receive_imf(bob, raw, false).await?;
 
-    let msg = bob.get_last_msg().await;
+    let msg = load_imf_email(bob, raw).await;
     assert!(msg.get_showpadlock());
 
     Ok(())
@@ -3082,8 +3084,8 @@ async fn test_auto_accept_for_bots() -> Result<()> {
 async fn test_auto_accept_group_for_bots() -> Result<()> {
     let t = TestContext::new_alice().await;
     t.set_config(Config::Bot, Some("1")).await.unwrap();
-    receive_imf(&t, GRP_MAIL, false).await?;
-    let msg = t.get_last_msg().await;
+    let msg = load_imf_email(&t, GRP_MAIL).await;
+
     let chat = chat::Chat::load_from_db(&t, msg.chat_id).await?;
     assert!(!chat.is_contact_request());
     Ok(())
@@ -3556,9 +3558,9 @@ async fn test_messed_up_message_id() -> Result<()> {
     let t = TestContext::new_bob().await;
 
     let raw = include_bytes!("../../test-data/message/messed_up_message_id.eml");
-    receive_imf(&t, raw, false).await?;
+    let msg = load_imf_email(&t, raw).await;
     assert_eq!(
-        t.get_last_msg().await.rfc724_mid,
+        msg.rfc724_mid,
         "0bb9ffe1-2596-d997-95b4-1fef8cc4808e@example.org"
     );
 

src/sql.rs

@@ -883,6 +883,24 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
+    // Cleanup `imap` and `imap_sync` entries for deleted transports.
+    //
+    // Transports may be deleted directly or via sync messages,
+    // so it is easier to cleanup orphaned entries in a single place.
+    context
+        .sql
+        .execute(
+            "DELETE FROM imap WHERE transport_id NOT IN (SELECT transports.id FROM transports)",
+            (),
+        )
+        .await
+        .log_err(context)
+        .ok();
+    context.sql.execute(
+        "DELETE FROM imap_sync WHERE transport_id NOT IN (SELECT transports.id FROM transports)",
+        (),
+    ).await.log_err(context).ok();
+
     // Delete POI locations
     // which don't have corresponding message.
     delete_orphaned_poi_locations(context)

src/stats.rs

@@ -554,7 +554,6 @@ async fn get_message_stats(context: &Context) -> Result<BTreeMap<Chattype, Messa
 }
 
 pub(crate) async fn update_message_stats(context: &Context) -> Result<()> {
-    info!(context, "Updating message statistics.");
     for chattype in [Chattype::Single, Chattype::Group, Chattype::OutBroadcast] {
         update_message_stats_inner(context, chattype).await?;
     }

src/test_utils.rs

@@ -715,7 +715,8 @@ impl TestContext {
     }
 
     pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec<SentMessage<'a>> {
-        self.ctx
+        let sent_msgs = self
+            .ctx
             .sql
             .query_map_vec(
                 "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?",
@@ -737,7 +738,23 @@ impl TestContext {
                 sender_context: &self.ctx,
                 recipients,
             })
-            .collect()
+            .collect();
+        self.ctx
+            .sql
+            .execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))
+            .await
+            .expect("Delete smtp jobs");
+        update_msg_state(&self.ctx, msg_id, MessageState::OutDelivered)
+            .await
+            .expect("Update message state");
+        self.sql
+            .execute(
+                "UPDATE msgs SET timestamp_sent=? WHERE id=?",
+                (time(), msg_id),
+            )
+            .await
+            .expect("Update timestamp_sent");
+        sent_msgs
     }
 
     /// Parses a message.

src/tests/pre_messages/sending.rs

@@ -56,23 +56,27 @@ async fn test_sending_pre_message() -> Result<()> {
             .is_some()
     );
 
+    let post_rfc724_mid = post_message_parsed
+        .headers
+        .get_header_value(HeaderDef::MessageId);
     assert_eq!(
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
+        post_rfc724_mid,
         Some(format!("<{}>", msg.rfc724_mid)),
         "Post-Message should have the rfc message id of the database message"
     );
 
+    let pre_rfc724_mid = pre_message_parsed
+        .headers
+        .get_header_value(HeaderDef::MessageId);
     assert_ne!(
-        pre_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
+        pre_rfc724_mid, post_rfc724_mid,
         "message ids of Pre-Message and Post-Message should be different"
     );
+    assert_eq!(
+        pre_rfc724_mid,
+        Some(format!("<{}>", msg.pre_rfc724_mid)),
+        "Unexpected pre-message RFC 724 ID"
+    );
 
     let decrypted_post_message = bob.parse_msg(post_message).await;
     assert_eq!(decrypted_post_message.decrypting_failed, false);
@@ -86,9 +90,7 @@ async fn test_sending_pre_message() -> Result<()> {
         decrypted_pre_message
             .get_header(HeaderDef::ChatPostMessageId)
             .map(String::from),
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId)
+        post_rfc724_mid,
     );
     assert!(
         pre_message_parsed
@@ -98,6 +100,25 @@ async fn test_sending_pre_message() -> Result<()> {
         "no Chat-Post-Message-ID header in unprotected headers of Pre-Message"
     );
 
+    chat::resend_msgs(alice, &[msg_id]).await?;
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+    assert_eq!(smtp_rows.len(), 2);
+
+    let pre_message_parsed = mailparse::parse_mail(smtp_rows[0].payload.as_bytes())?;
+    assert_eq!(
+        pre_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        pre_rfc724_mid
+    );
+    let post_message_parsed = mailparse::parse_mail(smtp_rows[1].payload.as_bytes())?;
+    assert_eq!(
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        post_rfc724_mid
+    );
+
     Ok(())
 }
 

src/tests/verified_chats.rs

@@ -246,46 +246,6 @@ async fn test_old_message_4() -> Result<()> {
     Ok(())
 }
 
-/// Alice is offline for some time.
-/// When they come online, first their mvbox is synced and then their inbox.
-/// This test tests that the messages are still in the right order.
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_old_message_5() -> Result<()> {
-    let alice = TestContext::new_alice().await;
-    let msg_sent = receive_imf(
-        &alice,
-        b"From: alice@example.org\n\
-          To: Bob <bob@example.net>\n\
-          Message-ID: <1234-2-4@example.org>\n\
-          Date: Sat, 07 Dec 2019 19:00:27 +0000\n\
-          \n\
-          Happy birthday, Bob!\n",
-        true,
-    )
-    .await?
-    .unwrap();
-
-    let msg_incoming = receive_imf(
-        &alice,
-        b"From: Bob <bob@example.net>\n\
-          To: alice@example.org\n\
-          Message-ID: <1234-2-3@example.org>\n\
-          Date: Sun, 07 Dec 2019 19:00:26 +0000\n\
-          \n\
-          Happy birthday to me, Alice!\n",
-        false,
-    )
-    .await?
-    .unwrap();
-
-    assert!(msg_sent.sort_timestamp == msg_incoming.sort_timestamp);
-    alice
-        .golden_test_chat(msg_sent.chat_id, "test_old_message_5")
-        .await;
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_mdn_doesnt_disable_verification() -> Result<()> {
     let mut tcm = TestContextManager::new();

src/tools.rs

@@ -21,7 +21,6 @@ pub use std::time::SystemTime as Time;
 #[cfg(not(test))]
 pub use std::time::SystemTime;
 
-use crate::log::LogExt as _;
 use anyhow::{Context as _, Result, bail, ensure};
 use base64::Engine as _;
 use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
@@ -249,8 +248,6 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
                 true,
             )
             .await
-            .context("Failed to add bad time warning")
-            .log_err(context)
             .ok();
         } else {
             warn!(context, "Can't convert current timestamp");

src/tools/tools_tests.rs

@@ -64,9 +64,11 @@ DKIM Results: Passed=true";
 
 async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
     let t = TestContext::new_alice().await;
-    receive_imf(&t, raw, false).await.unwrap();
-    let msg = t.get_last_msg().await;
-    let msg_info = msg.id.get_info(&t).await.unwrap();
+    let received = receive_imf(&t, raw, false).await.unwrap().unwrap();
+
+    assert_eq!(received.msg_ids.len(), 1);
+    let msg_id = received.msg_ids[0];
+    let msg_info = msg_id.get_info(&t).await.unwrap();
 
     // Ignore the first rows of the msg_info because they contain a
     // received time that depends on the test time which makes it impossible to

test-data/golden/receive_imf_delayed_removal_is_ignored

@@ -2,9 +2,9 @@ Group#Chat#1001: Group [5 member(s)]
 --------------------------------------------------------------------------------
 Msg#1001: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO]
 Msg#1002🔒: Me (Contact#Contact#Self): populate  √
+Msg#1007🔒:  (Contact#Contact#1001): Member fiona@example.net removed by bob@example.net. [FRESH][INFO]
 Msg#1003: info (Contact#Contact#Info): Member dom@example.net added. [NOTICED][INFO]
 Msg#1004: info (Contact#Contact#Info): Member fiona@example.net removed. [NOTICED][INFO]
 Msg#1005🔒:  (Contact#Contact#1001): Member elena@example.net added by bob@example.net. [FRESH][INFO]
 Msg#1006🔒: Me (Contact#Contact#Self): You added member fiona@example.net. [INFO] o
-Msg#1007🔒:  (Contact#Contact#1001): Member fiona@example.net removed by bob@example.net. [FRESH][INFO]
 --------------------------------------------------------------------------------

test-data/golden/receive_imf_older_message_from_2nd_device

@@ -1,5 +1,5 @@
 Single#Chat#1001: bob@example.net [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png
 --------------------------------------------------------------------------------
-Msg#1001: Me (Contact#Contact#Self): We share this account  √
 Msg#1002: Me (Contact#Contact#Self): I'm Alice too  √
+Msg#1001: Me (Contact#Contact#Self): We share this account  √
 --------------------------------------------------------------------------------

test-data/golden/test_old_message_5

@@ -1,5 +0,0 @@
-Single#Chat#11001: Bob [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png
---------------------------------------------------------------------------------
-Msg#11001: Me (Contact#Contact#Self): Happy birthday, Bob!  √
-Msg#11002:  (Contact#Contact#11001): Happy birthday to me, Alice! [FRESH]
---------------------------------------------------------------------------------

test-data/golden/test_sync_broadcast_alice1

@@ -1,7 +1,7 @@
 OutBroadcast#Chat#1001: Channel [0 member(s)]
 --------------------------------------------------------------------------------
 Msg#1001: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO]
-Msg#1006🔒: Me (Contact#Contact#Self): Member bob@example.net added. [INFO] √
 Msg#1008🔒: Me (Contact#Contact#Self): hi  √
+Msg#1006🔒: Me (Contact#Contact#Self): Member bob@example.net added. [INFO] √
 Msg#1009🔒: Me (Contact#Contact#Self): You removed member bob@example.net. [INFO] √
 --------------------------------------------------------------------------------