chore(release): prepare for 2.48.0

This is to avoid sorting incoming messages that
are slightly in the past above system messages
about SecureJoin. SecureJoin messages are
timed according to smeared timestamp,
so even in the local tests they are in the future
by a few seconds.

.github/workflows/dependabot.yml

@@ -10,7 +10,7 @@ permissions:
 jobs:
   dependabot:
     runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
+    if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == github.event.pull_request.head.repo.full_name
     steps:
       - name: Dependabot metadata
         id: metadata

.github/workflows/upload-docs.yml

@@ -1,16 +1,18 @@
-name: Build & deploy documentation on rs.delta.chat, c.delta.chat, and py.delta.chat
+name: Build & deploy documentation on rs.delta.chat, c.delta.chat, py.delta.chat and cffi.delta.chat
 
 on:
   push:
     branches:
       - main
-      - build_jsonrpc_docs_ci
 
 permissions: {}
 
 jobs:
   build-rs:
     runs-on: ubuntu-latest
+    environment:
+      name: rs.delta.chat
+      url: https://rs.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -23,12 +25,15 @@ jobs:
       - name: Upload to rs.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.RS_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.USERNAME }}@rs.delta.chat:/var/www/html/rs/"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.RS_DOCS_SSH_USER }}@rs.delta.chat:/var/www/html/rs.delta.chat/"
 
   build-python:
     runs-on: ubuntu-latest
+    environment:
+      name: py.delta.chat
+      url: https://py.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -42,12 +47,15 @@ jobs:
       - name: Upload to py.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.PY_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "${{ secrets.PY_DOCS_SSH_USER }}@py.delta.chat:/var/www/html/py.delta.chat"
 
   build-c:
     runs-on: ubuntu-latest
+    environment:
+      name: c.delta.chat
+      url: https://c.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -61,12 +69,16 @@ jobs:
       - name: Upload to c.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.C_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "${{ secrets.C_DOCS_SSH_USER }}@c.delta.chat:/var/www/html/c.delta.chat"
 
   build-ts:
     runs-on: ubuntu-latest
+    environment:
+      name: js.jsonrpc.delta.chat
+      url: https://js.jsonrpc.delta.chat/
+
     defaults:
       run:
         working-directory: ./deltachat-jsonrpc/typescript
@@ -90,6 +102,27 @@ jobs:
       - name: Upload to js.jsonrpc.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.JS_JSONRPC_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.USERNAME }}@js.jsonrpc.delta.chat:/var/www/html/js-jsonrpc/"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.JS_JSONRPC_DOCS_SSH_USER }}@js.jsonrpc.delta.chat:/var/www/html/js.jsonrpc.delta.chat/"
+
+  build-cffi:
+    runs-on: ubuntu-latest
+    environment:
+      name: cffi.delta.chat
+      url: https://cffi.delta.chat/
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          show-progress: false
+          persist-credentials: false
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat_ffi --no-deps
+      - name: Upload to cffi.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CFFI_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.CFFI_DOCS_SSH_USER }}@delta.chat:/var/www/html/cffi.delta.chat/"

.github/workflows/upload-ffi-docs.yml

@@ -1,31 +0,0 @@
-# GitHub Actions workflow
-# to build `deltachat_ffi` crate documentation
-# and upload it to <https://cffi.delta.chat/>
-
-name: Build & Deploy Documentation on cffi.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-permissions: {}
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          show-progress: false
-          persist-credentials: false
-      - name: Build the documentation with cargo
-        run: |
-          cargo doc --package deltachat_ffi --no-deps
-      - name: Upload to cffi.delta.chat
-        run: |
-          mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
-          chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"

CHANGELOG.md

@@ -1,5 +1,61 @@
 # Changelog
 
+## [2.48.0] - 2026-03-30
+
+### Fixes
+
+- Fix reordering problems in multi-relay setups by not sorting received messages below the last seen one.
+- Always sort "Messages are end-to-end encrypted" notice to the beginning.
+- Make Message-ID of pre-messages stable across resends ([#8007](https://github.com/chatmail/core/pull/8007)).
+- Delete `imap_markseen` entries not corresponding to any `imap` rows.
+- Cleanup `imap` and `imap_sync` records without transport in housekeeping.
+- When receiving MDN, mark all preceding messages as noticed, even having same timestamp ([#7928](https://github.com/chatmail/core/pull/7928)).
+- Remove migration 108 preventing upgrades from core 1.86.0 to the latest version.
+
+### Features / Changes
+
+- Improve IMAP loop logs.
+- Add decryption error to the device message about outgoing message decryption failure.
+- Log received message sort timestamp.
+
+### Performance
+
+- Move sorting outside of SQL query in `store_seen_flags_on_imap`.
+
+### API-Changes
+
+- Add JSON-RPC API `markfresh_chat()`.
+- ffi: Correctly declare `dc_event_channel_new()` as having no params ([#7831](https://github.com/chatmail/core/pull/7831)).
+
+### Refactor
+
+- Remove `wal_checkpoint_mutex`, lock `write_mutex` before getting sql connection instead.
+- Replace async `RwLock` with sync `RwLock` for stock strings.
+- Cleanup remaining Autocrypt Setup Message processing in `mimeparser`.
+- SecureJoin: do not check for self address in forwarding protection.
+- Fix clippy warnings.
+
+### CI
+
+- Update {c,py}.delta.chat website deployments.
+- Use environments for {rs,cffi,js.jsonrpc}.delta.chat deployments.
+- Fix https://docs.zizmor.sh/audits/#bot-conditions.
+
+### Documentation
+
+- Add SQL performance tips to STYLE.md.
+
+### Tests
+
+- Remove `test_old_message_5`.
+- Do not rely on loading newest chat in `load_imf_email()`.
+- Use `load_imf_email()` more.
+- The message is sorted correctly in the chat even if it arrives late.
+
+### Miscellaneous Tasks
+
+- cargo: update rustls-webpki to 0.103.10.
+
 ## [2.47.0] - 2026-03-24
 
 ### Fixes
@@ -7984,3 +8040,4 @@ https://github.com/chatmail/core/pulls?q=is%3Apr+is%3Aclosed
 [2.45.0]: https://github.com/chatmail/core/compare/v2.44.0..v2.45.0
 [2.46.0]: https://github.com/chatmail/core/compare/v2.45.0..v2.46.0
 [2.47.0]: https://github.com/chatmail/core/compare/v2.46.0..v2.47.0
+[2.48.0]: https://github.com/chatmail/core/compare/v2.47.0..v2.48.0

Cargo.lock

@@ -1307,7 +1307,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "2.48.0-dev"
+version = "2.48.0"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -1416,7 +1416,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "2.48.0-dev"
+version = "2.48.0"
 dependencies = [
  "anyhow",
  "async-channel 2.5.0",
@@ -1437,7 +1437,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "2.48.0-dev"
+version = "2.48.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1453,7 +1453,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "2.48.0-dev"
+version = "2.48.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1482,7 +1482,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "2.48.0-dev"
+version = "2.48.0"
 dependencies = [
  "anyhow",
  "deltachat",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "2.48.0-dev"
+version = "2.48.0"
 edition = "2024"
 license = "MPL-2.0"
 rust-version = "1.88"

STYLE.md

@@ -68,6 +68,12 @@ keyword doesn't help here.
 Consider adding context to `anyhow` errors for SQL statements using `.context()` so that it's
 possible to understand from logs which statement failed. See [Errors](#errors) for more info.
 
+When changing complex SQL queries, test them on a new database with `EXPLAIN QUERY PLAN`
+to make sure that indexes are used and large tables are not going to be scanned.
+Never run `ANALYZE` on the databases,
+this makes query planner unpredictable
+and may make performance significantly worse: <https://github.com/chatmail/core/issues/6585>
+
 ## Errors
 
 Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "2.48.0-dev"
+version = "2.48.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "2.48.0-dev"
+version = "2.48.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 license = "MPL-2.0"

deltachat-jsonrpc/typescript/package.json

@@ -54,5 +54,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "2.48.0-dev"
+  "version": "2.48.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "2.48.0-dev"
+version = "2.48.0"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/chatmail/core"

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "2.48.0-dev"
+version = "2.48.0"
 license = "MPL-2.0"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "2.48.0-dev"
+version = "2.48.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "2.48.0-dev"
+  "version": "2.48.0"
 }

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "2.48.0-dev"
+version = "2.48.0"
 license = "MPL-2.0"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"

release-date.in

@@ -1 +1 @@
-2026-03-24
+2026-03-30

src/chat.rs

@@ -477,12 +477,17 @@ impl ChatId {
     /// Adds message "Messages are end-to-end encrypted".
     pub(crate) async fn add_e2ee_notice(self, context: &Context, timestamp: i64) -> Result<()> {
         let text = stock_str::messages_e2ee_info_msg(context);
+
+        // Sort this notice to the very beginning of the chat.
+        // We don't want any message to appear before this notice
+        // which is normally added when encrypted chat is created.
+        let sort_timestamp = 0;
         add_info_msg_with_cmd(
             context,
             self,
             &text,
             SystemMessage::ChatE2ee,
-            Some(timestamp),
+            Some(sort_timestamp),
             timestamp,
             None,
             None,
@@ -925,6 +930,17 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
             .unwrap_or(0))
     }
 
+    /// Returns timestamp of us joining the chat if we are the member of the chat.
+    pub(crate) async fn join_timestamp(self, context: &Context) -> Result<Option<i64>> {
+        context
+            .sql
+            .query_get_value(
+                "SELECT add_timestamp FROM chats_contacts WHERE chat_id=? AND contact_id=?",
+                (self, ContactId::SELF),
+            )
+            .await
+    }
+
     /// Returns timestamp of the latest message in the chat,
     /// including hidden messages or a draft if there is one.
     pub(crate) async fn get_timestamp(self, context: &Context) -> Result<Option<i64>> {
@@ -1212,15 +1228,11 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
     /// corresponding event in case of a system message (usually the current system time).
     /// `always_sort_to_bottom` makes this adjust the returned timestamp up so that the message goes
     /// to the chat bottom.
-    /// `received` -- whether the message is received. Otherwise being sent.
-    /// `incoming` -- whether the message is incoming.
     pub(crate) async fn calc_sort_timestamp(
         self,
         context: &Context,
         message_timestamp: i64,
         always_sort_to_bottom: bool,
-        received: bool,
-        incoming: bool,
     ) -> Result<i64> {
         let mut sort_timestamp = cmp::min(message_timestamp, smeared_time(context));
 
@@ -1240,38 +1252,6 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
                     (self, MessageState::OutDraft),
                 )
                 .await?
-        } else if received {
-            // Received messages shouldn't mingle with just sent ones and appear somewhere in the
-            // middle of the chat, so we go after the newest non fresh message.
-            //
-            // But if a received outgoing message is older than some seen message, better sort the
-            // received message purely by timestamp. We could place it just before that seen
-            // message, but anyway the user may not notice it.
-            //
-            // NB: Received outgoing messages may break sorting of fresh incoming ones, but this
-            // shouldn't happen frequently. Seen incoming messages don't really break sorting of
-            // fresh ones, they rather mean that older incoming messages are actually seen as well.
-            context
-                .sql
-                .query_row_optional(
-                    "SELECT MAX(timestamp), MAX(IIF(state=?,timestamp_sent,0))
-                     FROM msgs
-                     WHERE chat_id=? AND hidden=0 AND state>?
-                     HAVING COUNT(*) > 0",
-                    (MessageState::InSeen, self, MessageState::InFresh),
-                    |row| {
-                        let ts: i64 = row.get(0)?;
-                        let ts_sent_seen: i64 = row.get(1)?;
-                        Ok((ts, ts_sent_seen))
-                    },
-                )
-                .await?
-                .and_then(|(ts, ts_sent_seen)| {
-                    match incoming || ts_sent_seen <= message_timestamp {
-                        true => Some(ts),
-                        false => None,
-                    }
-                })
         } else {
             None
         };
@@ -1282,7 +1262,16 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
             sort_timestamp = last_msg_time;
         }
 
-        Ok(sort_timestamp)
+        if let Some(join_timestamp) = self.join_timestamp(context).await? {
+            // If we are the member of the chat, don't add messages
+            // before the timestamp of us joining it.
+            // This is needed to avoid sorting "Member added"
+            // or automatically sent bot welcome messages
+            // above SecureJoin system messages.
+            Ok(std::cmp::max(sort_timestamp, join_timestamp))
+        } else {
+            Ok(sort_timestamp)
+        }
     }
 }
 
@@ -4938,15 +4927,8 @@ pub(crate) async fn add_info_msg_with_cmd(
         ts
     } else {
         let sort_to_bottom = true;
-        let (received, incoming) = (false, false);
         chat_id
-            .calc_sort_timestamp(
-                context,
-                smeared_time(context),
-                sort_to_bottom,
-                received,
-                incoming,
-            )
+            .calc_sort_timestamp(context, smeared_time(context), sort_to_bottom)
             .await?
     };
 

src/chat/chat_tests.rs

@@ -2792,7 +2792,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&vc_pubkey).await;
-        assert!(parsed_by_bob.decrypting_failed);
+        assert!(parsed_by_bob.decryption_error.is_some());
 
         charlie.recv_msg_trash(&vc_pubkey).await;
     }
@@ -2821,7 +2821,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&member_added).await;
-        assert!(parsed_by_bob.decrypting_failed);
+        assert!(parsed_by_bob.decryption_error.is_some());
 
         let rcvd = charlie.recv_msg(&member_added).await;
         assert_eq!(rcvd.param.get_cmd(), SystemMessage::MemberAddedToGroup);
@@ -2836,7 +2836,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&hi_msg).await;
-        assert_eq!(parsed_by_bob.decrypting_failed, false);
+        assert!(parsed_by_bob.decryption_error.is_none());
     }
 
     tcm.section("Alice removes Charlie. Bob must not see it.");
@@ -2853,7 +2853,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&member_removed).await;
-        assert!(parsed_by_bob.decrypting_failed);
+        assert!(parsed_by_bob.decryption_error.is_some());
 
         let rcvd = charlie.recv_msg(&member_removed).await;
         assert_eq!(rcvd.param.get_cmd(), SystemMessage::MemberRemovedFromGroup);
@@ -6112,3 +6112,54 @@ async fn test_leftgrps() -> Result<()> {
 
     Ok(())
 }
+
+/// Tests that if the message arrives late,
+/// it can still be sorted above the last seen message.
+///
+/// Versions 2.47 and below always sorted incoming messages
+/// after the last seen message, but with
+/// the introduction of multi-relay it is possible
+/// that some messages arrive only to some relays
+/// and are fetched after the already arrived seen message.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_late_message_above_seen() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let charlie = &tcm.charlie().await;
+
+    let alice_chat_id = alice
+        .create_group_with_members("Group", &[bob, charlie])
+        .await;
+    let alice_sent = alice.send_text(alice_chat_id, "Hello everyone!").await;
+    let bob_chat_id = bob.recv_msg(&alice_sent).await.chat_id;
+    bob_chat_id.accept(bob).await?;
+    let charlie_chat_id = charlie.recv_msg(&alice_sent).await.chat_id;
+    charlie_chat_id.accept(charlie).await?;
+
+    // Bob sends a message, but the message is delayed.
+    let bob_sent = bob.send_text(bob_chat_id, "Hello from Bob!").await;
+    SystemTime::shift(Duration::from_secs(1000));
+
+    let charlie_sent = charlie
+        .send_text(charlie_chat_id, "Hello from Charlie!")
+        .await;
+
+    // Alice immediately receives a message from Charlie and reads it.
+    let alice_received_from_charlie = alice.recv_msg(&charlie_sent).await;
+    assert_eq!(
+        alice_received_from_charlie.get_text(),
+        "Hello from Charlie!"
+    );
+    message::markseen_msgs(alice, vec![alice_received_from_charlie.id]).await?;
+
+    // Bob message arrives later, it should be above the message from Charlie.
+    let alice_received_from_bob = alice.recv_msg(&bob_sent).await;
+    assert_eq!(alice_received_from_bob.get_text(), "Hello from Bob!");
+
+    // The last message in the chat is still from Charlie.
+    let last_msg = alice.get_last_msg_in(alice_chat_id).await;
+    assert_eq!(last_msg.get_text(), "Hello from Charlie!");
+
+    Ok(())
+}

src/chatlist.rs

@@ -142,7 +142,7 @@ impl Chatlist {
                    AND c.blocked!=1
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2 AND add_timestamp >= remove_timestamp)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY c.archived=?3 DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                 (MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned),
                 process_row,
             ).await?
@@ -168,7 +168,7 @@ impl Chatlist {
                    AND c.blocked!=1
                    AND c.archived=1
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft,),
                     process_row,
                 )
@@ -204,7 +204,7 @@ impl Chatlist {
                    AND IFNULL(c.name_normalized,c.name) LIKE ?3
                    AND (NOT ?4 OR EXISTS (SELECT 1 FROM msgs m WHERE m.chat_id = c.id AND m.state == ?5 AND hidden=0))
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, str_like_cmd, only_unread, MessageState::InFresh),
                     process_row,
                 )
@@ -253,7 +253,7 @@ impl Chatlist {
                        AND NOT c.archived=?
                        AND (c.type!=? OR c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=? AND add_timestamp >= remove_timestamp))
                      GROUP BY c.id
-                     ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                     ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (
                         MessageState::OutDraft, skip_id, ChatVisibility::Archived,
                         Chattype::Group, ContactId::SELF,
@@ -279,7 +279,7 @@ impl Chatlist {
                        AND (c.blocked=0 OR c.blocked=2)
                        AND NOT c.archived=?
                      GROUP BY c.id
-                     ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                     ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, ChatVisibility::Archived, ChatVisibility::Pinned),
                     process_row,
                 ).await?

src/configure.rs

@@ -243,11 +243,6 @@ impl Context {
                         Ok((id, add_timestamp))
                     },
                 )?;
-                transaction.execute("DELETE FROM imap WHERE transport_id=?", (transport_id,))?;
-                transaction.execute(
-                    "DELETE FROM imap_sync WHERE transport_id=?",
-                    (transport_id,),
-                )?;
 
                 // Removal timestamp should not be lower than addition timestamp
                 // to be accepted by other devices when synced.

src/imap.rs

@@ -323,8 +323,7 @@ impl Imap {
         if !ratelimit_duration.is_zero() {
             warn!(
                 context,
-                "Transport {}: IMAP got rate limited, waiting for {} until can connect.",
-                self.transport_id,
+                "IMAP got rate limited, waiting for {} until can connect.",
                 duration_to_str(ratelimit_duration),
             );
             let interrupted = async {
@@ -336,16 +335,12 @@ impl Imap {
             if interrupted {
                 info!(
                     context,
-                    "Transport {}: Connecting to IMAP without waiting for ratelimit due to interrupt.",
-                    self.transport_id
+                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
                 );
             }
         }
 
-        info!(
-            context,
-            "Transport {}: Connecting to IMAP server.", self.transport_id
-        );
+        info!(context, "Connecting to IMAP server.");
         self.connectivity.set_connecting(context);
 
         self.conn_last_try = tools::Time::now();
@@ -360,10 +355,7 @@ impl Imap {
         let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
         let mut first_error = None;
         for lp in login_params {
-            info!(
-                context,
-                "Transport {}: IMAP trying to connect to {}.", self.transport_id, &lp.connection
-            );
+            info!(context, "IMAP trying to connect to {}.", &lp.connection);
             let connection_candidate = lp.connection.clone();
             let client = match Client::connect(
                 context,
@@ -411,10 +403,7 @@ impl Imap {
                     let resync_request_sender = self.resync_request_sender.clone();
 
                     let session = if capabilities.can_compress {
-                        info!(
-                            context,
-                            "Transport {}: Enabling IMAP compression.", self.transport_id
-                        );
+                        info!(context, "Enabling IMAP compression.");
                         let compressed_session = session
                             .compress(|s| {
                                 let session_stream: Box<dyn SessionStream> = Box::new(s);
@@ -447,10 +436,7 @@ impl Imap {
                         lp.user
                     )));
                     self.connectivity.set_preparing(context);
-                    info!(
-                        context,
-                        "Transport {}: Successfully logged into IMAP server.", self.transport_id
-                    );
+                    info!(context, "Successfully logged into IMAP server.");
                     return Ok(session);
                 }
 
@@ -458,10 +444,7 @@ impl Imap {
                     let imap_user = lp.user.to_owned();
                     let message = stock_str::cannot_login(context, &imap_user);
 
-                    warn!(
-                        context,
-                        "Transport {}: IMAP failed to login: {err:#}.", self.transport_id
-                    );
+                    warn!(context, "IMAP failed to login: {err:#}.");
                     first_error.get_or_insert(format_err!("{message} ({err:#})"));
 
                     // If it looks like the password is wrong, send a notification:
@@ -480,11 +463,7 @@ impl Imap {
                             )
                             .await
                             {
-                                warn!(
-                                    context,
-                                    "Transport {}: Failed to add device message: {e:#}.",
-                                    self.transport_id
-                                );
+                                warn!(context, "Failed to add device message: {e:#}.");
                             } else {
                                 context
                                     .set_config_internal(Config::NotifyAboutWrongPw, None)
@@ -546,21 +525,10 @@ impl Imap {
             bail!("IMAP operation attempted while it is torn down");
         }
 
-        let transport_id = session.transport_id();
-        info!(
-            context,
-            "Transport {transport_id}: fetch_move_delete start."
-        );
-
         let msgs_fetched = self
             .fetch_new_messages(context, session, watch_folder, folder_meaning)
             .await
             .context("fetch_new_messages")?;
-
-        info!(
-            context,
-            "Transport {transport_id}: fetch_move_delete finished fetch_new_messages."
-        );
         if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
             // New messages were fetched and shall be deleted later, restart ephemeral loop.
             // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
@@ -599,18 +567,10 @@ impl Imap {
             return Ok(false);
         }
 
-        info!(
-            context,
-            "Transport {transport_id}: fetch_new_messages selects folder {folder:?}."
-        );
         let folder_exists = session
             .select_with_uidvalidity(context, folder)
             .await
             .with_context(|| format!("Failed to select folder {folder:?}"))?;
-        info!(
-            context,
-            "Transport {transport_id}: fetch_new_messages selected folder {folder:?}."
-        );
 
         if !session.new_mail {
             info!(
@@ -1143,16 +1103,22 @@ impl Session {
             return Ok(());
         }
 
+        context
+            .sql
+            .execute(
+                "DELETE FROM imap_markseen WHERE id NOT IN (SELECT imap.id FROM imap)",
+                (),
+            )
+            .await?;
+
         let transport_id = self.transport_id();
-        info!(context, "Transport {transport_id}: Storing seen flags.");
-        let rows = context
+        let mut rows = context
             .sql
             .query_map_vec(
                 "SELECT imap.id, uid, folder FROM imap, imap_markseen
                  WHERE imap.id = imap_markseen.id
                  AND imap.transport_id=?
-                 AND target = folder
-                 ORDER BY folder, uid",
+                 AND target = folder",
                 (transport_id,),
                 |row| {
                     let rowid: i64 = row.get(0)?;
@@ -1163,6 +1129,16 @@ impl Session {
             )
             .await?;
 
+        // Number of SQL results is expected to be low as
+        // we usually don't have many messages to mark on IMAP at once.
+        // We are sorting outside of SQL to avoid SQLite constructing a query plan
+        // that scans the whole `imap` table. Scanning `imap_markseen` is fine
+        // as it should not have many items.
+        // If you change the SQL query, test it with `EXPLAIN QUERY PLAN`.
+        rows.sort_unstable_by(|(_rowid1, uid1, folder1), (_rowid2, uid2, folder2)| {
+            (folder1, uid1).cmp(&(folder2, uid2))
+        });
+
         for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
             let folder_exists = match self.select_with_uidvalidity(context, &folder).await {
                 Err(err) => {
@@ -1179,15 +1155,13 @@ impl Session {
             } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
                 warn!(
                     context,
-                    "Transport {transport_id}: Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
+                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
                 );
                 continue;
             } else {
                 info!(
                     context,
-                    "Transport {transport_id}: Marked messages {} in folder {} as seen.",
-                    uid_set,
-                    folder
+                    "Marked messages {} in folder {} as seen.", uid_set, folder
                 );
             }
             context
@@ -1202,10 +1176,6 @@ impl Session {
                 .await
                 .context("Cannot remove messages marked as seen from imap_markseen table")?;
         }
-        info!(
-            context,
-            "Transport {transport_id}: Finished storing seen flags."
-        );
 
         Ok(())
     }
@@ -1546,10 +1516,9 @@ impl Session {
             return Ok(());
         }
 
-        let transport_id = self.transport_id();
         info!(
             context,
-            "Transport {transport_id}: Server supports metadata, retrieving server comment and admin contact."
+            "Server supports metadata, retrieving server comment and admin contact."
         );
 
         let mut comment = None;
@@ -1582,8 +1551,7 @@ impl Session {
                         } else {
                             warn!(
                                 context,
-                                "Transport {transport_id}: Got invalid URL from iroh relay metadata: {:?}.",
-                                value
+                                "Got invalid URL from iroh relay metadata: {:?}.", value
                             );
                         }
                     }
@@ -1612,7 +1580,6 @@ impl Session {
             create_fallback_ice_servers()
         };
 
-        info!(context, "Transport {transport_id}: Got IMAP metadata.");
         *lock = Some(ServerMetadata {
             comment,
             admin,

src/imex.rs

@@ -1137,4 +1137,16 @@ mod tests {
 
         Ok(())
     }
+
+    /// Tests importing a backup from Delta Chat 1.30.3 for Android (core v1.86.0).
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_import_ancient_backup() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let context = &tcm.unconfigured().await;
+
+        let backup_path = Path::new("test-data/core-1.86.0-backup.tar");
+        imex(context, ImexMode::ImportBackup, backup_path, None).await?;
+
+        Ok(())
+    }
 }

src/message.rs

@@ -815,7 +815,11 @@ impl Message {
 
     /// Returns the timestamp of the message for sorting.
     pub fn get_sort_timestamp(&self) -> i64 {
-        self.timestamp_sort
+        if self.timestamp_sort != 0 {
+            self.timestamp_sort
+        } else {
+            self.timestamp_sent
+        }
     }
 
     /// Returns the text of the message.

src/mimefactory.rs

@@ -852,7 +852,13 @@ impl MimeFactory {
 
         let rfc724_mid = match &self.loaded {
             Loaded::Message { msg, .. } => match &self.pre_message_mode {
-                PreMessageMode::Pre { .. } => create_outgoing_rfc724_mid(),
+                PreMessageMode::Pre { .. } => {
+                    if msg.pre_rfc724_mid.is_empty() {
+                        create_outgoing_rfc724_mid()
+                    } else {
+                        msg.pre_rfc724_mid.clone()
+                    }
+                }
                 _ => msg.rfc724_mid.clone(),
             },
             Loaded::Mdn { .. } => create_outgoing_rfc724_mid(),

src/mimeparser.rs

@@ -86,7 +86,9 @@ pub(crate) struct MimeMessage {
     /// messages to this address to post them to the list.
     pub list_post: Option<String>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
-    pub decrypting_failed: bool,
+
+    /// Decryption error if decryption of the message has failed.
+    pub decryption_error: Option<String>,
 
     /// Valid signature fingerprint if a message is an
     /// Autocrypt encrypted and signed message and corresponding intended recipient fingerprints
@@ -664,7 +666,7 @@ impl MimeMessage {
             from,
             incoming,
             chat_disposition_notification_to,
-            decrypting_failed: mail.is_err(),
+            decryption_error: mail.err().map(|err| format!("{err:#}")),
 
             // only non-empty if it was a valid autocrypt message
             signature,
@@ -905,7 +907,7 @@ impl MimeMessage {
             && let Some(ref subject) = self.get_subject()
         {
             let mut prepend_subject = true;
-            if !self.decrypting_failed {
+            if self.decryption_error.is_none() {
                 let colon = subject.find(':');
                 if colon == Some(2)
                     || colon == Some(3)
@@ -946,7 +948,7 @@ impl MimeMessage {
         self.parse_attachments();
 
         // See if an MDN is requested from the other side
-        if !self.decrypting_failed
+        if self.decryption_error.is_none()
             && !self.parts.is_empty()
             && let Some(ref dn_to) = self.chat_disposition_notification_to
         {
@@ -1078,7 +1080,7 @@ impl MimeMessage {
     #[cfg(test)]
     /// Returns whether the decrypted data contains the given `&str`.
     pub(crate) fn decoded_data_contains(&self, s: &str) -> bool {
-        assert!(!self.decrypting_failed);
+        assert!(self.decryption_error.is_none());
         let decoded_str = str::from_utf8(&self.decoded_data).unwrap();
         decoded_str.contains(s)
     }

src/receive_imf.rs

@@ -727,7 +727,7 @@ pub(crate) async fn receive_imf_inner(
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
             .unwrap_or_default();
 
-        let allow_creation = if mime_parser.decrypting_failed {
+        let allow_creation = if mime_parser.decryption_error.is_some() {
             false
         } else if is_dc_message == MessengerMessage::No
             && !context.get_config_bool(Config::IsChatmail).await?
@@ -1010,11 +1010,11 @@ UPDATE config SET value=? WHERE keyname='configured_addr' AND value!=?1
                         && msg.chat_visibility == ChatVisibility::Archived;
                     updated_chats
                         .entry(msg.chat_id)
-                        .and_modify(|ts| *ts = cmp::max(*ts, msg.timestamp_sort))
-                        .or_insert(msg.timestamp_sort);
+                        .and_modify(|pos| *pos = cmp::max(*pos, (msg.timestamp_sort, msg.id)))
+                        .or_insert((msg.timestamp_sort, msg.id));
                 }
             }
-            for (chat_id, timestamp_sort) in updated_chats {
+            for (chat_id, (timestamp_sort, msg_id)) in updated_chats {
                 context
                     .sql
                     .execute(
@@ -1023,12 +1023,13 @@ UPDATE msgs SET state=? WHERE
     state=? AND
     hidden=0 AND
     chat_id=? AND
-    timestamp<?",
+    (timestamp,id)<(?,?)",
                         (
                             MessageState::InNoticed,
                             MessageState::InFresh,
                             chat_id,
                             timestamp_sort,
+                            msg_id,
                         ),
                     )
                     .await
@@ -1211,14 +1212,18 @@ async fn decide_chat_assignment(
     {
         info!(context, "Call state changed (TRASH).");
         true
-    } else if mime_parser.decrypting_failed && !mime_parser.incoming {
+    } else if let Some(ref decryption_error) = mime_parser.decryption_error
+        && !mime_parser.incoming
+    {
         // Outgoing undecryptable message.
         let last_time = context
             .get_config_i64(Config::LastCantDecryptOutgoingMsgs)
             .await?;
         let now = tools::time();
         let update_config = if last_time.saturating_add(24 * 60 * 60) <= now {
-            let txt = "⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions.";
+            let txt = format!(
+                "⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions. (Error: {decryption_error}, {rfc724_mid})."
+            );
             let mut msg = Message::new_text(txt.to_string());
             chat::add_device_msg(context, None, Some(&mut msg))
                 .await
@@ -1835,6 +1840,16 @@ async fn add_parts(
         }
     }
 
+    // Sort message to the bottom if we are not in the chat
+    // so if we are added via QR code scan
+    // the message about our addition goes after all the info messages.
+    // Info messages are sorted by local smeared_timestamp()
+    // which advances quickly during SecureJoin,
+    // while "member added" message may have older timestamp
+    // corresponding to the sender clock.
+    // In practice inviter clock may even be slightly in the past.
+    let sort_to_bottom = !chat.is_self_in_chat(context).await?;
+
     let is_location_kml = mime_parser.location_kml.is_some();
     let mut group_changes = match chat.typ {
         _ if chat.id.is_special() => GroupChangesInfo::default(),
@@ -1885,16 +1900,8 @@ async fn add_parts(
     };
     let in_fresh = state == MessageState::InFresh;
 
-    let sort_to_bottom = false;
-    let received = true;
     let sort_timestamp = chat_id
-        .calc_sort_timestamp(
-            context,
-            mime_parser.timestamp_sent,
-            sort_to_bottom,
-            received,
-            mime_parser.incoming,
-        )
+        .calc_sort_timestamp(context, mime_parser.timestamp_sent, sort_to_bottom)
         .await?;
 
     // Apply ephemeral timer changes to the chat.
@@ -2290,7 +2297,7 @@ RETURNING id
                     if trash { 0 } else { ephemeral_timestamp },
                     if trash {
                         DownloadState::Done
-                    } else if mime_parser.decrypting_failed {
+                    } else if mime_parser.decryption_error.is_some() {
                         DownloadState::Undecipherable
                     } else if let PreMessageMode::Pre {..} = mime_parser.pre_message {
                         DownloadState::Available
@@ -2354,7 +2361,7 @@ RETURNING id
 
     info!(
         context,
-        "Message has {icnt} parts and is assigned to chat #{chat_id}."
+        "Message has {icnt} parts and is assigned to chat #{chat_id}, timestamp={sort_timestamp}."
     );
 
     if !chat_id.is_trash() && !hidden {
@@ -2703,7 +2710,7 @@ async fn lookup_or_create_adhoc_group(
     allow_creation: bool,
     create_blocked: Blocked,
 ) -> Result<Option<(ChatId, Blocked, bool)>> {
-    if mime_parser.decrypting_failed {
+    if mime_parser.decryption_error.is_some() {
         warn!(
             context,
             "Not creating ad-hoc group for message that cannot be decrypted."
@@ -2925,7 +2932,7 @@ async fn create_group(
 
     if let Some(chat_id) = chat_id {
         Ok(Some((chat_id, chat_id_blocked)))
-    } else if mime_parser.decrypting_failed {
+    } else if mime_parser.decryption_error.is_some() {
         // It is possible that the message was sent to a valid,
         // yet unknown group, which was rejected because
         // Chat-Group-Name, which is in the encrypted part, was

src/receive_imf/receive_imf_tests.rs

@@ -819,9 +819,12 @@ async fn load_imf_email(context: &Context, imf_raw: &[u8]) -> Message {
         .set_config(Config::ShowEmails, Some("2"))
         .await
         .unwrap();
-    receive_imf(context, imf_raw, false).await.unwrap();
-    let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
-    let msg_id = chats.get_msg_id(0).unwrap().unwrap();
+    let received_msg = receive_imf(context, imf_raw, false)
+        .await
+        .expect("receive_imf failure")
+        .expect("No message received");
+    assert_eq!(received_msg.msg_ids.len(), 1);
+    let msg_id = received_msg.msg_ids[0];
     Message::load_from_db(context, msg_id).await.unwrap()
 }
 
@@ -2872,9 +2875,8 @@ async fn test_rfc1847_encapsulation() -> Result<()> {
 
     // Alice sends a message to Bob using Thunderbird.
     let raw = include_bytes!("../../test-data/message/rfc1847_encapsulation.eml");
-    receive_imf(bob, raw, false).await?;
 
-    let msg = bob.get_last_msg().await;
+    let msg = load_imf_email(bob, raw).await;
     assert!(msg.get_showpadlock());
 
     Ok(())
@@ -3082,8 +3084,8 @@ async fn test_auto_accept_for_bots() -> Result<()> {
 async fn test_auto_accept_group_for_bots() -> Result<()> {
     let t = TestContext::new_alice().await;
     t.set_config(Config::Bot, Some("1")).await.unwrap();
-    receive_imf(&t, GRP_MAIL, false).await?;
-    let msg = t.get_last_msg().await;
+    let msg = load_imf_email(&t, GRP_MAIL).await;
+
     let chat = chat::Chat::load_from_db(&t, msg.chat_id).await?;
     assert!(!chat.is_contact_request());
     Ok(())
@@ -3327,7 +3329,7 @@ async fn test_outgoing_undecryptable() -> Result<()> {
     assert!(
         dev_msg
             .text
-            .contains("⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions.")
+            .starts_with("⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions. (Error:")
     );
 
     let raw = include_bytes!("../../test-data/message/thunderbird_encrypted_signed.eml");
@@ -3556,9 +3558,9 @@ async fn test_messed_up_message_id() -> Result<()> {
     let t = TestContext::new_bob().await;
 
     let raw = include_bytes!("../../test-data/message/messed_up_message_id.eml");
-    receive_imf(&t, raw, false).await?;
+    let msg = load_imf_email(&t, raw).await;
     assert_eq!(
-        t.get_last_msg().await.rfc724_mid,
+        msg.rfc724_mid,
         "0bb9ffe1-2596-d997-95b4-1fef8cc4808e@example.org"
     );
 

src/securejoin.rs

@@ -450,10 +450,8 @@ pub(crate) async fn handle_securejoin_handshake(
     ) {
         let mut self_found = false;
         let self_fingerprint = load_self_public_key(context).await?.dc_fingerprint();
-        for (addr, key) in &mime_message.gossiped_keys {
-            if key.public_key.dc_fingerprint() == self_fingerprint
-                && context.is_self_addr(addr).await?
-            {
+        for key in mime_message.gossiped_keys.values() {
+            if key.public_key.dc_fingerprint() == self_fingerprint {
                 self_found = true;
                 break;
             }

src/sql.rs

@@ -883,6 +883,24 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
+    // Cleanup `imap` and `imap_sync` entries for deleted transports.
+    //
+    // Transports may be deleted directly or via sync messages,
+    // so it is easier to cleanup orphaned entries in a single place.
+    context
+        .sql
+        .execute(
+            "DELETE FROM imap WHERE transport_id NOT IN (SELECT transports.id FROM transports)",
+            (),
+        )
+        .await
+        .log_err(context)
+        .ok();
+    context.sql.execute(
+        "DELETE FROM imap_sync WHERE transport_id NOT IN (SELECT transports.id FROM transports)",
+        (),
+    ).await.log_err(context).ok();
+
     // Delete POI locations
     // which don't have corresponding message.
     delete_orphaned_poi_locations(context)

src/sql/migrations.rs

@@ -16,7 +16,6 @@ use crate::constants::ShowEmails;
 use crate::context::Context;
 use crate::key::DcKey;
 use crate::log::warn;
-use crate::message::MsgId;
 use crate::provider::get_provider_info;
 use crate::sql::Sql;
 use crate::tools::{Time, inc_and_check, time_elapsed};
@@ -734,12 +733,6 @@ impl Sql {
         Ok(())
     }
 
-    async fn set_db_version_in_cache(&self, version: i32) -> Result<()> {
-        let mut lock = self.config_cache.write().await;
-        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
-        Ok(())
-    }
-
     async fn execute_migration(&self, query: &str, version: i32) -> Result<()> {
         self.execute_migration_transaction(
             |transaction| {
@@ -1612,51 +1605,11 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
         .await?;
     }
 
-    if dbversion < 108 {
-        let version = 108;
-        let chunk_size = context.get_max_smtp_rcpt_to().await?;
-        sql.transaction(move |trans| {
-            Sql::set_db_version_trans(trans, version)?;
-            let id_max =
-                trans.query_row("SELECT IFNULL((SELECT MAX(id) FROM smtp), 0)", (), |row| {
-                    let id_max: i64 = row.get(0)?;
-                    Ok(id_max)
-                })?;
-            while let Some((id, rfc724_mid, mime, msg_id, recipients, retries)) = trans
-                .query_row(
-                    "SELECT id, rfc724_mid, mime, msg_id, recipients, retries FROM smtp \
-                    WHERE id<=? LIMIT 1",
-                    (id_max,),
-                    |row| {
-                        let id: i64 = row.get(0)?;
-                        let rfc724_mid: String = row.get(1)?;
-                        let mime: String = row.get(2)?;
-                        let msg_id: MsgId = row.get(3)?;
-                        let recipients: String = row.get(4)?;
-                        let retries: i64 = row.get(5)?;
-                        Ok((id, rfc724_mid, mime, msg_id, recipients, retries))
-                    },
-                )
-                .optional()?
-            {
-                trans.execute("DELETE FROM smtp WHERE id=?", (id,))?;
-                let recipients = recipients.split(' ').collect::<Vec<_>>();
-                for recipients in recipients.chunks(chunk_size) {
-                    let recipients = recipients.join(" ");
-                    trans.execute(
-                        "INSERT INTO smtp (rfc724_mid, mime, msg_id, recipients, retries) \
-                        VALUES (?, ?, ?, ?, ?)",
-                        (&rfc724_mid, &mime, msg_id, recipients, retries),
-                    )?;
-                }
-            }
-            Ok(())
-        })
-        .await
-        .with_context(|| format!("migration failed for version {version}"))?;
-
-        sql.set_db_version_in_cache(version).await?;
-    }
+    // Migration 108 is removed, it was using high level code
+    // to split SMTP queue messages into chunks with smaller number of recipients
+    // and started to fail later as high level code started
+    // expecting `transports` table that is only added in future migrations.
+    // Migration 108 was not changing the database schema.
 
     if dbversion < 109 {
         sql.execute_migration(

src/stats.rs

@@ -554,7 +554,6 @@ async fn get_message_stats(context: &Context) -> Result<BTreeMap<Chattype, Messa
 }
 
 pub(crate) async fn update_message_stats(context: &Context) -> Result<()> {
-    info!(context, "Updating message statistics.");
     for chattype in [Chattype::Single, Chattype::Group, Chattype::OutBroadcast] {
         update_message_stats_inner(context, chattype).await?;
     }

src/test_utils.rs

@@ -715,7 +715,8 @@ impl TestContext {
     }
 
     pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec<SentMessage<'a>> {
-        self.ctx
+        let sent_msgs = self
+            .ctx
             .sql
             .query_map_vec(
                 "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?",
@@ -737,7 +738,23 @@ impl TestContext {
                 sender_context: &self.ctx,
                 recipients,
             })
-            .collect()
+            .collect();
+        self.ctx
+            .sql
+            .execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))
+            .await
+            .expect("Delete smtp jobs");
+        update_msg_state(&self.ctx, msg_id, MessageState::OutDelivered)
+            .await
+            .expect("Update message state");
+        self.sql
+            .execute(
+                "UPDATE msgs SET timestamp_sent=? WHERE id=?",
+                (time(), msg_id),
+            )
+            .await
+            .expect("Update timestamp_sent");
+        sent_msgs
     }
 
     /// Parses a message.

src/tests/pre_messages/sending.rs

@@ -56,26 +56,30 @@ async fn test_sending_pre_message() -> Result<()> {
             .is_some()
     );
 
+    let post_rfc724_mid = post_message_parsed
+        .headers
+        .get_header_value(HeaderDef::MessageId);
     assert_eq!(
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
+        post_rfc724_mid,
         Some(format!("<{}>", msg.rfc724_mid)),
         "Post-Message should have the rfc message id of the database message"
     );
 
+    let pre_rfc724_mid = pre_message_parsed
+        .headers
+        .get_header_value(HeaderDef::MessageId);
     assert_ne!(
-        pre_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
+        pre_rfc724_mid, post_rfc724_mid,
         "message ids of Pre-Message and Post-Message should be different"
     );
+    assert_eq!(
+        pre_rfc724_mid,
+        Some(format!("<{}>", msg.pre_rfc724_mid)),
+        "Unexpected pre-message RFC 724 ID"
+    );
 
     let decrypted_post_message = bob.parse_msg(post_message).await;
-    assert_eq!(decrypted_post_message.decrypting_failed, false);
+    assert!(decrypted_post_message.decryption_error.is_none());
     assert_eq!(
         decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId),
         false
@@ -86,9 +90,7 @@ async fn test_sending_pre_message() -> Result<()> {
         decrypted_pre_message
             .get_header(HeaderDef::ChatPostMessageId)
             .map(String::from),
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId)
+        post_rfc724_mid,
     );
     assert!(
         pre_message_parsed
@@ -98,6 +100,25 @@ async fn test_sending_pre_message() -> Result<()> {
         "no Chat-Post-Message-ID header in unprotected headers of Pre-Message"
     );
 
+    chat::resend_msgs(alice, &[msg_id]).await?;
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+    assert_eq!(smtp_rows.len(), 2);
+
+    let pre_message_parsed = mailparse::parse_mail(smtp_rows[0].payload.as_bytes())?;
+    assert_eq!(
+        pre_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        pre_rfc724_mid
+    );
+    let post_message_parsed = mailparse::parse_mail(smtp_rows[1].payload.as_bytes())?;
+    assert_eq!(
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        post_rfc724_mid
+    );
+
     Ok(())
 }
 

src/tests/verified_chats.rs

@@ -246,46 +246,6 @@ async fn test_old_message_4() -> Result<()> {
     Ok(())
 }
 
-/// Alice is offline for some time.
-/// When they come online, first their mvbox is synced and then their inbox.
-/// This test tests that the messages are still in the right order.
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_old_message_5() -> Result<()> {
-    let alice = TestContext::new_alice().await;
-    let msg_sent = receive_imf(
-        &alice,
-        b"From: alice@example.org\n\
-          To: Bob <bob@example.net>\n\
-          Message-ID: <1234-2-4@example.org>\n\
-          Date: Sat, 07 Dec 2019 19:00:27 +0000\n\
-          \n\
-          Happy birthday, Bob!\n",
-        true,
-    )
-    .await?
-    .unwrap();
-
-    let msg_incoming = receive_imf(
-        &alice,
-        b"From: Bob <bob@example.net>\n\
-          To: alice@example.org\n\
-          Message-ID: <1234-2-3@example.org>\n\
-          Date: Sun, 07 Dec 2019 19:00:26 +0000\n\
-          \n\
-          Happy birthday to me, Alice!\n",
-        false,
-    )
-    .await?
-    .unwrap();
-
-    assert!(msg_sent.sort_timestamp == msg_incoming.sort_timestamp);
-    alice
-        .golden_test_chat(msg_sent.chat_id, "test_old_message_5")
-        .await;
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_mdn_doesnt_disable_verification() -> Result<()> {
     let mut tcm = TestContextManager::new();

src/tools.rs

@@ -21,7 +21,6 @@ pub use std::time::SystemTime as Time;
 #[cfg(not(test))]
 pub use std::time::SystemTime;
 
-use crate::log::LogExt as _;
 use anyhow::{Context as _, Result, bail, ensure};
 use base64::Engine as _;
 use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
@@ -249,8 +248,6 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
                 true,
             )
             .await
-            .context("Failed to add bad time warning")
-            .log_err(context)
             .ok();
         } else {
             warn!(context, "Can't convert current timestamp");

src/tools/tools_tests.rs

@@ -64,9 +64,11 @@ DKIM Results: Passed=true";
 
 async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
     let t = TestContext::new_alice().await;
-    receive_imf(&t, raw, false).await.unwrap();
-    let msg = t.get_last_msg().await;
-    let msg_info = msg.id.get_info(&t).await.unwrap();
+    let received = receive_imf(&t, raw, false).await.unwrap().unwrap();
+
+    assert_eq!(received.msg_ids.len(), 1);
+    let msg_id = received.msg_ids[0];
+    let msg_info = msg_id.get_info(&t).await.unwrap();
 
     // Ignore the first rows of the msg_info because they contain a
     // received time that depends on the test time which makes it impossible to

test-data/golden/receive_imf_delayed_removal_is_ignored

@@ -2,9 +2,9 @@ Group#Chat#1001: Group [5 member(s)]
 --------------------------------------------------------------------------------
 Msg#1001: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO]
 Msg#1002🔒: Me (Contact#Contact#Self): populate  √
+Msg#1007🔒:  (Contact#Contact#1001): Member fiona@example.net removed by bob@example.net. [FRESH][INFO]
 Msg#1003: info (Contact#Contact#Info): Member dom@example.net added. [NOTICED][INFO]
 Msg#1004: info (Contact#Contact#Info): Member fiona@example.net removed. [NOTICED][INFO]
 Msg#1005🔒:  (Contact#Contact#1001): Member elena@example.net added by bob@example.net. [FRESH][INFO]
 Msg#1006🔒: Me (Contact#Contact#Self): You added member fiona@example.net. [INFO] o
-Msg#1007🔒:  (Contact#Contact#1001): Member fiona@example.net removed by bob@example.net. [FRESH][INFO]
 --------------------------------------------------------------------------------

test-data/golden/receive_imf_older_message_from_2nd_device

@@ -1,5 +1,5 @@
 Single#Chat#1001: bob@example.net [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png
 --------------------------------------------------------------------------------
-Msg#1001: Me (Contact#Contact#Self): We share this account  √
 Msg#1002: Me (Contact#Contact#Self): I'm Alice too  √
+Msg#1001: Me (Contact#Contact#Self): We share this account  √
 --------------------------------------------------------------------------------

test-data/golden/test_old_message_5

@@ -1,5 +0,0 @@
-Single#Chat#11001: Bob [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png
---------------------------------------------------------------------------------
-Msg#11001: Me (Contact#Contact#Self): Happy birthday, Bob!  √
-Msg#11002:  (Contact#Contact#11001): Happy birthday to me, Alice! [FRESH]
---------------------------------------------------------------------------------

test-data/golden/test_sync_broadcast_alice1

@@ -1,7 +1,7 @@
 OutBroadcast#Chat#1001: Channel [0 member(s)]
 --------------------------------------------------------------------------------
 Msg#1001: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO]
-Msg#1006🔒: Me (Contact#Contact#Self): Member bob@example.net added. [INFO] √
 Msg#1008🔒: Me (Contact#Contact#Self): hi  √
+Msg#1006🔒: Me (Contact#Contact#Self): Member bob@example.net added. [INFO] √
 Msg#1009🔒: Me (Contact#Contact#Self): You removed member bob@example.net. [INFO] √
 --------------------------------------------------------------------------------