more debug logging

.github/workflows/dependabot.yml

@@ -10,7 +10,7 @@ permissions:
 jobs:
   dependabot:
     runs-on: ubuntu-latest
-    if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == github.event.pull_request.head.repo.full_name
+    if: ${{ github.actor == 'dependabot[bot]' }}
     steps:
       - name: Dependabot metadata
         id: metadata

.github/workflows/upload-docs.yml

@@ -1,18 +1,16 @@
-name: Build & deploy documentation on rs.delta.chat, c.delta.chat, py.delta.chat and cffi.delta.chat
+name: Build & deploy documentation on rs.delta.chat, c.delta.chat, and py.delta.chat
 
 on:
   push:
     branches:
       - main
+      - build_jsonrpc_docs_ci
 
 permissions: {}
 
 jobs:
   build-rs:
     runs-on: ubuntu-latest
-    environment:
-      name: rs.delta.chat
-      url: https://rs.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -25,15 +23,12 @@ jobs:
       - name: Upload to rs.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.RS_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.RS_DOCS_SSH_USER }}@rs.delta.chat:/var/www/html/rs.delta.chat/"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.USERNAME }}@rs.delta.chat:/var/www/html/rs/"
 
   build-python:
     runs-on: ubuntu-latest
-    environment:
-      name: py.delta.chat
-      url: https://py.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -47,15 +42,12 @@ jobs:
       - name: Upload to py.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.PY_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "${{ secrets.PY_DOCS_SSH_USER }}@py.delta.chat:/var/www/html/py.delta.chat"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
 
   build-c:
     runs-on: ubuntu-latest
-    environment:
-      name: c.delta.chat
-      url: https://c.delta.chat/
 
     steps:
       - uses: actions/checkout@v6
@@ -69,16 +61,12 @@ jobs:
       - name: Upload to c.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.C_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "${{ secrets.C_DOCS_SSH_USER }}@c.delta.chat:/var/www/html/c.delta.chat"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
 
   build-ts:
     runs-on: ubuntu-latest
-    environment:
-      name: js.jsonrpc.delta.chat
-      url: https://js.jsonrpc.delta.chat/
-
     defaults:
       run:
         working-directory: ./deltachat-jsonrpc/typescript
@@ -102,27 +90,6 @@ jobs:
       - name: Upload to js.jsonrpc.delta.chat
         run: |
           mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.JS_JSONRPC_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
-          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.JS_JSONRPC_DOCS_SSH_USER }}@js.jsonrpc.delta.chat:/var/www/html/js.jsonrpc.delta.chat/"
-
-  build-cffi:
-    runs-on: ubuntu-latest
-    environment:
-      name: cffi.delta.chat
-      url: https://cffi.delta.chat/
-
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          show-progress: false
-          persist-credentials: false
-      - name: Build the documentation with cargo
-        run: |
-          cargo doc --package deltachat_ffi --no-deps
-      - name: Upload to cffi.delta.chat
-        run: |
-          mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.CFFI_DOCS_SSH_KEY }}" > "$HOME/.ssh/key"
-          chmod 600 "$HOME/.ssh/key"
-          rsync -avzh --delete -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.CFFI_DOCS_SSH_USER }}@delta.chat:/var/www/html/cffi.delta.chat/"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.USERNAME }}@js.jsonrpc.delta.chat:/var/www/html/js-jsonrpc/"

.github/workflows/upload-ffi-docs.yml

@@ -0,0 +1,31 @@
+# GitHub Actions workflow
+# to build `deltachat_ffi` crate documentation
+# and upload it to <https://cffi.delta.chat/>
+
+name: Build & Deploy Documentation on cffi.delta.chat
+
+on:
+  push:
+    branches:
+      - main
+
+permissions: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          show-progress: false
+          persist-credentials: false
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat_ffi --no-deps
+      - name: Upload to cffi.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"

CHANGELOG.md

@@ -1,61 +1,5 @@
 # Changelog
 
-## [2.48.0] - 2026-03-30
-
-### Fixes
-
-- Fix reordering problems in multi-relay setups by not sorting received messages below the last seen one.
-- Always sort "Messages are end-to-end encrypted" notice to the beginning.
-- Make Message-ID of pre-messages stable across resends ([#8007](https://github.com/chatmail/core/pull/8007)).
-- Delete `imap_markseen` entries not corresponding to any `imap` rows.
-- Cleanup `imap` and `imap_sync` records without transport in housekeeping.
-- When receiving MDN, mark all preceding messages as noticed, even having same timestamp ([#7928](https://github.com/chatmail/core/pull/7928)).
-- Remove migration 108 preventing upgrades from core 1.86.0 to the latest version.
-
-### Features / Changes
-
-- Improve IMAP loop logs.
-- Add decryption error to the device message about outgoing message decryption failure.
-- Log received message sort timestamp.
-
-### Performance
-
-- Move sorting outside of SQL query in `store_seen_flags_on_imap`.
-
-### API-Changes
-
-- Add JSON-RPC API `markfresh_chat()`.
-- ffi: Correctly declare `dc_event_channel_new()` as having no params ([#7831](https://github.com/chatmail/core/pull/7831)).
-
-### Refactor
-
-- Remove `wal_checkpoint_mutex`, lock `write_mutex` before getting sql connection instead.
-- Replace async `RwLock` with sync `RwLock` for stock strings.
-- Cleanup remaining Autocrypt Setup Message processing in `mimeparser`.
-- SecureJoin: do not check for self address in forwarding protection.
-- Fix clippy warnings.
-
-### CI
-
-- Update {c,py}.delta.chat website deployments.
-- Use environments for {rs,cffi,js.jsonrpc}.delta.chat deployments.
-- Fix https://docs.zizmor.sh/audits/#bot-conditions.
-
-### Documentation
-
-- Add SQL performance tips to STYLE.md.
-
-### Tests
-
-- Remove `test_old_message_5`.
-- Do not rely on loading newest chat in `load_imf_email()`.
-- Use `load_imf_email()` more.
-- The message is sorted correctly in the chat even if it arrives late.
-
-### Miscellaneous Tasks
-
-- cargo: update rustls-webpki to 0.103.10.
-
 ## [2.47.0] - 2026-03-24
 
 ### Fixes
@@ -8040,4 +7984,3 @@ https://github.com/chatmail/core/pulls?q=is%3Apr+is%3Aclosed
 [2.45.0]: https://github.com/chatmail/core/compare/v2.44.0..v2.45.0
 [2.46.0]: https://github.com/chatmail/core/compare/v2.45.0..v2.46.0
 [2.47.0]: https://github.com/chatmail/core/compare/v2.46.0..v2.47.0
-[2.48.0]: https://github.com/chatmail/core/compare/v2.47.0..v2.48.0

Cargo.lock

@@ -1307,7 +1307,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "2.48.0"
+version = "2.48.0-dev"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -1416,7 +1416,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "2.48.0"
+version = "2.48.0-dev"
 dependencies = [
  "anyhow",
  "async-channel 2.5.0",
@@ -1437,7 +1437,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "2.48.0"
+version = "2.48.0-dev"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1453,7 +1453,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "2.48.0"
+version = "2.48.0-dev"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1482,7 +1482,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "2.48.0"
+version = "2.48.0-dev"
 dependencies = [
  "anyhow",
  "deltachat",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "2.48.0"
+version = "2.48.0-dev"
 edition = "2024"
 license = "MPL-2.0"
 rust-version = "1.88"

STYLE.md

@@ -68,12 +68,6 @@ keyword doesn't help here.
 Consider adding context to `anyhow` errors for SQL statements using `.context()` so that it's
 possible to understand from logs which statement failed. See [Errors](#errors) for more info.
 
-When changing complex SQL queries, test them on a new database with `EXPLAIN QUERY PLAN`
-to make sure that indexes are used and large tables are not going to be scanned.
-Never run `ANALYZE` on the databases,
-this makes query planner unpredictable
-and may make performance significantly worse: <https://github.com/chatmail/core/issues/6585>
-
 ## Errors
 
 Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "2.48.0"
+version = "2.48.0-dev"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "2.48.0"
+version = "2.48.0-dev"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 license = "MPL-2.0"

deltachat-jsonrpc/typescript/package.json

@@ -54,5 +54,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "2.48.0"
+  "version": "2.48.0-dev"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "2.48.0"
+version = "2.48.0-dev"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/chatmail/core"

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "2.48.0"
+version = "2.48.0-dev"
 license = "MPL-2.0"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "2.48.0"
+version = "2.48.0-dev"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "2.48.0"
+  "version": "2.48.0-dev"
 }

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "2.48.0"
+version = "2.48.0-dev"
 license = "MPL-2.0"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"

release-date.in

@@ -1 +1 @@
-2026-03-30
+2026-03-24

src/chat.rs

@@ -477,17 +477,12 @@ impl ChatId {
     /// Adds message "Messages are end-to-end encrypted".
     pub(crate) async fn add_e2ee_notice(self, context: &Context, timestamp: i64) -> Result<()> {
         let text = stock_str::messages_e2ee_info_msg(context);
-
-        // Sort this notice to the very beginning of the chat.
-        // We don't want any message to appear before this notice
-        // which is normally added when encrypted chat is created.
-        let sort_timestamp = 0;
         add_info_msg_with_cmd(
             context,
             self,
             &text,
             SystemMessage::ChatE2ee,
-            Some(sort_timestamp),
+            Some(timestamp),
             timestamp,
             None,
             None,
@@ -930,17 +925,6 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
             .unwrap_or(0))
     }
 
-    /// Returns timestamp of us joining the chat if we are the member of the chat.
-    pub(crate) async fn join_timestamp(self, context: &Context) -> Result<Option<i64>> {
-        context
-            .sql
-            .query_get_value(
-                "SELECT add_timestamp FROM chats_contacts WHERE chat_id=? AND contact_id=?",
-                (self, ContactId::SELF),
-            )
-            .await
-    }
-
     /// Returns timestamp of the latest message in the chat,
     /// including hidden messages or a draft if there is one.
     pub(crate) async fn get_timestamp(self, context: &Context) -> Result<Option<i64>> {
@@ -1228,11 +1212,15 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
     /// corresponding event in case of a system message (usually the current system time).
     /// `always_sort_to_bottom` makes this adjust the returned timestamp up so that the message goes
     /// to the chat bottom.
+    /// `received` -- whether the message is received. Otherwise being sent.
+    /// `incoming` -- whether the message is incoming.
     pub(crate) async fn calc_sort_timestamp(
         self,
         context: &Context,
         message_timestamp: i64,
         always_sort_to_bottom: bool,
+        received: bool,
+        incoming: bool,
     ) -> Result<i64> {
         let mut sort_timestamp = cmp::min(message_timestamp, smeared_time(context));
 
@@ -1252,6 +1240,38 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
                     (self, MessageState::OutDraft),
                 )
                 .await?
+        } else if received {
+            // Received messages shouldn't mingle with just sent ones and appear somewhere in the
+            // middle of the chat, so we go after the newest non fresh message.
+            //
+            // But if a received outgoing message is older than some seen message, better sort the
+            // received message purely by timestamp. We could place it just before that seen
+            // message, but anyway the user may not notice it.
+            //
+            // NB: Received outgoing messages may break sorting of fresh incoming ones, but this
+            // shouldn't happen frequently. Seen incoming messages don't really break sorting of
+            // fresh ones, they rather mean that older incoming messages are actually seen as well.
+            context
+                .sql
+                .query_row_optional(
+                    "SELECT MAX(timestamp), MAX(IIF(state=?,timestamp_sent,0))
+                     FROM msgs
+                     WHERE chat_id=? AND hidden=0 AND state>?
+                     HAVING COUNT(*) > 0",
+                    (MessageState::InSeen, self, MessageState::InFresh),
+                    |row| {
+                        let ts: i64 = row.get(0)?;
+                        let ts_sent_seen: i64 = row.get(1)?;
+                        Ok((ts, ts_sent_seen))
+                    },
+                )
+                .await?
+                .and_then(|(ts, ts_sent_seen)| {
+                    match incoming || ts_sent_seen <= message_timestamp {
+                        true => Some(ts),
+                        false => None,
+                    }
+                })
         } else {
             None
         };
@@ -1262,16 +1282,7 @@ SELECT id, rfc724_mid, pre_rfc724_mid, timestamp, ?, 1 FROM msgs WHERE chat_id=?
             sort_timestamp = last_msg_time;
         }
 
-        if let Some(join_timestamp) = self.join_timestamp(context).await? {
-            // If we are the member of the chat, don't add messages
-            // before the timestamp of us joining it.
-            // This is needed to avoid sorting "Member added"
-            // or automatically sent bot welcome messages
-            // above SecureJoin system messages.
-            Ok(std::cmp::max(sort_timestamp, join_timestamp))
-        } else {
-            Ok(sort_timestamp)
-        }
+        Ok(sort_timestamp)
     }
 }
 
@@ -4927,8 +4938,15 @@ pub(crate) async fn add_info_msg_with_cmd(
         ts
     } else {
         let sort_to_bottom = true;
+        let (received, incoming) = (false, false);
         chat_id
-            .calc_sort_timestamp(context, smeared_time(context), sort_to_bottom)
+            .calc_sort_timestamp(
+                context,
+                smeared_time(context),
+                sort_to_bottom,
+                received,
+                incoming,
+            )
             .await?
     };
 

src/chat/chat_tests.rs

@@ -2792,7 +2792,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&vc_pubkey).await;
-        assert!(parsed_by_bob.decryption_error.is_some());
+        assert!(parsed_by_bob.decrypting_failed);
 
         charlie.recv_msg_trash(&vc_pubkey).await;
     }
@@ -2821,7 +2821,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&member_added).await;
-        assert!(parsed_by_bob.decryption_error.is_some());
+        assert!(parsed_by_bob.decrypting_failed);
 
         let rcvd = charlie.recv_msg(&member_added).await;
         assert_eq!(rcvd.param.get_cmd(), SystemMessage::MemberAddedToGroup);
@@ -2836,7 +2836,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&hi_msg).await;
-        assert!(parsed_by_bob.decryption_error.is_none());
+        assert_eq!(parsed_by_bob.decrypting_failed, false);
     }
 
     tcm.section("Alice removes Charlie. Bob must not see it.");
@@ -2853,7 +2853,7 @@ async fn test_broadcast_members_cant_see_each_other() -> Result<()> {
         assert_eq!(parsed.decoded_data_contains("bob@example.net"), false);
 
         let parsed_by_bob = bob.parse_msg(&member_removed).await;
-        assert!(parsed_by_bob.decryption_error.is_some());
+        assert!(parsed_by_bob.decrypting_failed);
 
         let rcvd = charlie.recv_msg(&member_removed).await;
         assert_eq!(rcvd.param.get_cmd(), SystemMessage::MemberRemovedFromGroup);
@@ -6112,54 +6112,3 @@ async fn test_leftgrps() -> Result<()> {
 
     Ok(())
 }
-
-/// Tests that if the message arrives late,
-/// it can still be sorted above the last seen message.
-///
-/// Versions 2.47 and below always sorted incoming messages
-/// after the last seen message, but with
-/// the introduction of multi-relay it is possible
-/// that some messages arrive only to some relays
-/// and are fetched after the already arrived seen message.
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_late_message_above_seen() -> Result<()> {
-    let mut tcm = TestContextManager::new();
-    let alice = &tcm.alice().await;
-    let bob = &tcm.bob().await;
-    let charlie = &tcm.charlie().await;
-
-    let alice_chat_id = alice
-        .create_group_with_members("Group", &[bob, charlie])
-        .await;
-    let alice_sent = alice.send_text(alice_chat_id, "Hello everyone!").await;
-    let bob_chat_id = bob.recv_msg(&alice_sent).await.chat_id;
-    bob_chat_id.accept(bob).await?;
-    let charlie_chat_id = charlie.recv_msg(&alice_sent).await.chat_id;
-    charlie_chat_id.accept(charlie).await?;
-
-    // Bob sends a message, but the message is delayed.
-    let bob_sent = bob.send_text(bob_chat_id, "Hello from Bob!").await;
-    SystemTime::shift(Duration::from_secs(1000));
-
-    let charlie_sent = charlie
-        .send_text(charlie_chat_id, "Hello from Charlie!")
-        .await;
-
-    // Alice immediately receives a message from Charlie and reads it.
-    let alice_received_from_charlie = alice.recv_msg(&charlie_sent).await;
-    assert_eq!(
-        alice_received_from_charlie.get_text(),
-        "Hello from Charlie!"
-    );
-    message::markseen_msgs(alice, vec![alice_received_from_charlie.id]).await?;
-
-    // Bob message arrives later, it should be above the message from Charlie.
-    let alice_received_from_bob = alice.recv_msg(&bob_sent).await;
-    assert_eq!(alice_received_from_bob.get_text(), "Hello from Bob!");
-
-    // The last message in the chat is still from Charlie.
-    let last_msg = alice.get_last_msg_in(alice_chat_id).await;
-    assert_eq!(last_msg.get_text(), "Hello from Charlie!");
-
-    Ok(())
-}

src/chatlist.rs

@@ -142,7 +142,7 @@ impl Chatlist {
                    AND c.blocked!=1
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2 AND add_timestamp >= remove_timestamp)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                 (MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned),
                 process_row,
             ).await?
@@ -168,7 +168,7 @@ impl Chatlist {
                    AND c.blocked!=1
                    AND c.archived=1
                  GROUP BY c.id
-                 ORDER BY IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft,),
                     process_row,
                 )
@@ -204,7 +204,7 @@ impl Chatlist {
                    AND IFNULL(c.name_normalized,c.name) LIKE ?3
                    AND (NOT ?4 OR EXISTS (SELECT 1 FROM msgs m WHERE m.chat_id = c.id AND m.state == ?5 AND hidden=0))
                  GROUP BY c.id
-                 ORDER BY IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, str_like_cmd, only_unread, MessageState::InFresh),
                     process_row,
                 )
@@ -253,7 +253,7 @@ impl Chatlist {
                        AND NOT c.archived=?
                        AND (c.type!=? OR c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=? AND add_timestamp >= remove_timestamp))
                      GROUP BY c.id
-                     ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
+                     ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (
                         MessageState::OutDraft, skip_id, ChatVisibility::Archived,
                         Chattype::Group, ContactId::SELF,
@@ -279,7 +279,7 @@ impl Chatlist {
                        AND (c.blocked=0 OR c.blocked=2)
                        AND NOT c.archived=?
                      GROUP BY c.id
-                     ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(NULLIF(m.timestamp,0),c.created_timestamp) DESC, m.id DESC;",
+                     ORDER BY c.id=0 DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (MessageState::OutDraft, skip_id, ChatVisibility::Archived, ChatVisibility::Pinned),
                     process_row,
                 ).await?

src/configure.rs

@@ -243,6 +243,11 @@ impl Context {
                         Ok((id, add_timestamp))
                     },
                 )?;
+                transaction.execute("DELETE FROM imap WHERE transport_id=?", (transport_id,))?;
+                transaction.execute(
+                    "DELETE FROM imap_sync WHERE transport_id=?",
+                    (transport_id,),
+                )?;
 
                 // Removal timestamp should not be lower than addition timestamp
                 // to be accepted by other devices when synced.

src/imap.rs

@@ -323,7 +323,8 @@ impl Imap {
         if !ratelimit_duration.is_zero() {
             warn!(
                 context,
-                "IMAP got rate limited, waiting for {} until can connect.",
+                "Transport {}: IMAP got rate limited, waiting for {} until can connect.",
+                self.transport_id,
                 duration_to_str(ratelimit_duration),
             );
             let interrupted = async {
@@ -335,12 +336,16 @@ impl Imap {
             if interrupted {
                 info!(
                     context,
-                    "Connecting to IMAP without waiting for ratelimit due to interrupt."
+                    "Transport {}: Connecting to IMAP without waiting for ratelimit due to interrupt.",
+                    self.transport_id
                 );
             }
         }
 
-        info!(context, "Connecting to IMAP server.");
+        info!(
+            context,
+            "Transport {}: Connecting to IMAP server.", self.transport_id
+        );
         self.connectivity.set_connecting(context);
 
         self.conn_last_try = tools::Time::now();
@@ -355,7 +360,10 @@ impl Imap {
         let login_params = prioritize_server_login_params(&context.sql, &self.lp, "imap").await?;
         let mut first_error = None;
         for lp in login_params {
-            info!(context, "IMAP trying to connect to {}.", &lp.connection);
+            info!(
+                context,
+                "Transport {}: IMAP trying to connect to {}.", self.transport_id, &lp.connection
+            );
             let connection_candidate = lp.connection.clone();
             let client = match Client::connect(
                 context,
@@ -403,7 +411,10 @@ impl Imap {
                     let resync_request_sender = self.resync_request_sender.clone();
 
                     let session = if capabilities.can_compress {
-                        info!(context, "Enabling IMAP compression.");
+                        info!(
+                            context,
+                            "Transport {}: Enabling IMAP compression.", self.transport_id
+                        );
                         let compressed_session = session
                             .compress(|s| {
                                 let session_stream: Box<dyn SessionStream> = Box::new(s);
@@ -436,7 +447,10 @@ impl Imap {
                         lp.user
                     )));
                     self.connectivity.set_preparing(context);
-                    info!(context, "Successfully logged into IMAP server.");
+                    info!(
+                        context,
+                        "Transport {}: Successfully logged into IMAP server.", self.transport_id
+                    );
                     return Ok(session);
                 }
 
@@ -444,7 +458,10 @@ impl Imap {
                     let imap_user = lp.user.to_owned();
                     let message = stock_str::cannot_login(context, &imap_user);
 
-                    warn!(context, "IMAP failed to login: {err:#}.");
+                    warn!(
+                        context,
+                        "Transport {}: IMAP failed to login: {err:#}.", self.transport_id
+                    );
                     first_error.get_or_insert(format_err!("{message} ({err:#})"));
 
                     // If it looks like the password is wrong, send a notification:
@@ -463,7 +480,11 @@ impl Imap {
                             )
                             .await
                             {
-                                warn!(context, "Failed to add device message: {e:#}.");
+                                warn!(
+                                    context,
+                                    "Transport {}: Failed to add device message: {e:#}.",
+                                    self.transport_id
+                                );
                             } else {
                                 context
                                     .set_config_internal(Config::NotifyAboutWrongPw, None)
@@ -525,10 +546,21 @@ impl Imap {
             bail!("IMAP operation attempted while it is torn down");
         }
 
+        let transport_id = session.transport_id();
+        info!(
+            context,
+            "Transport {transport_id}: fetch_move_delete start."
+        );
+
         let msgs_fetched = self
             .fetch_new_messages(context, session, watch_folder, folder_meaning)
             .await
             .context("fetch_new_messages")?;
+
+        info!(
+            context,
+            "Transport {transport_id}: fetch_move_delete finished fetch_new_messages."
+        );
         if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
             // New messages were fetched and shall be deleted later, restart ephemeral loop.
             // Note that the `Config::DeleteDeviceAfter` timer starts as soon as the messages are
@@ -567,10 +599,18 @@ impl Imap {
             return Ok(false);
         }
 
+        info!(
+            context,
+            "Transport {transport_id}: fetch_new_messages selects folder {folder:?}."
+        );
         let folder_exists = session
             .select_with_uidvalidity(context, folder)
             .await
             .with_context(|| format!("Failed to select folder {folder:?}"))?;
+        info!(
+            context,
+            "Transport {transport_id}: fetch_new_messages selected folder {folder:?}."
+        );
 
         if !session.new_mail {
             info!(
@@ -1103,22 +1143,16 @@ impl Session {
             return Ok(());
         }
 
-        context
-            .sql
-            .execute(
-                "DELETE FROM imap_markseen WHERE id NOT IN (SELECT imap.id FROM imap)",
-                (),
-            )
-            .await?;
-
         let transport_id = self.transport_id();
-        let mut rows = context
+        info!(context, "Transport {transport_id}: Storing seen flags.");
+        let rows = context
             .sql
             .query_map_vec(
                 "SELECT imap.id, uid, folder FROM imap, imap_markseen
                  WHERE imap.id = imap_markseen.id
                  AND imap.transport_id=?
-                 AND target = folder",
+                 AND target = folder
+                 ORDER BY folder, uid",
                 (transport_id,),
                 |row| {
                     let rowid: i64 = row.get(0)?;
@@ -1129,16 +1163,6 @@ impl Session {
             )
             .await?;
 
-        // Number of SQL results is expected to be low as
-        // we usually don't have many messages to mark on IMAP at once.
-        // We are sorting outside of SQL to avoid SQLite constructing a query plan
-        // that scans the whole `imap` table. Scanning `imap_markseen` is fine
-        // as it should not have many items.
-        // If you change the SQL query, test it with `EXPLAIN QUERY PLAN`.
-        rows.sort_unstable_by(|(_rowid1, uid1, folder1), (_rowid2, uid2, folder2)| {
-            (folder1, uid1).cmp(&(folder2, uid2))
-        });
-
         for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
             let folder_exists = match self.select_with_uidvalidity(context, &folder).await {
                 Err(err) => {
@@ -1155,13 +1179,15 @@ impl Session {
             } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
                 warn!(
                     context,
-                    "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
+                    "Transport {transport_id}: Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}."
                 );
                 continue;
             } else {
                 info!(
                     context,
-                    "Marked messages {} in folder {} as seen.", uid_set, folder
+                    "Transport {transport_id}: Marked messages {} in folder {} as seen.",
+                    uid_set,
+                    folder
                 );
             }
             context
@@ -1176,6 +1202,10 @@ impl Session {
                 .await
                 .context("Cannot remove messages marked as seen from imap_markseen table")?;
         }
+        info!(
+            context,
+            "Transport {transport_id}: Finished storing seen flags."
+        );
 
         Ok(())
     }
@@ -1516,9 +1546,10 @@ impl Session {
             return Ok(());
         }
 
+        let transport_id = self.transport_id();
         info!(
             context,
-            "Server supports metadata, retrieving server comment and admin contact."
+            "Transport {transport_id}: Server supports metadata, retrieving server comment and admin contact."
         );
 
         let mut comment = None;
@@ -1551,7 +1582,8 @@ impl Session {
                         } else {
                             warn!(
                                 context,
-                                "Got invalid URL from iroh relay metadata: {:?}.", value
+                                "Transport {transport_id}: Got invalid URL from iroh relay metadata: {:?}.",
+                                value
                             );
                         }
                     }
@@ -1580,6 +1612,7 @@ impl Session {
             create_fallback_ice_servers()
         };
 
+        info!(context, "Transport {transport_id}: Got IMAP metadata.");
         *lock = Some(ServerMetadata {
             comment,
             admin,

src/imex.rs

@@ -1137,16 +1137,4 @@ mod tests {
 
         Ok(())
     }
-
-    /// Tests importing a backup from Delta Chat 1.30.3 for Android (core v1.86.0).
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_import_ancient_backup() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let context = &tcm.unconfigured().await;
-
-        let backup_path = Path::new("test-data/core-1.86.0-backup.tar");
-        imex(context, ImexMode::ImportBackup, backup_path, None).await?;
-
-        Ok(())
-    }
 }

src/message.rs

@@ -815,11 +815,7 @@ impl Message {
 
     /// Returns the timestamp of the message for sorting.
     pub fn get_sort_timestamp(&self) -> i64 {
-        if self.timestamp_sort != 0 {
-            self.timestamp_sort
-        } else {
-            self.timestamp_sent
-        }
+        self.timestamp_sort
     }
 
     /// Returns the text of the message.

src/mimefactory.rs

@@ -852,13 +852,7 @@ impl MimeFactory {
 
         let rfc724_mid = match &self.loaded {
             Loaded::Message { msg, .. } => match &self.pre_message_mode {
-                PreMessageMode::Pre { .. } => {
-                    if msg.pre_rfc724_mid.is_empty() {
-                        create_outgoing_rfc724_mid()
-                    } else {
-                        msg.pre_rfc724_mid.clone()
-                    }
-                }
+                PreMessageMode::Pre { .. } => create_outgoing_rfc724_mid(),
                 _ => msg.rfc724_mid.clone(),
             },
             Loaded::Mdn { .. } => create_outgoing_rfc724_mid(),

src/mimeparser.rs

@@ -86,9 +86,7 @@ pub(crate) struct MimeMessage {
     /// messages to this address to post them to the list.
     pub list_post: Option<String>,
     pub chat_disposition_notification_to: Option<SingleInfo>,
-
-    /// Decryption error if decryption of the message has failed.
-    pub decryption_error: Option<String>,
+    pub decrypting_failed: bool,
 
     /// Valid signature fingerprint if a message is an
     /// Autocrypt encrypted and signed message and corresponding intended recipient fingerprints
@@ -666,7 +664,7 @@ impl MimeMessage {
             from,
             incoming,
             chat_disposition_notification_to,
-            decryption_error: mail.err().map(|err| format!("{err:#}")),
+            decrypting_failed: mail.is_err(),
 
             // only non-empty if it was a valid autocrypt message
             signature,
@@ -907,7 +905,7 @@ impl MimeMessage {
             && let Some(ref subject) = self.get_subject()
         {
             let mut prepend_subject = true;
-            if self.decryption_error.is_none() {
+            if !self.decrypting_failed {
                 let colon = subject.find(':');
                 if colon == Some(2)
                     || colon == Some(3)
@@ -948,7 +946,7 @@ impl MimeMessage {
         self.parse_attachments();
 
         // See if an MDN is requested from the other side
-        if self.decryption_error.is_none()
+        if !self.decrypting_failed
             && !self.parts.is_empty()
             && let Some(ref dn_to) = self.chat_disposition_notification_to
         {
@@ -1080,7 +1078,7 @@ impl MimeMessage {
     #[cfg(test)]
     /// Returns whether the decrypted data contains the given `&str`.
     pub(crate) fn decoded_data_contains(&self, s: &str) -> bool {
-        assert!(self.decryption_error.is_none());
+        assert!(!self.decrypting_failed);
         let decoded_str = str::from_utf8(&self.decoded_data).unwrap();
         decoded_str.contains(s)
     }

src/receive_imf.rs

@@ -727,7 +727,7 @@ pub(crate) async fn receive_imf_inner(
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
             .unwrap_or_default();
 
-        let allow_creation = if mime_parser.decryption_error.is_some() {
+        let allow_creation = if mime_parser.decrypting_failed {
             false
         } else if is_dc_message == MessengerMessage::No
             && !context.get_config_bool(Config::IsChatmail).await?
@@ -1010,11 +1010,11 @@ UPDATE config SET value=? WHERE keyname='configured_addr' AND value!=?1
                         && msg.chat_visibility == ChatVisibility::Archived;
                     updated_chats
                         .entry(msg.chat_id)
-                        .and_modify(|pos| *pos = cmp::max(*pos, (msg.timestamp_sort, msg.id)))
-                        .or_insert((msg.timestamp_sort, msg.id));
+                        .and_modify(|ts| *ts = cmp::max(*ts, msg.timestamp_sort))
+                        .or_insert(msg.timestamp_sort);
                 }
             }
-            for (chat_id, (timestamp_sort, msg_id)) in updated_chats {
+            for (chat_id, timestamp_sort) in updated_chats {
                 context
                     .sql
                     .execute(
@@ -1023,13 +1023,12 @@ UPDATE msgs SET state=? WHERE
     state=? AND
     hidden=0 AND
     chat_id=? AND
-    (timestamp,id)<(?,?)",
+    timestamp<?",
                         (
                             MessageState::InNoticed,
                             MessageState::InFresh,
                             chat_id,
                             timestamp_sort,
-                            msg_id,
                         ),
                     )
                     .await
@@ -1212,18 +1211,14 @@ async fn decide_chat_assignment(
     {
         info!(context, "Call state changed (TRASH).");
         true
-    } else if let Some(ref decryption_error) = mime_parser.decryption_error
-        && !mime_parser.incoming
-    {
+    } else if mime_parser.decrypting_failed && !mime_parser.incoming {
         // Outgoing undecryptable message.
         let last_time = context
             .get_config_i64(Config::LastCantDecryptOutgoingMsgs)
             .await?;
         let now = tools::time();
         let update_config = if last_time.saturating_add(24 * 60 * 60) <= now {
-            let txt = format!(
-                "⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions. (Error: {decryption_error}, {rfc724_mid})."
-            );
+            let txt = "⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions.";
             let mut msg = Message::new_text(txt.to_string());
             chat::add_device_msg(context, None, Some(&mut msg))
                 .await
@@ -1840,16 +1835,6 @@ async fn add_parts(
         }
     }
 
-    // Sort message to the bottom if we are not in the chat
-    // so if we are added via QR code scan
-    // the message about our addition goes after all the info messages.
-    // Info messages are sorted by local smeared_timestamp()
-    // which advances quickly during SecureJoin,
-    // while "member added" message may have older timestamp
-    // corresponding to the sender clock.
-    // In practice inviter clock may even be slightly in the past.
-    let sort_to_bottom = !chat.is_self_in_chat(context).await?;
-
     let is_location_kml = mime_parser.location_kml.is_some();
     let mut group_changes = match chat.typ {
         _ if chat.id.is_special() => GroupChangesInfo::default(),
@@ -1900,8 +1885,16 @@ async fn add_parts(
     };
     let in_fresh = state == MessageState::InFresh;
 
+    let sort_to_bottom = false;
+    let received = true;
     let sort_timestamp = chat_id
-        .calc_sort_timestamp(context, mime_parser.timestamp_sent, sort_to_bottom)
+        .calc_sort_timestamp(
+            context,
+            mime_parser.timestamp_sent,
+            sort_to_bottom,
+            received,
+            mime_parser.incoming,
+        )
         .await?;
 
     // Apply ephemeral timer changes to the chat.
@@ -2297,7 +2290,7 @@ RETURNING id
                     if trash { 0 } else { ephemeral_timestamp },
                     if trash {
                         DownloadState::Done
-                    } else if mime_parser.decryption_error.is_some() {
+                    } else if mime_parser.decrypting_failed {
                         DownloadState::Undecipherable
                     } else if let PreMessageMode::Pre {..} = mime_parser.pre_message {
                         DownloadState::Available
@@ -2361,7 +2354,7 @@ RETURNING id
 
     info!(
         context,
-        "Message has {icnt} parts and is assigned to chat #{chat_id}, timestamp={sort_timestamp}."
+        "Message has {icnt} parts and is assigned to chat #{chat_id}."
     );
 
     if !chat_id.is_trash() && !hidden {
@@ -2710,7 +2703,7 @@ async fn lookup_or_create_adhoc_group(
     allow_creation: bool,
     create_blocked: Blocked,
 ) -> Result<Option<(ChatId, Blocked, bool)>> {
-    if mime_parser.decryption_error.is_some() {
+    if mime_parser.decrypting_failed {
         warn!(
             context,
             "Not creating ad-hoc group for message that cannot be decrypted."
@@ -2932,7 +2925,7 @@ async fn create_group(
 
     if let Some(chat_id) = chat_id {
         Ok(Some((chat_id, chat_id_blocked)))
-    } else if mime_parser.decryption_error.is_some() {
+    } else if mime_parser.decrypting_failed {
         // It is possible that the message was sent to a valid,
         // yet unknown group, which was rejected because
         // Chat-Group-Name, which is in the encrypted part, was

src/receive_imf/receive_imf_tests.rs

@@ -819,12 +819,9 @@ async fn load_imf_email(context: &Context, imf_raw: &[u8]) -> Message {
         .set_config(Config::ShowEmails, Some("2"))
         .await
         .unwrap();
-    let received_msg = receive_imf(context, imf_raw, false)
-        .await
-        .expect("receive_imf failure")
-        .expect("No message received");
-    assert_eq!(received_msg.msg_ids.len(), 1);
-    let msg_id = received_msg.msg_ids[0];
+    receive_imf(context, imf_raw, false).await.unwrap();
+    let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+    let msg_id = chats.get_msg_id(0).unwrap().unwrap();
     Message::load_from_db(context, msg_id).await.unwrap()
 }
 
@@ -2875,8 +2872,9 @@ async fn test_rfc1847_encapsulation() -> Result<()> {
 
     // Alice sends a message to Bob using Thunderbird.
     let raw = include_bytes!("../../test-data/message/rfc1847_encapsulation.eml");
+    receive_imf(bob, raw, false).await?;
 
-    let msg = load_imf_email(bob, raw).await;
+    let msg = bob.get_last_msg().await;
     assert!(msg.get_showpadlock());
 
     Ok(())
@@ -3084,8 +3082,8 @@ async fn test_auto_accept_for_bots() -> Result<()> {
 async fn test_auto_accept_group_for_bots() -> Result<()> {
     let t = TestContext::new_alice().await;
     t.set_config(Config::Bot, Some("1")).await.unwrap();
-    let msg = load_imf_email(&t, GRP_MAIL).await;
-
+    receive_imf(&t, GRP_MAIL, false).await?;
+    let msg = t.get_last_msg().await;
     let chat = chat::Chat::load_from_db(&t, msg.chat_id).await?;
     assert!(!chat.is_contact_request());
     Ok(())
@@ -3329,7 +3327,7 @@ async fn test_outgoing_undecryptable() -> Result<()> {
     assert!(
         dev_msg
             .text
-            .starts_with("⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions. (Error:")
+            .contains("⚠️ It seems you are using Delta Chat on multiple devices that cannot decrypt each other's outgoing messages. To fix this, on the older device use \"Settings / Add Second Device\" and follow the instructions.")
     );
 
     let raw = include_bytes!("../../test-data/message/thunderbird_encrypted_signed.eml");
@@ -3558,9 +3556,9 @@ async fn test_messed_up_message_id() -> Result<()> {
     let t = TestContext::new_bob().await;
 
     let raw = include_bytes!("../../test-data/message/messed_up_message_id.eml");
-    let msg = load_imf_email(&t, raw).await;
+    receive_imf(&t, raw, false).await?;
     assert_eq!(
-        msg.rfc724_mid,
+        t.get_last_msg().await.rfc724_mid,
         "0bb9ffe1-2596-d997-95b4-1fef8cc4808e@example.org"
     );
 

src/securejoin.rs

@@ -450,8 +450,10 @@ pub(crate) async fn handle_securejoin_handshake(
     ) {
         let mut self_found = false;
         let self_fingerprint = load_self_public_key(context).await?.dc_fingerprint();
-        for key in mime_message.gossiped_keys.values() {
-            if key.public_key.dc_fingerprint() == self_fingerprint {
+        for (addr, key) in &mime_message.gossiped_keys {
+            if key.public_key.dc_fingerprint() == self_fingerprint
+                && context.is_self_addr(addr).await?
+            {
                 self_found = true;
                 break;
             }

src/sql.rs

@@ -883,24 +883,6 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
-    // Cleanup `imap` and `imap_sync` entries for deleted transports.
-    //
-    // Transports may be deleted directly or via sync messages,
-    // so it is easier to cleanup orphaned entries in a single place.
-    context
-        .sql
-        .execute(
-            "DELETE FROM imap WHERE transport_id NOT IN (SELECT transports.id FROM transports)",
-            (),
-        )
-        .await
-        .log_err(context)
-        .ok();
-    context.sql.execute(
-        "DELETE FROM imap_sync WHERE transport_id NOT IN (SELECT transports.id FROM transports)",
-        (),
-    ).await.log_err(context).ok();
-
     // Delete POI locations
     // which don't have corresponding message.
     delete_orphaned_poi_locations(context)

src/sql/migrations.rs

@@ -16,6 +16,7 @@ use crate::constants::ShowEmails;
 use crate::context::Context;
 use crate::key::DcKey;
 use crate::log::warn;
+use crate::message::MsgId;
 use crate::provider::get_provider_info;
 use crate::sql::Sql;
 use crate::tools::{Time, inc_and_check, time_elapsed};
@@ -733,6 +734,12 @@ impl Sql {
         Ok(())
     }
 
+    async fn set_db_version_in_cache(&self, version: i32) -> Result<()> {
+        let mut lock = self.config_cache.write().await;
+        lock.insert(VERSION_CFG.to_string(), Some(format!("{version}")));
+        Ok(())
+    }
+
     async fn execute_migration(&self, query: &str, version: i32) -> Result<()> {
         self.execute_migration_transaction(
             |transaction| {
@@ -1605,11 +1612,51 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
         .await?;
     }
 
-    // Migration 108 is removed, it was using high level code
-    // to split SMTP queue messages into chunks with smaller number of recipients
-    // and started to fail later as high level code started
-    // expecting `transports` table that is only added in future migrations.
-    // Migration 108 was not changing the database schema.
+    if dbversion < 108 {
+        let version = 108;
+        let chunk_size = context.get_max_smtp_rcpt_to().await?;
+        sql.transaction(move |trans| {
+            Sql::set_db_version_trans(trans, version)?;
+            let id_max =
+                trans.query_row("SELECT IFNULL((SELECT MAX(id) FROM smtp), 0)", (), |row| {
+                    let id_max: i64 = row.get(0)?;
+                    Ok(id_max)
+                })?;
+            while let Some((id, rfc724_mid, mime, msg_id, recipients, retries)) = trans
+                .query_row(
+                    "SELECT id, rfc724_mid, mime, msg_id, recipients, retries FROM smtp \
+                    WHERE id<=? LIMIT 1",
+                    (id_max,),
+                    |row| {
+                        let id: i64 = row.get(0)?;
+                        let rfc724_mid: String = row.get(1)?;
+                        let mime: String = row.get(2)?;
+                        let msg_id: MsgId = row.get(3)?;
+                        let recipients: String = row.get(4)?;
+                        let retries: i64 = row.get(5)?;
+                        Ok((id, rfc724_mid, mime, msg_id, recipients, retries))
+                    },
+                )
+                .optional()?
+            {
+                trans.execute("DELETE FROM smtp WHERE id=?", (id,))?;
+                let recipients = recipients.split(' ').collect::<Vec<_>>();
+                for recipients in recipients.chunks(chunk_size) {
+                    let recipients = recipients.join(" ");
+                    trans.execute(
+                        "INSERT INTO smtp (rfc724_mid, mime, msg_id, recipients, retries) \
+                        VALUES (?, ?, ?, ?, ?)",
+                        (&rfc724_mid, &mime, msg_id, recipients, retries),
+                    )?;
+                }
+            }
+            Ok(())
+        })
+        .await
+        .with_context(|| format!("migration failed for version {version}"))?;
+
+        sql.set_db_version_in_cache(version).await?;
+    }
 
     if dbversion < 109 {
         sql.execute_migration(

src/stats.rs

@@ -554,6 +554,7 @@ async fn get_message_stats(context: &Context) -> Result<BTreeMap<Chattype, Messa
 }
 
 pub(crate) async fn update_message_stats(context: &Context) -> Result<()> {
+    info!(context, "Updating message statistics.");
     for chattype in [Chattype::Single, Chattype::Group, Chattype::OutBroadcast] {
         update_message_stats_inner(context, chattype).await?;
     }

src/test_utils.rs

@@ -715,8 +715,7 @@ impl TestContext {
     }
 
     pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec<SentMessage<'a>> {
-        let sent_msgs = self
-            .ctx
+        self.ctx
             .sql
             .query_map_vec(
                 "SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?",
@@ -738,23 +737,7 @@ impl TestContext {
                 sender_context: &self.ctx,
                 recipients,
             })
-            .collect();
-        self.ctx
-            .sql
-            .execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))
-            .await
-            .expect("Delete smtp jobs");
-        update_msg_state(&self.ctx, msg_id, MessageState::OutDelivered)
-            .await
-            .expect("Update message state");
-        self.sql
-            .execute(
-                "UPDATE msgs SET timestamp_sent=? WHERE id=?",
-                (time(), msg_id),
-            )
-            .await
-            .expect("Update timestamp_sent");
-        sent_msgs
+            .collect()
     }
 
     /// Parses a message.

src/tests/pre_messages/sending.rs

@@ -56,30 +56,26 @@ async fn test_sending_pre_message() -> Result<()> {
             .is_some()
     );
 
-    let post_rfc724_mid = post_message_parsed
-        .headers
-        .get_header_value(HeaderDef::MessageId);
     assert_eq!(
-        post_rfc724_mid,
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
         Some(format!("<{}>", msg.rfc724_mid)),
         "Post-Message should have the rfc message id of the database message"
     );
 
-    let pre_rfc724_mid = pre_message_parsed
-        .headers
-        .get_header_value(HeaderDef::MessageId);
     assert_ne!(
-        pre_rfc724_mid, post_rfc724_mid,
+        pre_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
         "message ids of Pre-Message and Post-Message should be different"
     );
-    assert_eq!(
-        pre_rfc724_mid,
-        Some(format!("<{}>", msg.pre_rfc724_mid)),
-        "Unexpected pre-message RFC 724 ID"
-    );
 
     let decrypted_post_message = bob.parse_msg(post_message).await;
-    assert!(decrypted_post_message.decryption_error.is_none());
+    assert_eq!(decrypted_post_message.decrypting_failed, false);
     assert_eq!(
         decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId),
         false
@@ -90,7 +86,9 @@ async fn test_sending_pre_message() -> Result<()> {
         decrypted_pre_message
             .get_header(HeaderDef::ChatPostMessageId)
             .map(String::from),
-        post_rfc724_mid,
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId)
     );
     assert!(
         pre_message_parsed
@@ -100,25 +98,6 @@ async fn test_sending_pre_message() -> Result<()> {
         "no Chat-Post-Message-ID header in unprotected headers of Pre-Message"
     );
 
-    chat::resend_msgs(alice, &[msg_id]).await?;
-    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-    assert_eq!(smtp_rows.len(), 2);
-
-    let pre_message_parsed = mailparse::parse_mail(smtp_rows[0].payload.as_bytes())?;
-    assert_eq!(
-        pre_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
-        pre_rfc724_mid
-    );
-    let post_message_parsed = mailparse::parse_mail(smtp_rows[1].payload.as_bytes())?;
-    assert_eq!(
-        post_message_parsed
-            .headers
-            .get_header_value(HeaderDef::MessageId),
-        post_rfc724_mid
-    );
-
     Ok(())
 }
 

src/tests/verified_chats.rs

@@ -246,6 +246,46 @@ async fn test_old_message_4() -> Result<()> {
     Ok(())
 }
 
+/// Alice is offline for some time.
+/// When they come online, first their mvbox is synced and then their inbox.
+/// This test tests that the messages are still in the right order.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_old_message_5() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let msg_sent = receive_imf(
+        &alice,
+        b"From: alice@example.org\n\
+          To: Bob <bob@example.net>\n\
+          Message-ID: <1234-2-4@example.org>\n\
+          Date: Sat, 07 Dec 2019 19:00:27 +0000\n\
+          \n\
+          Happy birthday, Bob!\n",
+        true,
+    )
+    .await?
+    .unwrap();
+
+    let msg_incoming = receive_imf(
+        &alice,
+        b"From: Bob <bob@example.net>\n\
+          To: alice@example.org\n\
+          Message-ID: <1234-2-3@example.org>\n\
+          Date: Sun, 07 Dec 2019 19:00:26 +0000\n\
+          \n\
+          Happy birthday to me, Alice!\n",
+        false,
+    )
+    .await?
+    .unwrap();
+
+    assert!(msg_sent.sort_timestamp == msg_incoming.sort_timestamp);
+    alice
+        .golden_test_chat(msg_sent.chat_id, "test_old_message_5")
+        .await;
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_mdn_doesnt_disable_verification() -> Result<()> {
     let mut tcm = TestContextManager::new();

src/tools.rs

@@ -21,6 +21,7 @@ pub use std::time::SystemTime as Time;
 #[cfg(not(test))]
 pub use std::time::SystemTime;
 
+use crate::log::LogExt as _;
 use anyhow::{Context as _, Result, bail, ensure};
 use base64::Engine as _;
 use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
@@ -248,6 +249,8 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
                 true,
             )
             .await
+            .context("Failed to add bad time warning")
+            .log_err(context)
             .ok();
         } else {
             warn!(context, "Can't convert current timestamp");

src/tools/tools_tests.rs

@@ -64,11 +64,9 @@ DKIM Results: Passed=true";
 
 async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
     let t = TestContext::new_alice().await;
-    let received = receive_imf(&t, raw, false).await.unwrap().unwrap();
-
-    assert_eq!(received.msg_ids.len(), 1);
-    let msg_id = received.msg_ids[0];
-    let msg_info = msg_id.get_info(&t).await.unwrap();
+    receive_imf(&t, raw, false).await.unwrap();
+    let msg = t.get_last_msg().await;
+    let msg_info = msg.id.get_info(&t).await.unwrap();
 
     // Ignore the first rows of the msg_info because they contain a
     // received time that depends on the test time which makes it impossible to

test-data/golden/receive_imf_delayed_removal_is_ignored

@@ -2,9 +2,9 @@ Group#Chat#1001: Group [5 member(s)]
 --------------------------------------------------------------------------------
 Msg#1001: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO]
 Msg#1002🔒: Me (Contact#Contact#Self): populate  √
-Msg#1007🔒:  (Contact#Contact#1001): Member fiona@example.net removed by bob@example.net. [FRESH][INFO]
 Msg#1003: info (Contact#Contact#Info): Member dom@example.net added. [NOTICED][INFO]
 Msg#1004: info (Contact#Contact#Info): Member fiona@example.net removed. [NOTICED][INFO]
 Msg#1005🔒:  (Contact#Contact#1001): Member elena@example.net added by bob@example.net. [FRESH][INFO]
 Msg#1006🔒: Me (Contact#Contact#Self): You added member fiona@example.net. [INFO] o
+Msg#1007🔒:  (Contact#Contact#1001): Member fiona@example.net removed by bob@example.net. [FRESH][INFO]
 --------------------------------------------------------------------------------

test-data/golden/receive_imf_older_message_from_2nd_device

@@ -1,5 +1,5 @@
 Single#Chat#1001: bob@example.net [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png
 --------------------------------------------------------------------------------
-Msg#1002: Me (Contact#Contact#Self): I'm Alice too  √
 Msg#1001: Me (Contact#Contact#Self): We share this account  √
+Msg#1002: Me (Contact#Contact#Self): I'm Alice too  √
 --------------------------------------------------------------------------------

test-data/golden/test_old_message_5

@@ -0,0 +1,5 @@
+Single#Chat#11001: Bob [bob@example.net] Icon: 4138c52e5bc1c576cda7dd44d088c07.png
+--------------------------------------------------------------------------------
+Msg#11001: Me (Contact#Contact#Self): Happy birthday, Bob!  √
+Msg#11002:  (Contact#Contact#11001): Happy birthday to me, Alice! [FRESH]
+--------------------------------------------------------------------------------

test-data/golden/test_sync_broadcast_alice1

@@ -1,7 +1,7 @@
 OutBroadcast#Chat#1001: Channel [0 member(s)]
 --------------------------------------------------------------------------------
 Msg#1001: info (Contact#Contact#Info): Messages are end-to-end encrypted. [NOTICED][INFO]
-Msg#1008🔒: Me (Contact#Contact#Self): hi  √
 Msg#1006🔒: Me (Contact#Contact#Self): Member bob@example.net added. [INFO] √
+Msg#1008🔒: Me (Contact#Contact#Self): hi  √
 Msg#1009🔒: Me (Contact#Contact#Self): You removed member bob@example.net. [INFO] √
 --------------------------------------------------------------------------------