chore(release): prepare for 1.156.1

Say in the README that cargo-bolero@0.8.0
should be installed as newer versions
do not work currently.

Update `mailparse` to 0.16.0
because this is what Delta Chat currently uses.

I also added fuzzer outputs to .gitignore

.github/workflows/deltachat-rpc-server.yml

@@ -35,7 +35,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Build deltachat-rpc-server binaries
         run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-linux
@@ -60,7 +59,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Build deltachat-rpc-server binaries
         run: nix build .#deltachat-rpc-server-${{ matrix.arch }}
@@ -112,7 +110,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Build deltachat-rpc-server binaries
         run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-android
@@ -140,7 +137,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Download Linux aarch64 binary
         uses: actions/download-artifact@v4

.github/workflows/nix.yml

@@ -24,7 +24,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - run: nix fmt
 
       # Check that formatting does not change anything.
@@ -85,7 +84,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - run: nix build .#${{ matrix.installable }}
 
   build-macos:
@@ -105,5 +103,4 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - run: nix build .#${{ matrix.installable }}

.github/workflows/repl.yml

@@ -19,7 +19,6 @@ jobs:
           show-progress: false
           persist-credentials: false
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - name: Build
         run: nix build .#deltachat-repl-win64
       - name: Upload binary

.github/workflows/upload-docs.yml

@@ -37,7 +37,6 @@ jobs:
           persist-credentials: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - name: Build Python documentation
         run: nix build .#python-docs
       - name: Upload to py.delta.chat
@@ -57,7 +56,6 @@ jobs:
           persist-credentials: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - uses: DeterminateSystems/nix-installer-action@main
-      - uses: DeterminateSystems/magic-nix-cache-action@main
       - name: Build C documentation
         run: nix build .#docs
       - name: Upload to c.delta.chat

.gitignore

@@ -1,7 +1,9 @@
-/target
+target/
 **/*.rs.bk
 /build
 /dist
+/fuzz/fuzz_targets/corpus/
+/fuzz/fuzz_targets/crashes/
 
 # ignore vi temporaries
 *~

CHANGELOG.md

@@ -1,5 +1,88 @@
 # Changelog
 
+## [1.156.1] - 2025-02-28
+
+### Fixes
+
+- Update mailparse to 0.16.1 to fix panic when parsing a message.
+- Add Chat-Group-Name-Timestamp header and use it to update group names ([#6412](https://github.com/deltachat/deltachat-core-rust/pull/6412)).
+- Log tokio::fs::metadata errors.
+
+### Build system
+
+- Update fuzzing setup.
+
+## [1.156.0] - 2025-02-26
+
+### API-Changes
+
+- Save messages API in JSON RPC ([#6554](https://github.com/deltachat/deltachat-core-rust/pull/6554)).
+- jsonrpc: Add `MessageObject.is_edited`.
+- jsonrpc: Add `send_edit_request`.
+- Deduplicate blob files in the JsonRPC API ([#6470](https://github.com/deltachat/deltachat-core-rust/pull/6470)).
+- Message deletion request API ([#6576](https://github.com/deltachat/deltachat-core-rust/pull/6576))
+
+### Features / Changes
+
+- Edit message's text ([#6550](https://github.com/deltachat/deltachat-core-rust/pull/6550))
+- Sync message deletion to other devices ([#6573](https://github.com/deltachat/deltachat-core-rust/pull/6573))
+- Allow scanning multiple securejoin QR codes in parallel.
+- When reactions are seen, remove notification from second device ([#6480](https://github.com/deltachat/deltachat-core-rust/pull/6480)).
+- Enable bcc-self automatically when doing Autocrypt Setup Message.
+- Don't send a notification when a group member left ([#6575](https://github.com/deltachat/deltachat-core-rust/pull/6575)).
+- Fail on too new backups ([#6580](https://github.com/deltachat/deltachat-core-rust/pull/6580)).
+
+### Fixes
+
+- Make it impossible to overwrite default key.
+- Do not allow to edit html messages ([#6564](https://github.com/deltachat/deltachat-core-rust/pull/6564)).
+- `get_config(Config::Selfavatar)` returns the path, not the name ([#6570](https://github.com/deltachat/deltachat-core-rust/pull/6570)).
+- `chat::save_msgs`: Interrupt inbox loop to send a sync message.
+- Do not delete files if cannot read their metadata.
+
+### Build system
+
+- nix: Update hashes of git dependencies.
+- Update some dependencies.
+
+### CI
+
+- Remove deprecated DeterminateSystems/magic-nix-cache-action.
+
+### Refactor
+
+- Use mail-builder instead of lettre_email.
+- Move even even more tests into their own files ([#6559](https://github.com/deltachat/deltachat-core-rust/pull/6559)).
+- Remove `Message.set_file()`, `dc_msg_set_file()` and related code ([#6558](https://github.com/deltachat/deltachat-core-rust/pull/6558)).
+- Remove unused blob functions ([#6563](https://github.com/deltachat/deltachat-core-rust/pull/6563)).
+- Let `BlobObject::from_name()` take `&str` ([#6571](https://github.com/deltachat/deltachat-core-rust/pull/6571)).
+- Don't use traits where it's not necessary ([#6567](https://github.com/deltachat/deltachat-core-rust/pull/6567)).
+
+## [1.155.6] - 2025-02-17
+
+### Features / Changes
+
+- Sort past members by the timestamp of removal.
+- Use UUID v4 to generate Message-IDs.
+
+### Fixes
+
+- Use dedicated ID for sync messages affecting device chat.
+- Do not allow non-members to change ephemeral timer settings.
+- Show padlock when the message is not sent over the network.
+
+### Build system
+
+- Remove deprecated node module.
+
+### CI
+
+- Audit workflows with zizmor.
+
+### Documentation
+
+- Improve docstrings ([#6496](https://github.com/deltachat/deltachat-core-rust/pull/6496)).
+
 ## [1.155.5] - 2025-02-14
 
 ### Fixes
@@ -5802,3 +5885,6 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.155.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.2..v1.155.3
 [1.155.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.3..v1.155.4
 [1.155.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.4..v1.155.5
+[1.155.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.5..v1.155.6
+[1.156.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.6..v1.156.0
+[1.156.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.156.0..v1.156.1

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.155.5"
+version = "1.156.1"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.81"
@@ -50,7 +50,6 @@ brotli = { version = "7", default-features=false, features = ["std"] }
 bytes = "1"
 chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 data-encoding = "2.7.0"
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = "0.2"
 escaper = "0.1"
 fast-socks5 = "0.10"
@@ -67,9 +66,9 @@ image = { version = "0.25.5", default-features=false, features = ["gif", "jpeg",
 iroh-gossip = { version = "0.32", default-features = false, features = ["net"] }
 iroh = { version = "0.32", default-features = false }
 kamadak-exif = "0.6.1"
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = { workspace = true }
-mailparse = "0.16"
+mail-builder = { git = "https://github.com/stalwartlabs/mail-builder", branch = "main", default-features = false }
+mailparse = "0.16.1"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
@@ -159,6 +158,10 @@ harness = false
 name = "get_chat_msgs"
 harness = false
 
+[[bench]]
+name = "marknoticed_chat"
+harness = false
+
 [[bench]]
 name = "get_chatlist"
 harness = false

README.md

@@ -139,7 +139,7 @@ $ cargo test -- --ignored
 
 Install [`cargo-bolero`](https://github.com/camshaft/bolero) with
 ```sh
-$ cargo install cargo-bolero
+$ cargo install cargo-bolero@0.8.0
 ```
 
 Run fuzzing tests with

benches/marknoticed_chat.rs

@@ -0,0 +1,94 @@
+#![recursion_limit = "256"]
+use std::path::Path;
+
+use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion};
+use deltachat::chat::{self, ChatId};
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::Events;
+use futures_lite::future::block_on;
+use tempfile::tempdir;
+
+async fn marknoticed_chat_benchmark(context: &Context, chats: &[ChatId]) {
+    for c in chats.iter().take(20) {
+        chat::marknoticed_chat(context, *c).await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
+        let chats: Vec<_> = rt.block_on(async {
+            let context = Context::new(Path::new(&path), 100, Events::new(), StockStrings::new())
+                .await
+                .unwrap();
+            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
+            let len = chatlist.len();
+            (1..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
+        });
+
+        // This mainly tests the performance of marknoticed_chat()
+        // when nothing has to be done
+        c.bench_function(
+            "chat::marknoticed_chat (mark 20 chats as noticed repeatedly)",
+            |b| {
+                let dir = tempdir().unwrap();
+                let dir = dir.path();
+                let new_db = dir.join("dc.db");
+                std::fs::copy(&path, &new_db).unwrap();
+
+                let context = block_on(async {
+                    Context::new(Path::new(&new_db), 100, Events::new(), StockStrings::new())
+                        .await
+                        .unwrap()
+                });
+
+                b.to_async(&rt)
+                    .iter(|| marknoticed_chat_benchmark(&context, black_box(&chats)))
+            },
+        );
+
+        // If the first 20 chats contain fresh messages or reactions,
+        // this tests the performance of marking them as noticed.
+        c.bench_function(
+            "chat::marknoticed_chat (mark 20 chats as noticed, resetting after every iteration)",
+            |b| {
+                b.to_async(&rt).iter_batched(
+                    || {
+                        let dir = tempdir().unwrap();
+                        let new_db = dir.path().join("dc.db");
+                        std::fs::copy(&path, &new_db).unwrap();
+
+                        let context = block_on(async {
+                            Context::new(
+                                Path::new(&new_db),
+                                100,
+                                Events::new(),
+                                StockStrings::new(),
+                            )
+                            .await
+                            .unwrap()
+                        });
+                        (dir, context)
+                    },
+                    |(_dir, context)| {
+                        let chats = &chats;
+                        async move {
+                            marknoticed_chat_benchmark(black_box(&context), black_box(chats)).await
+                        }
+                    },
+                    BatchSize::PerIteration,
+                );
+            },
+        );
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.155.5"
+version = "1.156.1"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -975,7 +975,7 @@ uint32_t        dc_get_chat_id_by_contact_id (dc_context_t* context, uint32_t co
  * ~~~
  * dc_msg_t* msg = dc_msg_new(context, DC_MSG_IMAGE);
  *
- * dc_msg_set_file(msg, "/file/to/send.jpg", NULL);
+ * dc_msg_set_file_and_deduplicate(msg, "/file/to/send.jpg", NULL, NULL);
  * dc_send_msg(context, chat_id, msg);
  *
  * dc_msg_unref(msg);
@@ -1039,6 +1039,38 @@ uint32_t        dc_send_msg_sync                  (dc_context_t* context, uint32
 uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t chat_id, const char* text_to_send);
 
 
+/**
+ * Send chat members a request to edit the given message's text.
+ *
+ * Only outgoing messages sent by self can be edited.
+ * Edited messages should be flagged as such in the UI, see dc_msg_is_edited().
+ * UI is informed about changes using the event #DC_EVENT_MSGS_CHANGED.
+ * If the text is not changed, no event and no edit request message are sent.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @param msg_id The message ID of the message to edit.
+ * @param new_text The new text.
+ *      This must not be NULL nor empty.
+ */
+void            dc_send_edit_request         (dc_context_t* context, uint32_t msg_id, const char* new_text);
+
+
+/**
+ * Send chat members a request to delete the given messages.
+ *
+ * Only outgoing messages can be deleted this way
+ * and all messages must be in the same chat.
+ * No tombstone or sth. like that is left.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @param msg_ids An array of uint32_t containing all message IDs to delete.
+ * @param msg_cnt The number of messages IDs in the msg_ids array.
+ */
+ void            dc_send_delete_request       (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
+
+
 /**
  * Send invitation to a videochat.
  *
@@ -1946,7 +1978,7 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
 
 
 /**
- * Delete messages. The messages are deleted on the current device and
+ * Delete messages. The messages are deleted on all devices and
  * on the IMAP server.
  *
  * @memberof dc_context_t
@@ -2459,8 +2491,9 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_MISMATCH           220 // id=contact
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
-#define         DC_QR_BACKUP                 251
+#define         DC_QR_BACKUP                 251 // deprecated
 #define         DC_QR_BACKUP2                252
+#define         DC_QR_BACKUP_TOO_NEW         255
 #define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
 #define         DC_QR_PROXY                  271 // text1=address (e.g. "127.0.0.1:9050")
 #define         DC_QR_ADDR                   320 // id=contact
@@ -4432,6 +4465,20 @@ int             dc_msg_is_sent                (const dc_msg_t* msg);
 int             dc_msg_is_forwarded           (const dc_msg_t* msg);
 
 
+/**
+ * Check if the message was edited.
+ *
+ * Edited messages should be marked by the UI as such,
+ * e.g. by the text "Edited" beside the time.
+ * To edit messages, use dc_send_edit_request().
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return 1=message is edited, 0=message not edited.
+ */
+ int             dc_msg_is_edited             (const dc_msg_t* msg);
+
+
 /**
  * Check if the message is an informational message, created by the
  * device or by another users. Such messages are not "typed" by the user but
@@ -4741,22 +4788,6 @@ void            dc_msg_set_subject            (dc_msg_t* msg, const char* subjec
 void            dc_msg_set_override_sender_name(dc_msg_t* msg, const char* name);
 
 
-/**
- * Set the file associated with a message object.
- * This does not alter any information in the database
- * nor copy or move the file or checks if the file exist.
- * All this can be done with dc_send_msg() later.
- *
- * @memberof dc_msg_t
- * @param msg The message object.
- * @param file If the message object is used in dc_send_msg() later,
- *     this must be the full path of the image file to send.
- * @param filemime The MIME type of the file. NULL if you don't know or don't care.
- * @deprecated 2025-01-21 Use dc_msg_set_file_and_deduplicate instead
- */
-void            dc_msg_set_file               (dc_msg_t* msg, const char* file, const char* filemime);
-
-
 /**
  * Sets the file associated with a message.
  *
@@ -4784,7 +4815,7 @@ void            dc_msg_set_file_and_deduplicate(dc_msg_t* msg, const char* file,
 
 /**
  * Set the dimensions associated with message object.
- * Typically this is the width and the height of an image or video associated using dc_msg_set_file().
+ * Typically this is the width and the height of an image or video associated using dc_msg_set_file_and_deduplicate().
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
  *
  * @memberof dc_msg_t
@@ -4797,7 +4828,7 @@ void            dc_msg_set_dimension          (dc_msg_t* msg, int width, int hei
 
 /**
  * Set the duration associated with message object.
- * Typically this is the duration of an audio or video associated using dc_msg_set_file().
+ * Typically this is the duration of an audio or video associated using dc_msg_set_file_and_deduplicate().
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
  *
  * @memberof dc_msg_t
@@ -5436,7 +5467,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
  * If you want to define the type of a dc_msg_t object for sending,
  * use dc_msg_new().
  * Depending on the type, you will set more properties using e.g.
- * dc_msg_set_text() or dc_msg_set_file().
+ * dc_msg_set_text() or dc_msg_set_file_and_deduplicate().
  * To finally send the message, use dc_send_msg().
  *
  * To get the types of dc_msg_t objects received, use dc_msg_get_viewtype().
@@ -5457,7 +5488,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 /**
  * Image message.
  * If the image is an animated GIF, the type #DC_MSG_GIF should be used.
- * File, width, and height are set via dc_msg_set_file(), dc_msg_set_dimension()
+ * File, width, and height are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_dimension()
  * and retrieved via dc_msg_get_file(), dc_msg_get_width(), and dc_msg_get_height().
  *
  * Before sending, the image is recoded to an reasonable size,
@@ -5470,7 +5501,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Animated GIF message.
- * File, width, and height are set via dc_msg_set_file(), dc_msg_set_dimension()
+ * File, width, and height are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_dimension()
  * and retrieved via dc_msg_get_file(), dc_msg_get_width(), and dc_msg_get_height().
  */
 #define DC_MSG_GIF       21
@@ -5488,7 +5519,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Message containing an audio file.
- * File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+ * File and duration are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_duration()
  * and retrieved via dc_msg_get_file(), and dc_msg_get_duration().
  */
 #define DC_MSG_AUDIO     40
@@ -5497,7 +5528,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 /**
  * A voice message that was directly recorded by the user.
  * For all other audio messages, the type #DC_MSG_AUDIO should be used.
- * File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+ * File and duration are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_duration()
  * and retrieved via dc_msg_get_file(), and dc_msg_get_duration().
  */
 #define DC_MSG_VOICE     41
@@ -5506,7 +5537,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 /**
  * Video messages.
  * File, width, height, and duration
- * are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
+ * are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_dimension(), dc_msg_set_duration()
  * and retrieved via
  * dc_msg_get_file(), dc_msg_get_width(),
  * dc_msg_get_height(), and dc_msg_get_duration().
@@ -5516,7 +5547,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Message containing any file, e.g. a PDF.
- * The file is set via dc_msg_set_file()
+ * The file is set via dc_msg_set_file_and_deduplicate()
  * and retrieved via dc_msg_get_file().
  */
 #define DC_MSG_FILE      60

deltachat-ffi/src/lib.rs

@@ -1041,6 +1041,42 @@ pub unsafe extern "C" fn dc_send_text_msg(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_send_edit_request(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    new_text: *const libc::c_char,
+) {
+    if context.is_null() || new_text.is_null() {
+        eprintln!("ignoring careless call to dc_send_edit_request()");
+        return;
+    }
+    let ctx = &*context;
+    let new_text = to_string_lossy(new_text);
+
+    block_on(chat::send_edit_request(ctx, MsgId::new(msg_id), new_text))
+        .unwrap_or_log_default(ctx, "Failed to send text edit")
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_send_delete_request(
+    context: *mut dc_context_t,
+    msg_ids: *const u32,
+    msg_cnt: libc::c_int,
+) {
+    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
+        eprintln!("ignoring careless call to dc_send_delete_request()");
+        return;
+    }
+    let ctx = &*context;
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
+
+    block_on(message::delete_msgs_ex(ctx, &msg_ids, true))
+        .context("failed dc_send_delete_request() call")
+        .log_err(ctx)
+        .ok();
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_send_videochat_invitation(
     context: *mut dc_context_t,
@@ -3683,6 +3719,16 @@ pub unsafe extern "C" fn dc_msg_is_forwarded(msg: *mut dc_msg_t) -> libc::c_int
     ffi_msg.message.is_forwarded().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_is_edited(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_is_edited()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.is_edited().into()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_is_info(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -3818,23 +3864,6 @@ pub unsafe extern "C" fn dc_msg_set_override_sender_name(
         .set_override_sender_name(to_opt_string_lossy(name))
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_set_file(
-    msg: *mut dc_msg_t,
-    file: *const libc::c_char,
-    filemime: *const libc::c_char,
-) {
-    if msg.is_null() || file.is_null() {
-        eprintln!("ignoring careless call to dc_msg_set_file()");
-        return;
-    }
-    let ffi_msg = &mut *msg;
-    ffi_msg.message.set_file(
-        to_string_lossy(file),
-        to_opt_string_lossy(filemime).as_deref(),
-    )
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_file_and_deduplicate(
     msg: *mut dc_msg_t,

deltachat-ffi/src/lot.rs

@@ -50,6 +50,7 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(Cow::Borrowed(fingerprint)),
                 Qr::Account { domain } => Some(Cow::Borrowed(domain)),
                 Qr::Backup2 { .. } => None,
+                Qr::BackupTooNew { .. } => None,
                 Qr::WebrtcInstance { domain, .. } => Some(Cow::Borrowed(domain)),
                 Qr::Proxy { host, port, .. } => Some(Cow::Owned(format!("{host}:{port}"))),
                 Qr::Addr { draft, .. } => draft.as_deref().map(Cow::Borrowed),
@@ -103,6 +104,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
                 Qr::Account { .. } => LotState::QrAccount,
                 Qr::Backup2 { .. } => LotState::QrBackup2,
+                Qr::BackupTooNew { .. } => LotState::QrBackupTooNew,
                 Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
                 Qr::Proxy { .. } => LotState::QrProxy,
                 Qr::Addr { .. } => LotState::QrAddr,
@@ -129,6 +131,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::Backup2 { .. } => Default::default(),
+                Qr::BackupTooNew { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
                 Qr::Proxy { .. } => Default::default(),
                 Qr::Addr { contact_id, .. } => contact_id.to_u32(),
@@ -178,10 +181,10 @@ pub enum LotState {
     /// text1=domain
     QrAccount = 250,
 
-    QrBackup = 251,
-
     QrBackup2 = 252,
 
+    QrBackupTooNew = 255,
+
     /// text1=domain, text2=instance pattern
     QrWebrtcInstance = 260,
 

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.155.5"
+version = "1.156.1"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"

deltachat-jsonrpc/src/api.rs

@@ -1305,6 +1305,12 @@ impl CommandApi {
         Ok(results)
     }
 
+    async fn save_msgs(&self, account_id: u32, message_ids: Vec<u32>) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let message_ids: Vec<MsgId> = message_ids.into_iter().map(MsgId::new).collect();
+        chat::save_msgs(&ctx, &message_ids).await
+    }
+
     // ---------------------------------------------
     //  contact
     // ---------------------------------------------
@@ -1944,7 +1950,7 @@ impl CommandApi {
         let ctx = self.get_context(account_id).await?;
 
         let mut msg = Message::new(Viewtype::Sticker);
-        msg.set_file(&sticker_path, None);
+        msg.set_file_and_deduplicate(&ctx, Path::new(&sticker_path), None, None)?;
 
         // JSON-rpc does not need heuristics to turn [Viewtype::Sticker] into [Viewtype::Image]
         msg.force_sticker();
@@ -1998,6 +2004,16 @@ impl CommandApi {
         Ok(msg_id)
     }
 
+    async fn send_edit_request(
+        &self,
+        account_id: u32,
+        msg_id: u32,
+        new_text: String,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        chat::send_edit_request(&ctx, MsgId::new(msg_id), new_text).await
+    }
+
     /// Checks if messages can be sent to a given chat.
     async fn can_send(&self, account_id: u32, chat_id: u32) -> Result<bool> {
         let ctx = self.get_context(account_id).await?;
@@ -2164,12 +2180,14 @@ impl CommandApi {
 
     // mimics the old desktop call, will get replaced with something better in the composer rewrite,
     // the better version will just be sending the current draft, though there will be probably something similar with more options to this for the corner cases like setting a marker on the map
+    #[expect(clippy::too_many_arguments)]
     async fn misc_send_msg(
         &self,
         account_id: u32,
         chat_id: u32,
         text: Option<String>,
         file: Option<String>,
+        filename: Option<String>,
         location: Option<(f64, f64)>,
         quoted_message_id: Option<u32>,
     ) -> Result<(u32, MessageObject)> {
@@ -2181,7 +2199,7 @@ impl CommandApi {
         });
         message.set_text(text.unwrap_or_default());
         if let Some(file) = file {
-            message.set_file(file, None);
+            message.set_file_and_deduplicate(&ctx, Path::new(&file), filename.as_deref(), None)?;
         }
         if let Some((latitude, longitude)) = location {
             message.set_location(latitude, longitude);
@@ -2209,12 +2227,14 @@ impl CommandApi {
     // the better version should support:
     // - changing viewtype to enable/disable compression
     // - keeping same message id as long as attachment does not change for webxdc messages
+    #[expect(clippy::too_many_arguments)]
     async fn misc_set_draft(
         &self,
         account_id: u32,
         chat_id: u32,
         text: Option<String>,
         file: Option<String>,
+        filename: Option<String>,
         quoted_message_id: Option<u32>,
         view_type: Option<MessageViewtype>,
     ) -> Result<()> {
@@ -2231,7 +2251,7 @@ impl CommandApi {
         ));
         draft.set_text(text.unwrap_or_default());
         if let Some(file) = file {
-            draft.set_file(file, None);
+            draft.set_file_and_deduplicate(&ctx, Path::new(&file), filename.as_deref(), None)?;
         }
         if let Some(id) = quoted_message_id {
             draft

deltachat-jsonrpc/src/api/types/message.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use crate::api::VcardContact;
 use anyhow::{Context as _, Result};
 use deltachat::chat::Chat;
@@ -37,6 +39,8 @@ pub struct MessageObject {
 
     text: String,
 
+    is_edited: bool,
+
     /// Check if a message has a POI location bound to it.
     /// These locations are also returned by `get_locations` method.
     /// The UI may decide to display a special icon beside such messages.
@@ -89,6 +93,10 @@ pub struct MessageObject {
 
     download_state: DownloadState,
 
+    original_msg_id: Option<u32>,
+
+    saved_message_id: Option<u32>,
+
     reactions: Option<JSONRPCReactions>,
 
     vcard_contact: Option<VcardContact>,
@@ -198,6 +206,7 @@ impl MessageObject {
             quote,
             parent_id,
             text: message.get_text(),
+            is_edited: message.is_edited(),
             has_location: message.has_location(),
             has_html: message.has_html(),
             view_type: message.get_viewtype().into(),
@@ -253,6 +262,16 @@ impl MessageObject {
 
             download_state,
 
+            original_msg_id: message
+                .get_original_msg_id(context)
+                .await?
+                .map(|id| id.to_u32()),
+
+            saved_message_id: message
+                .get_saved_msg_id(context)
+                .await?
+                .map(|id| id.to_u32()),
+
             reactions,
 
             vcard_contact: vcard_contacts.first().cloned(),
@@ -593,6 +612,7 @@ pub struct MessageData {
     pub html: Option<String>,
     pub viewtype: Option<MessageViewtype>,
     pub file: Option<String>,
+    pub filename: Option<String>,
     pub location: Option<(f64, f64)>,
     pub override_sender_name: Option<String>,
     /// Quoted message id. Takes preference over `quoted_text` (see below).
@@ -617,7 +637,12 @@ impl MessageData {
             message.set_override_sender_name(self.override_sender_name);
         }
         if let Some(file) = self.file {
-            message.set_file(file, None);
+            message.set_file_and_deduplicate(
+                context,
+                Path::new(&file),
+                self.filename.as_deref(),
+                None,
+            )?;
         }
         if let Some((latitude, longitude)) = self.location {
             message.set_location(latitude, longitude);

deltachat-jsonrpc/src/api/types/qr.rs

@@ -63,6 +63,7 @@ pub enum QrObject {
         /// Iroh node address.
         node_addr: String,
     },
+    BackupTooNew {},
     /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         domain: String,
@@ -100,11 +101,15 @@ pub enum QrObject {
     /// URL scanned.
     ///
     /// Ask the user if they want to open a browser or copy the URL to clipboard.
-    Url { url: String },
+    Url {
+        url: String,
+    },
     /// Text scanned.
     ///
     /// Ask the user if they want to copy the text to clipboard.
-    Text { text: String },
+    Text {
+        text: String,
+    },
     /// Ask the user if they want to withdraw their own QR code.
     WithdrawVerifyContact {
         /// Contact ID.
@@ -160,7 +165,9 @@ pub enum QrObject {
     /// `dclogin:` scheme parameters.
     ///
     /// Ask the user if they want to login with the email address.
-    Login { address: String },
+    Login {
+        address: String,
+    },
 }
 
 impl From<Qr> for QrObject {
@@ -217,6 +224,7 @@ impl From<Qr> for QrObject {
                 node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
                 auth_token,
             },
+            Qr::BackupTooNew {} => QrObject::BackupTooNew {},
             Qr::WebrtcInstance {
                 domain,
                 instance_pattern,

deltachat-jsonrpc/typescript/package.json

@@ -58,5 +58,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.155.5"
+  "version": "1.156.1"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.155.5"
+version = "1.156.1"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"

deltachat-repl/src/cmdline.rs

@@ -92,7 +92,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     context.emit_msgs_changed_without_ids();
 }
 
-async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
+async fn poke_eml_file(context: &Context, filename: &Path) -> Result<()> {
     let data = read_file(context, filename).await?;
 
     if let Err(err) = receive_imf(context, &data, false).await {
@@ -126,7 +126,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
         real_spec = rs.unwrap();
     }
     if let Some(suffix) = get_filesuffix_lc(&real_spec) {
-        if suffix == "eml" && poke_eml_file(context, &real_spec).await.is_ok() {
+        if suffix == "eml" && poke_eml_file(context, Path::new(&real_spec)).await.is_ok() {
             read_cnt += 1
         }
     } else {
@@ -140,7 +140,10 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
                     println!("Import: {path_plus_name}");
-                    if poke_eml_file(context, path_plus_name).await.is_ok() {
+                    if poke_eml_file(context, Path::new(&path_plus_name))
+                        .await
+                        .is_ok()
+                    {
                         read_cnt += 1
                     }
                 }
@@ -1278,7 +1281,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 
-            if let Ok(buf) = read_file(&context, &arg1).await {
+            if let Ok(buf) = read_file(&context, Path::new(arg1)).await {
                 let (width, height) = get_filemeta(&buf)?;
                 println!("width={width}, height={height}");
             } else {

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.155.5"
+version = "1.156.1"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -301,6 +301,13 @@ class Account:
             if event.kind == EventType.INCOMING_MSG:
                 return event
 
+    def wait_for_msgs_changed_event(self):
+        """Wait for messages changed event and return it."""
+        while True:
+            event = self.wait_for_event()
+            if event.kind == EventType.MSGS_CHANGED:
+                return event
+
     def wait_for_incoming_msg(self):
         """Wait for incoming message and return it.
 
@@ -352,3 +359,7 @@ class Account:
         """Import keys."""
         passphrase = ""  # Importing passphrase-protected keys is currently not supported.
         self._rpc.import_self_keys(self.id, str(path), passphrase)
+
+    def initiate_autocrypt_key_transfer(self) -> None:
+        """Send Autocrypt Setup Message."""
+        return self._rpc.initiate_autocrypt_key_transfer(self.id)

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -124,6 +124,7 @@ class Chat:
         html: Optional[str] = None,
         viewtype: Optional[ViewType] = None,
         file: Optional[str] = None,
+        filename: Optional[str] = None,
         location: Optional[tuple[float, float]] = None,
         override_sender_name: Optional[str] = None,
         quoted_msg: Optional[Union[int, Message]] = None,
@@ -137,6 +138,7 @@ class Chat:
             "html": html,
             "viewtype": viewtype,
             "file": file,
+            "filename": filename,
             "location": location,
             "overrideSenderName": override_sender_name,
             "quotedMessageId": quoted_msg,
@@ -172,13 +174,14 @@ class Chat:
         self,
         text: Optional[str] = None,
         file: Optional[str] = None,
+        filename: Optional[str] = None,
         quoted_msg: Optional[int] = None,
         viewtype: Optional[str] = None,
     ) -> None:
         """Set draft message."""
         if isinstance(quoted_msg, Message):
             quoted_msg = quoted_msg.id
-        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg, viewtype)
+        self._rpc.misc_set_draft(self.account.id, self.id, text, file, filename, quoted_msg, viewtype)
 
     def remove_draft(self) -> None:
         """Remove draft message."""

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -52,6 +52,9 @@ class Message:
         """Mark the message as seen."""
         self._rpc.markseen_msgs(self.account.id, [self.id])
 
+    def continue_autocrypt_key_transfer(self, setup_code: str) -> None:
+        self._rpc.continue_autocrypt_key_transfer(self.account.id, self.id, setup_code)
+
     def send_webxdc_status_update(self, update: Union[dict, str], description: str) -> None:
         """Send a webxdc status update. This message must be a webxdc."""
         if not isinstance(update, str):

deltachat-rpc-client/tests/test_key_transfer.py

@@ -0,0 +1,50 @@
+import pytest
+
+from deltachat_rpc_client import EventType
+from deltachat_rpc_client.rpc import JsonRpcError
+
+
+def wait_for_autocrypt_setup_message(account):
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.MSGS_CHANGED and event.msg_id != 0:
+            msg_id = event.msg_id
+            msg = account.get_message_by_id(msg_id)
+            if msg.get_snapshot().is_setupmessage:
+                return msg
+
+
+def test_autocrypt_setup_message_key_transfer(acfactory):
+    alice1 = acfactory.get_online_account()
+
+    alice2 = acfactory.get_unconfigured_account()
+    alice2.set_config("addr", alice1.get_config("addr"))
+    alice2.set_config("mail_pw", alice1.get_config("mail_pw"))
+    alice2.configure()
+
+    setup_code = alice1.initiate_autocrypt_key_transfer()
+    msg = wait_for_autocrypt_setup_message(alice2)
+
+    # Test that entering wrong code returns an error.
+    with pytest.raises(JsonRpcError):
+        msg.continue_autocrypt_key_transfer("7037-0673-6287-3013-4095-7956-5617-6806-6756")
+
+    msg.continue_autocrypt_key_transfer(setup_code)
+
+
+def test_ac_setup_message_twice(acfactory):
+    alice1 = acfactory.get_online_account()
+    alice2 = acfactory.get_unconfigured_account()
+    alice2.set_config("addr", alice1.get_config("addr"))
+    alice2.set_config("mail_pw", alice1.get_config("mail_pw"))
+    alice2.configure()
+
+    # Send the first Autocrypt Setup Message and ignore it.
+    _setup_code = alice1.initiate_autocrypt_key_transfer()
+    wait_for_autocrypt_setup_message(alice2)
+
+    # Send the second Autocrypt Setup Message and import it.
+    setup_code = alice1.initiate_autocrypt_key_transfer()
+    msg = wait_for_autocrypt_setup_message(alice2)
+
+    msg.continue_autocrypt_key_transfer(setup_code)

deltachat-rpc-client/tests/test_securejoin.py

@@ -4,6 +4,7 @@ import time
 import pytest
 
 from deltachat_rpc_client import Chat, EventType, SpecialContactId
+from deltachat_rpc_client.rpc import JsonRpcError
 
 
 def test_qr_setup_contact(acfactory, tmp_path) -> None:
@@ -26,17 +27,21 @@ def test_qr_setup_contact(acfactory, tmp_path) -> None:
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
-    # Test that if Bob changes the key, backwards verification is lost.
+    # Test that if Bob imports a key,
+    # backwards verification is not lost
+    # because default key is not changed.
     logging.info("Bob 2 is created")
     bob2 = acfactory.new_configured_account()
     bob2.export_self_keys(tmp_path)
 
-    logging.info("Bob imports a key")
-    bob.import_self_keys(tmp_path)
+    logging.info("Bob tries to import a key")
+    # Importing a second key is not allowed.
+    with pytest.raises(JsonRpcError):
+        bob.import_self_keys(tmp_path)
 
-    assert bob.get_config("key_id") == "2"
+    assert bob.get_config("key_id") == "1"
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
-    assert not bob_contact_alice_snapshot.is_verified
+    assert bob_contact_alice_snapshot.is_verified
 
 
 def test_qr_setup_contact_svg(acfactory) -> None:
@@ -653,12 +658,14 @@ def test_withdraw_securejoin_qr(acfactory):
     bob_chat = bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
+    alice.clear_all_events()
+
     snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
     assert snapshot.chat.get_basic_snapshot().is_protected
     bob_chat.leave()
 
-    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    snapshot = alice.get_message_by_id(alice.wait_for_msgs_changed_event().msg_id).get_snapshot()
     assert snapshot.text == "Group left by {}.".format(bob.get_config("addr"))
 
     logging.info("Alice withdraws QR code.")

deltachat-rpc-client/tests/test_something.py

@@ -287,6 +287,44 @@ def test_message(acfactory) -> None:
     assert reactions == snapshot.reactions
 
 
+def test_reaction_seen_on_another_dev(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+    alice.export_backup(tmp_path)
+    files = list(tmp_path.glob("*.tar"))
+    alice2 = acfactory.get_unconfigured_account()
+    alice2.import_backup(files[0])
+    alice2.start_io()
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello!")
+
+    event = bob.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+
+    message = bob.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    snapshot.chat.accept()
+    message.send_reaction("😎")
+    for a in [alice, alice2]:
+        while True:
+            event = a.wait_for_event()
+            if event.kind == EventType.INCOMING_REACTION:
+                break
+
+    alice2.clear_all_events()
+    alice_chat_bob.mark_noticed()
+    while True:
+        event = alice2.wait_for_event()
+        if event.kind == EventType.MSGS_NOTICED:
+            chat_id = event.chat_id
+            break
+    alice2_contact_bob = alice2.get_contact_by_addr(bob_addr)
+    alice2_chat_bob = alice2_contact_bob.create_chat()
+    assert chat_id == alice2_chat_bob.id
+
+
 def test_is_bot(acfactory) -> None:
     """Test that we can recognize messages submitted by bots."""
     alice, bob = acfactory.get_online_accounts(2)

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.155.5"
+version = "1.156.1"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "1.155.5"
+  "version": "1.156.1"
 }

deny.toml

@@ -1,7 +1,5 @@
 [advisories]
 ignore = [
-    "RUSTSEC-2020-0071",
-
     # Timing attack on RSA.
     # Delta Chat does not use RSA for new keys
     # and this requires precise measurement of the decryption time by the attacker.
@@ -9,9 +7,6 @@ ignore = [
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
 
-    # Unmaintained encoding
-    "RUSTSEC-2021-0153",
-
     # Unmaintained instant
     "RUSTSEC-2024-0384",
 
@@ -32,16 +27,12 @@ skip = [
      { name = "core-foundation", version = "0.9.4" },
      { name = "event-listener", version = "2.5.3" },
      { name = "generator", version = "0.7.5" },
-     { name = "getrandom", version = "<0.2" },
      { name = "http", version = "0.2.12" },
      { name = "loom", version = "0.5.6" },
      { name = "netlink-packet-route", version = "0.17.1" },
      { name = "nix", version = "0.26.4" },
      { name = "nix", version = "0.27.1" },
      { name = "quick-error", version = "<2.0" },
-     { name = "rand_chacha", version = "<0.3" },
-     { name = "rand_core", version = "<0.6" },
-     { name = "rand", version = "<0.8" },
      { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
@@ -51,11 +42,9 @@ skip = [
      { name = "syn", version = "1.0.109" },
      { name = "thiserror-impl", version = "1.0.69" },
      { name = "thiserror", version = "1.0.69" },
-     { name = "time", version = "<0.3" },
      { name = "tokio-tungstenite", version = "0.21.0" },
      { name = "tungstenite", version = "0.21.0" },
      { name = "unicode-width", version = "0.1.11" },
-     { name = "wasi", version = "<0.11" },
      { name = "windows" },
      { name = "windows_aarch64_gnullvm" },
      { name = "windows_aarch64_msvc" },
@@ -102,5 +91,5 @@ license-files = [
 [sources.allow-org]
 # Organisations which we allow git sources from.
 github = [
-       "deltachat",
+       "stalwartlabs",
 ]

flake.nix

@@ -88,8 +88,7 @@
         cargoLock = {
           lockFile = ./Cargo.lock;
           outputHashes = {
-            "email-0.0.20" = "sha256-cfR3D5jFQpw32bGsgapK2Uwuxmht+rRK/n1ZUmCb2WA=";
-            "lettre-0.9.2" = "sha256-+hU1cFacyyeC9UGVBpS14BWlJjHy90i/3ynMkKAzclk=";
+            "mail-builder-0.4.1" = "sha256-1hnsU76ProcX7iXT2UBjHnHbJ/ROT3077sLi3+yAV58=";
           };
         };
         mkRustPackage = packageName:

fuzz/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 bolero = "0.8"
 
 [dependencies]
-mailparse = "0.13"
+mailparse = "0.16"
 deltachat = { path = ".." }
 format-flowed = { path = "../format-flowed" }
 

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.155.5"
+version = "1.156.1"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.8"

python/src/deltachat/message.py

@@ -118,7 +118,7 @@ class Message:
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
             raise ValueError(f"path does not exist: {path!r}")
-        lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
+        lib.dc_msg_set_file_and_deduplicate(self._dc_msg, as_dc_charpointer(path), ffi.NULL, mtype)
 
     @props.with_doc
     def basename(self) -> str:
@@ -215,7 +215,7 @@ class Message:
         """extract key and use it as primary key for this account."""
         res = lib.dc_continue_key_transfer(self.account._dc_context, self.id, as_dc_charpointer(setup_code))
         if res == 0:
-            raise ValueError("could not decrypt")
+            raise ValueError("Importing the key from Autocrypt Setup Message failed")
 
     @props.with_doc
     def time_sent(self):

python/tests/test_0_complex_or_slow.py

@@ -510,6 +510,7 @@ def test_see_new_verified_member_after_going_online(acfactory, tmp_path, lp):
     """
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac2_addr = ac2.get_config("addr")
+    acfactory.remove_preconfigured_keys()
     ac1_offl = acfactory.new_online_configuring_account(cloned_from=ac1)
     for ac in [ac1, ac1_offl]:
         ac.set_config("bcc_self", "1")
@@ -560,6 +561,7 @@ def test_use_new_verified_group_after_going_online(acfactory, data, tmp_path, lp
       missing, cannot encrypt".
     """
     ac1, ac2 = acfactory.get_online_accounts(2)
+    acfactory.remove_preconfigured_keys()
     ac2_offl = acfactory.new_online_configuring_account(cloned_from=ac2)
     for ac in [ac2, ac2_offl]:
         ac.set_config("bcc_self", "1")
@@ -615,6 +617,7 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     - Now the seconds device has all members verified.
     """
     ac1, ac2 = acfactory.get_online_accounts(2)
+    acfactory.remove_preconfigured_keys()
     ac2_offl = acfactory.new_online_configuring_account(cloned_from=ac2)
     for ac in [ac2, ac2_offl]:
         ac.set_config("bcc_self", "1")

python/tests/test_1_online.py

@@ -85,27 +85,6 @@ def test_configure_unref(tmp_path):
     lib.dc_context_unref(dc_context)
 
 
-def test_export_import_self_keys(acfactory, tmp_path, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    dir = tmp_path / "exportdir"
-    dir.mkdir()
-    export_files = ac1.export_self_keys(str(dir))
-    assert len(export_files) == 2
-    for x in export_files:
-        assert x.startswith(str(dir))
-    (key_id,) = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
-    ac1._evtracker.consume_events()
-
-    lp.sec("exported keys (private and public)")
-    for name in dir.iterdir():
-        lp.indent(str(dir / name))
-    lp.sec("importing into existing account")
-    ac2.import_self_keys(str(dir))
-    (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*")
-    assert key_id2 == key_id
-
-
 def test_one_account_send_bcc_setting(acfactory, lp):
     ac1 = acfactory.new_online_configuring_account()
     ac2 = acfactory.new_online_configuring_account()
@@ -1597,53 +1576,6 @@ def test_import_export_online_all(acfactory, tmp_path, data, lp):
     assert ac2.get_latest_backupfile(str(backupdir)) == path2
 
 
-def test_ac_setup_message(acfactory, lp):
-    # note that the receiving account needs to be configured and running
-    # before the setup message is send. DC does not read old messages
-    # as of Jul2019
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
-
-    lp.sec("trigger ac setup message and return setupcode")
-    assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
-    setup_code = ac1.initiate_key_transfer()
-    ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-    msg = ac2.get_message_by_id(ev.data2)
-    assert msg.is_setup_message()
-    assert msg.get_setupcodebegin() == setup_code[:2]
-    lp.sec("try a bad setup code")
-    with pytest.raises(ValueError):
-        msg.continue_key_transfer(str(reversed(setup_code)))
-    lp.sec("try a good setup code")
-    print("*************** Incoming ASM File at: ", msg.filename)
-    print("*************** Setup Code: ", setup_code)
-    msg.continue_key_transfer(setup_code)
-    assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
-
-
-def test_ac_setup_message_twice(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
-
-    lp.sec("trigger ac setup message but ignore")
-    assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
-    ac1.initiate_key_transfer()
-    ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-
-    lp.sec("trigger second ac setup message, wait for receive ")
-    setup_code2 = ac1.initiate_key_transfer()
-    ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-    msg = ac2.get_message_by_id(ev.data2)
-    assert msg.is_setup_message()
-    assert msg.get_setupcodebegin() == setup_code2[:2]
-
-    lp.sec("process second setup message")
-    msg.continue_key_transfer(setup_code2)
-    assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
-
-
 def test_qr_email_capitalization(acfactory, lp):
     """Regression test for a bug
     that resulted in failure to propagate verification via gossip in a verified group

python/tests/test_3_offline.py

@@ -436,11 +436,11 @@ class TestOfflineChat:
         assert msg.id > 0
         assert msg.is_file()
         assert os.path.exists(msg.filename)
-        assert msg.filename.endswith(msg.basename)
+        assert msg.filename.endswith(".txt") == fn.endswith(".txt")
         assert msg.filemime == typeout
         msg2 = chat1.send_file(fp, typein)
         assert msg2 != msg
-        assert msg2.filename != msg.filename
+        assert msg2.filename == msg.filename
 
     def test_create_contact(self, acfactory):
         ac1 = acfactory.get_pseudo_configured_account()

release-date.in

@@ -1 +1 @@
-2025-02-14
+2025-02-28

scripts/set_core_version.py

@@ -67,7 +67,6 @@ def main():
     parser.add_argument("newversion")
 
     json_list = [
-        "package.json",
         "deltachat-jsonrpc/typescript/package.json",
         "deltachat-rpc-server/npm-package/package.json",
     ]

src/blob.rs

@@ -1,7 +1,6 @@
 //! # Blob directory management.
 
 use core::cmp::max;
-use std::ffi::OsStr;
 use std::io::{Cursor, Seek};
 use std::iter::FusedIterator;
 use std::mem;
@@ -14,8 +13,7 @@ use image::codecs::jpeg::JpegEncoder;
 use image::ImageReader;
 use image::{DynamicImage, GenericImage, GenericImageView, ImageFormat, Pixel, Rgba};
 use num_traits::FromPrimitive;
-use tokio::io::AsyncWriteExt;
-use tokio::{fs, io, task};
+use tokio::{fs, task};
 use tokio_stream::wrappers::ReadDirStream;
 
 use crate::config::Config;
@@ -48,73 +46,6 @@ enum ImageOutputFormat {
 }
 
 impl<'a> BlobObject<'a> {
-    /// Creates a new file, returning a tuple of the name and the handle.
-    async fn create_new_file(
-        context: &Context,
-        dir: &Path,
-        stem: &str,
-        ext: &str,
-    ) -> Result<(String, fs::File)> {
-        const MAX_ATTEMPT: u32 = 16;
-        let mut attempt = 0;
-        let mut name = format!("{stem}{ext}");
-        loop {
-            attempt += 1;
-            let path = dir.join(&name);
-            match fs::OpenOptions::new()
-                // Using `create_new(true)` in order to avoid race conditions
-                // when creating multiple files with the same name.
-                .create_new(true)
-                .write(true)
-                .open(&path)
-                .await
-            {
-                Ok(file) => return Ok((name, file)),
-                Err(err) => {
-                    if attempt >= MAX_ATTEMPT {
-                        return Err(err).context("failed to create file");
-                    } else if attempt == 1 && !dir.exists() {
-                        fs::create_dir_all(dir).await.log_err(context).ok();
-                    } else {
-                        name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
-                    }
-                }
-            }
-        }
-    }
-
-    /// Creates a new blob object with unique name by copying an existing file.
-    ///
-    /// This creates a new blob
-    /// and copies an existing file into it.  This is done in a
-    /// in way which avoids race-conditions when multiple files are
-    /// concurrently created.
-    pub async fn create_and_copy(context: &'a Context, src: &Path) -> Result<BlobObject<'a>> {
-        let mut src_file = fs::File::open(src)
-            .await
-            .with_context(|| format!("failed to open file {}", src.display()))?;
-        let (stem, ext) = BlobObject::sanitize_name_and_split_extension(&src.to_string_lossy());
-        let (name, mut dst_file) =
-            BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
-        let name_for_err = name.clone();
-        if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
-            // Attempt to remove the failed file, swallow errors resulting from that.
-            let path = context.get_blobdir().join(&name_for_err);
-            fs::remove_file(path).await.ok();
-            return Err(err).context("failed to copy file");
-        }
-
-        // Ensure that all buffered bytes are written
-        dst_file.flush().await?;
-
-        let blob = BlobObject {
-            blobdir: context.get_blobdir(),
-            name: format!("$BLOBDIR/{name}"),
-        };
-        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
-        Ok(blob)
-    }
-
     /// Creates a blob object by copying or renaming an existing file.
     /// If the source file is already in the blobdir, it will be renamed,
     /// otherwise it will be copied to the blobdir first.
@@ -209,27 +140,6 @@ impl<'a> BlobObject<'a> {
         })
     }
 
-    /// Creates a blob from a file, possibly copying it to the blobdir.
-    ///
-    /// If the source file is not a path to into the blob directory
-    /// the file will be copied into the blob directory first.  If the
-    /// source file is already in the blobdir it will not be copied
-    /// and only be created if it is a valid blobname, that is no
-    /// subdirectory is used and [BlobObject::sanitize_name_and_split_extension] does not
-    /// modify the filename.
-    ///
-    /// Paths into the blob directory may be either defined by an absolute path
-    /// or by the relative prefix `$BLOBDIR`.
-    pub async fn new_from_path(context: &'a Context, src: &Path) -> Result<BlobObject<'a>> {
-        if src.starts_with(context.get_blobdir()) {
-            BlobObject::from_path(context, src)
-        } else if src.starts_with("$BLOBDIR/") {
-            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
-        } else {
-            BlobObject::create_and_copy(context, src).await
-        }
-    }
-
     /// Returns a [BlobObject] for an existing blob from a path.
     ///
     /// The path must designate a file directly in the blobdir and
@@ -240,11 +150,11 @@ impl<'a> BlobObject<'a> {
         let rel_path = path
             .strip_prefix(context.get_blobdir())
             .with_context(|| format!("wrong blobdir: {}", path.display()))?;
-        if !BlobObject::is_acceptible_blob_name(rel_path) {
+        let name = rel_path.to_str().context("wrong name")?;
+        if !BlobObject::is_acceptible_blob_name(name) {
             return Err(format_err!("bad blob name: {}", rel_path.display()));
         }
-        let name = rel_path.to_str().context("wrong name")?;
-        BlobObject::from_name(context, name.to_string())
+        BlobObject::from_name(context, name)
     }
 
     /// Returns a [BlobObject] for an existing blob.
@@ -253,13 +163,13 @@ impl<'a> BlobObject<'a> {
     /// prefixed, as returned by [BlobObject::as_name].  This is how
     /// you want to create a [BlobObject] for a filename read from the
     /// database.
-    pub fn from_name(context: &'a Context, name: String) -> Result<BlobObject<'a>> {
-        let name: String = match name.starts_with("$BLOBDIR/") {
-            true => name.splitn(2, '/').last().unwrap().to_string(),
+    pub fn from_name(context: &'a Context, name: &str) -> Result<BlobObject<'a>> {
+        let name = match name.starts_with("$BLOBDIR/") {
+            true => name.splitn(2, '/').last().unwrap(),
             false => name,
         };
-        if !BlobObject::is_acceptible_blob_name(&name) {
-            return Err(format_err!("not an acceptable blob name: {}", &name));
+        if !BlobObject::is_acceptible_blob_name(name) {
+            return Err(format_err!("not an acceptable blob name: {}", name));
         }
         Ok(BlobObject {
             blobdir: context.get_blobdir(),
@@ -288,16 +198,6 @@ impl<'a> BlobObject<'a> {
         &self.name
     }
 
-    /// Returns the filename of the blob.
-    pub fn as_file_name(&self) -> &str {
-        self.name.rsplit('/').next().unwrap_or_default()
-    }
-
-    /// The path relative in the blob directory.
-    pub fn as_rel_path(&self) -> &Path {
-        Path::new(self.as_file_name())
-    }
-
     /// Returns the extension of the blob.
     ///
     /// If a blob's filename has an extension, it is always guaranteed
@@ -311,67 +211,21 @@ impl<'a> BlobObject<'a> {
         }
     }
 
-    /// The name is returned as a tuple, the first part
-    /// being the stem or basename and the second being an extension,
-    /// including the dot.  E.g. "foo.txt" is returned as `("foo",
-    /// ".txt")` while "bar" is returned as `("bar", "")`.
-    ///
-    /// The extension part will always be lowercased.
-    fn sanitize_name_and_split_extension(name: &str) -> (String, String) {
-        let name = sanitize_filename(name);
-        // Let's take a tricky filename,
-        // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
-        // Assume that the extension is 32 chars maximum.
-        let ext: String = name
-            .chars()
-            .rev()
-            .take_while(|c| {
-                (!c.is_ascii_punctuation() || *c == '.') && !c.is_whitespace() && !c.is_control()
-            })
-            .take(33)
-            .collect::<Vec<_>>()
-            .iter()
-            .rev()
-            .collect();
-        // ext == "nd_point_and_double_ending.tar.gz"
-
-        // Split it into "nd_point_and_double_ending" and "tar.gz":
-        let mut iter = ext.splitn(2, '.');
-        iter.next();
-
-        let ext = iter.next().unwrap_or_default();
-        let ext = if ext.is_empty() {
-            String::new()
-        } else {
-            format!(".{ext}")
-            // ".tar.gz"
-        };
-        let stem = name
-            .strip_suffix(&ext)
-            .unwrap_or_default()
-            .chars()
-            .take(64)
-            .collect();
-        (stem, ext.to_lowercase())
-    }
-
     /// Checks whether a name is a valid blob name.
     ///
     /// This is slightly less strict than stanitise_name, presumably
     /// someone already created a file with such a name so we just
-    /// ensure it's not actually a path in disguise is actually utf-8.
-    fn is_acceptible_blob_name(name: impl AsRef<OsStr>) -> bool {
-        let uname = match name.as_ref().to_str() {
-            Some(name) => name,
-            None => return false,
-        };
-        if uname.find('/').is_some() {
+    /// ensure it's not actually a path in disguise.
+    ///
+    /// Acceptible blob name always have to be valid utf-8.
+    fn is_acceptible_blob_name(name: &str) -> bool {
+        if name.find('/').is_some() {
             return false;
         }
-        if uname.find('\\').is_some() {
+        if name.find('\\').is_some() {
             return false;
         }
-        if uname.find('\0').is_some() {
+        if name.find('\0').is_some() {
             return false;
         }
         true

src/blob/blob_tests.rs

@@ -2,6 +2,7 @@ use std::time::Duration;
 
 use super::*;
 use crate::message::{Message, Viewtype};
+use crate::param::Param;
 use crate::sql;
 use crate::test_utils::{self, TestContext};
 use crate::tools::SystemTime;
@@ -44,20 +45,6 @@ async fn test_lowercase_ext() {
     );
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_as_file_name() {
-    let t = TestContext::new().await;
-    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
-    assert_eq!(blob.as_file_name(), FILE_DEDUPLICATED);
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_as_rel_path() {
-    let t = TestContext::new().await;
-    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
-    assert_eq!(blob.as_rel_path(), Path::new(FILE_DEDUPLICATED));
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_suffix() {
     let t = TestContext::new().await;
@@ -113,59 +100,19 @@ async fn test_create_long_names() {
     let t = TestContext::new().await;
     let s = format!("file.{}", "a".repeat(100));
     let blob = BlobObject::create_and_deduplicate_from_bytes(&t, b"data", &s).unwrap();
-    let blobname = blob.as_name().split('/').last().unwrap();
+    let blobname = blob.as_name().split('/').next_back().unwrap();
     assert!(blobname.len() < 70);
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_create_and_copy() {
-    let t = TestContext::new().await;
-    let src = t.dir.path().join("src");
-    fs::write(&src, b"boo").await.unwrap();
-    let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
-    assert_eq!(blob.as_name(), "$BLOBDIR/src");
-    let data = fs::read(blob.to_abs_path()).await.unwrap();
-    assert_eq!(data, b"boo");
-
-    let whoops = t.dir.path().join("whoops");
-    assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
-        .await
-        .is_err());
-    let whoops = t.get_blobdir().join("whoops");
-    assert!(!whoops.exists());
-}
-
-#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-async fn test_create_from_path() {
-    let t = TestContext::new().await;
-
-    let src_ext = t.dir.path().join("external");
-    fs::write(&src_ext, b"boo").await.unwrap();
-    let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-        .await
-        .unwrap();
-    assert_eq!(blob.as_name(), "$BLOBDIR/external");
-    let data = fs::read(blob.to_abs_path()).await.unwrap();
-    assert_eq!(data, b"boo");
-
-    let src_int = t.get_blobdir().join("internal");
-    fs::write(&src_int, b"boo").await.unwrap();
-    let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
-    assert_eq!(blob.as_name(), "$BLOBDIR/internal");
-    let data = fs::read(blob.to_abs_path()).await.unwrap();
-    assert_eq!(data, b"boo");
-}
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_create_from_name_long() {
     let t = TestContext::new().await;
     let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
     fs::write(&src_ext, b"boo").await.unwrap();
-    let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-        .await
-        .unwrap();
+    let blob = BlobObject::create_and_deduplicate(&t, &src_ext, &src_ext).unwrap();
     assert_eq!(
         blob.as_name(),
-        "$BLOBDIR/autocrypt-setup-message-4137848473.html"
+        "$BLOBDIR/06f010b24d1efe57ffab44a8ad20c54.html"
     );
 }
 
@@ -173,70 +120,12 @@ async fn test_create_from_name_long() {
 fn test_is_blob_name() {
     assert!(BlobObject::is_acceptible_blob_name("foo"));
     assert!(BlobObject::is_acceptible_blob_name("foo.txt"));
-    assert!(BlobObject::is_acceptible_blob_name("f".repeat(128)));
+    assert!(BlobObject::is_acceptible_blob_name(&"f".repeat(128)));
     assert!(!BlobObject::is_acceptible_blob_name("foo/bar"));
     assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
     assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
 }
 
-#[test]
-fn test_sanitise_name() {
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension(
-        "Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt",
-    );
-    assert_eq!(ext, ".txt");
-    assert!(!stem.is_empty());
-
-    // the extensions are kept together as between stem and extension a number may be added -
-    // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension("wot.tar.gz");
-    assert_eq!(stem, "wot");
-    assert_eq!(ext, ".tar.gz");
-
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension(".foo.bar");
-    assert_eq!(stem, "file");
-    assert_eq!(ext, ".foo.bar");
-
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension("foo?.bar");
-    assert!(stem.contains("foo"));
-    assert!(!stem.contains('?'));
-    assert_eq!(ext, ".bar");
-
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension("no-extension");
-    assert_eq!(stem, "no-extension");
-    assert_eq!(ext, "");
-
-    let (stem, ext) =
-        BlobObject::sanitize_name_and_split_extension("path/ignored\\this: is* forbidden?.c");
-    assert_eq!(ext, ".c");
-    assert!(!stem.contains("path"));
-    assert!(!stem.contains("ignored"));
-    assert!(stem.contains("this"));
-    assert!(stem.contains("forbidden"));
-    assert!(!stem.contains('/'));
-    assert!(!stem.contains('\\'));
-    assert!(!stem.contains(':'));
-    assert!(!stem.contains('*'));
-    assert!(!stem.contains('?'));
-
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension(
-        "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz",
-    );
-    assert_eq!(
-        stem,
-        "file.with_lots_of_characters_behind_point_and_double_ending"
-    );
-    assert_eq!(ext, ".tar.gz");
-
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension("a. tar.tar.gz");
-    assert_eq!(stem, "a. tar");
-    assert_eq!(ext, ".tar.gz");
-
-    let (stem, ext) = BlobObject::sanitize_name_and_split_extension("Guia_uso_GNB (v0.8).pdf");
-    assert_eq!(stem, "Guia_uso_GNB (v0.8)");
-    assert_eq!(ext, ".pdf");
-}
-
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_add_white_bg() {
     let t = TestContext::new().await;
@@ -249,7 +138,7 @@ async fn test_add_white_bg() {
         let avatar_src = t.dir.path().join("avatar.png");
         fs::write(&avatar_src, bytes).await.unwrap();
 
-        let mut blob = BlobObject::new_from_path(&t, &avatar_src).await.unwrap();
+        let mut blob = BlobObject::create_and_deduplicate(&t, &avatar_src, &avatar_src).unwrap();
         let img_wh = 128;
         let maybe_sticker = &mut false;
         let strict_limits = true;
@@ -298,7 +187,7 @@ async fn test_selfavatar_outside_blobdir() {
         constants::BALANCED_AVATAR_SIZE,
     );
 
-    let mut blob = BlobObject::new_from_path(&t, avatar_path).await.unwrap();
+    let mut blob = BlobObject::create_and_deduplicate(&t, avatar_path, avatar_path).unwrap();
     let maybe_sticker = &mut false;
     let strict_limits = true;
     blob.recode_to_size(&t, None, maybe_sticker, 1000, 3000, strict_limits)
@@ -337,7 +226,7 @@ async fn test_selfavatar_in_blobdir() {
         .unwrap();
     let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
     assert!(
-        avatar_cfg.ends_with("9e7f409ac5c92b942cc4f31cee2770a.png"),
+        avatar_cfg.ends_with("fa7418e646301203538041f60d03190.png"),
         "Avatar file name {avatar_cfg} should end with its hash"
     );
 
@@ -655,6 +544,12 @@ impl SendImageCheckMediaquality<'_> {
         alice_msg.save_file(&alice, &file_saved).await?;
         check_image_size(file_saved, compressed_width, compressed_height);
 
+        if original_width == compressed_width {
+            assert_extension(&alice, alice_msg, extension);
+        } else {
+            assert_extension(&alice, alice_msg, "jpg");
+        }
+
         let bob_msg = bob.recv_msg(&sent).await;
         assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
         assert_eq!(bob_msg.get_width() as u32, compressed_width);
@@ -673,10 +568,52 @@ impl SendImageCheckMediaquality<'_> {
         assert!(exif.is_none());
 
         let img = check_image_size(file_saved, compressed_width, compressed_height);
+
+        if original_width == compressed_width {
+            assert_extension(&bob, bob_msg, extension);
+        } else {
+            assert_extension(&bob, bob_msg, "jpg");
+        }
+
         Ok(img)
     }
 }
 
+fn assert_extension(context: &TestContext, msg: Message, extension: &str) {
+    assert!(msg
+        .param
+        .get(Param::File)
+        .unwrap()
+        .ends_with(&format!(".{extension}")));
+    assert!(msg
+        .param
+        .get(Param::Filename)
+        .unwrap()
+        .ends_with(&format!(".{extension}")));
+    assert!(msg
+        .get_filename()
+        .unwrap()
+        .ends_with(&format!(".{extension}")));
+    assert_eq!(
+        msg.get_file(context)
+            .unwrap()
+            .extension()
+            .unwrap()
+            .to_str()
+            .unwrap(),
+        extension
+    );
+    assert_eq!(
+        msg.param
+            .get_file_blob(context)
+            .unwrap()
+            .unwrap()
+            .suffix()
+            .unwrap(),
+        extension
+    );
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_send_big_gif_as_image() -> Result<()> {
     let bytes = include_bytes!("../../test-data/image/screenshot.gif");

src/chat.rs

@@ -3,6 +3,7 @@
 use std::cmp;
 use std::collections::{HashMap, HashSet};
 use std::fmt;
+use std::io::Cursor;
 use std::marker::Sync;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
@@ -11,6 +12,7 @@ use std::time::Duration;
 use anyhow::{anyhow, bail, ensure, Context as _, Result};
 use deltachat_contact_tools::{sanitize_bidi_characters, sanitize_single_line, ContactAddress};
 use deltachat_derive::{FromSql, ToSql};
+use mail_builder::mime::MimePart;
 use serde::{Deserialize, Serialize};
 use strum_macros::EnumIter;
 use tokio::task;
@@ -18,12 +20,11 @@ use tokio::task;
 use crate::aheader::EncryptPreference;
 use crate::blob::BlobObject;
 use crate::chatlist::Chatlist;
-use crate::chatlist_events;
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{
     self, Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK,
-    DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS,
+    DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_RESEND_USER_AVATAR_DAYS, EDITED_PREFIX,
     TIMESTAMP_SENT_TOLERANCE,
 };
 use crate::contact::{self, Contact, ContactId, Origin};
@@ -32,7 +33,6 @@ use crate::debug_logging::maybe_set_logging_xdc;
 use crate::download::DownloadState;
 use crate::ephemeral::{start_chat_ephemeral_timers, Timer as EphemeralTimer};
 use crate::events::EventType;
-use crate::html::new_html_mimepart;
 use crate::location;
 use crate::log::LogExt;
 use crate::message::{self, Message, MessageState, MsgId, Viewtype};
@@ -41,7 +41,6 @@ use crate::mimeparser::SystemMessage;
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::receive_imf::ReceivedMsg;
-use crate::securejoin::BobState;
 use crate::smtp::send_msg_to_smtp;
 use crate::stock_str;
 use crate::sync::{self, Sync::*, SyncData};
@@ -51,6 +50,7 @@ use crate::tools::{
     truncate_msg_text, IsNoneOrEmpty, SystemTime,
 };
 use crate::webxdc::StatusUpdateSerial;
+use crate::{chatlist_events, imap};
 
 /// An chat item, such as a message or a marker.
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -890,12 +890,6 @@ impl ChatId {
                 }
             }
             _ => {
-                let blob = msg
-                    .param
-                    .get_blob(Param::File, context)
-                    .await?
-                    .context("no file stored in params")?;
-                msg.param.set(Param::File, blob.as_name());
                 if msg.viewtype == Viewtype::File {
                     if let Some((better_type, _)) = message::guess_msgtype_from_suffix(msg)
                         // We do not do an automatic conversion to other viewtypes here so that
@@ -908,6 +902,10 @@ impl ChatId {
                     }
                 }
                 if msg.viewtype == Viewtype::Vcard {
+                    let blob = msg
+                        .param
+                        .get_file_blob(context)?
+                        .context("no file stored in params")?;
                     msg.try_set_vcard(context, &blob.to_abs_path()).await?;
                 }
             }
@@ -1050,6 +1048,14 @@ impl ChatId {
         Ok(count)
     }
 
+    pub(crate) async fn created_timestamp(self, context: &Context) -> Result<i64> {
+        Ok(context
+            .sql
+            .query_get_value("SELECT created_timestamp FROM chats WHERE id=?", (self,))
+            .await?
+            .unwrap_or(0))
+    }
+
     /// Returns timestamp of the latest message in the chat.
     pub(crate) async fn get_timestamp(self, context: &Context) -> Result<Option<i64>> {
         let timestamp = context
@@ -2157,14 +2163,18 @@ impl Chat {
         } else {
             None
         };
-        let new_mime_headers = new_mime_headers.map(|s| new_html_mimepart(s).build().as_string());
+        let new_mime_headers: Option<String> = new_mime_headers.map(|s| {
+            let html_part = MimePart::new("text/html", s);
+            let mut buffer = Vec::new();
+            let cursor = Cursor::new(&mut buffer);
+            html_part.write_part(cursor).ok();
+            String::from_utf8_lossy(&buffer).to_string()
+        });
         let new_mime_headers = new_mime_headers.or_else(|| match was_truncated {
             // We need to add some headers so that they are stripped before formatting HTML by
             // `MsgId::get_html()`, not a part of the actual text. Let's add "Content-Type", it's
             // anyway a useful metadata about the stored text.
-            true => Some(
-                "Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text + "\r\n",
-            ),
+            true => Some("Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text),
             false => None,
         });
         let new_mime_headers = match new_mime_headers {
@@ -2298,6 +2308,10 @@ impl Chat {
     async fn get_sync_id(&self, context: &Context) -> Result<Option<SyncId>> {
         match self.typ {
             Chattype::Single => {
+                if self.is_device_talk() {
+                    return Ok(Some(SyncId::Device));
+                }
+
                 let mut r = None;
                 for contact_id in get_chat_contacts(context, self.id).await? {
                     if contact_id == ContactId::SELF && !self.is_self_talk() {
@@ -2560,19 +2574,27 @@ pub(crate) async fn update_special_chat_names(context: &Context) -> Result<()> {
 /// Checks if there is a 1:1 chat in-progress SecureJoin for Bob and, if necessary, schedules a task
 /// unblocking the chat and notifying the user accordingly.
 pub(crate) async fn resume_securejoin_wait(context: &Context) -> Result<()> {
-    let Some(bobstate) = BobState::from_db(&context.sql).await? else {
-        return Ok(());
-    };
-    if !bobstate.in_progress() {
-        return Ok(());
-    }
-    let chat_id = bobstate.alice_chat();
-    let chat = Chat::load_from_db(context, chat_id).await?;
-    let timeout = chat
-        .check_securejoin_wait(context, constants::SECUREJOIN_WAIT_TIMEOUT)
+    let chat_ids: Vec<ChatId> = context
+        .sql
+        .query_map(
+            "SELECT chat_id FROM bobstate",
+            (),
+            |row| {
+                let chat_id: ChatId = row.get(0)?;
+                Ok(chat_id)
+            },
+            |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
         .await?;
-    if timeout > 0 {
-        chat_id.spawn_securejoin_wait(context, timeout);
+
+    for chat_id in chat_ids {
+        let chat = Chat::load_from_db(context, chat_id).await?;
+        let timeout = chat
+            .check_securejoin_wait(context, constants::SECUREJOIN_WAIT_TIMEOUT)
+            .await?;
+        if timeout > 0 {
+            chat_id.spawn_securejoin_wait(context, timeout);
+        }
     }
     Ok(())
 }
@@ -2736,8 +2758,7 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
     } else if msg.viewtype.has_file() {
         let mut blob = msg
             .param
-            .get_blob(Param::File, context)
-            .await?
+            .get_file_blob(context)?
             .with_context(|| format!("attachment missing for message of type #{}", msg.viewtype))?;
         let send_as_is = msg.viewtype == Viewtype::File;
 
@@ -2785,20 +2806,12 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
                 .recode_to_image_size(context, msg.get_filename(), &mut maybe_sticker)
                 .await?;
             msg.param.set(Param::Filename, new_name);
+            msg.param.set(Param::File, blob.as_name());
 
             if !maybe_sticker {
                 msg.viewtype = Viewtype::Image;
             }
         }
-        msg.param.set(Param::File, blob.as_name());
-        if let (Some(filename), Some(blob_ext)) = (msg.param.get(Param::Filename), blob.suffix()) {
-            let stem = match filename.rsplit_once('.') {
-                Some((stem, _)) => stem,
-                None => filename,
-            };
-            msg.param
-                .set(Param::Filename, stem.to_string() + "." + blob_ext);
-        }
 
         if !msg.param.exists(Param::MimeType) {
             if let Some((_, mime)) = message::guess_msgtype_from_suffix(msg) {
@@ -3020,10 +3033,17 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             context,
             "Message {} has no recipient, skipping smtp-send.", msg.id
         );
+        msg.param.set_int(Param::GuaranteeE2ee, 1);
+        msg.update_param(context).await?;
         msg.id.set_delivered(context).await?;
         msg.state = MessageState::OutDelivered;
         return Ok(Vec::new());
     }
+    if msg.param.get_cmd() == SystemMessage::GroupNameChanged {
+        msg.chat_id
+            .update_timestamp(context, Param::GroupNameTimestamp, msg.timestamp_sort)
+            .await?;
+    }
 
     let rendered_msg = match mimefactory.render(context).await {
         Ok(res) => Ok(res),
@@ -3124,6 +3144,66 @@ pub async fn send_text_msg(
     send_msg(context, chat_id, &mut msg).await
 }
 
+/// Sends chat members a request to edit the given message's text.
+pub async fn send_edit_request(context: &Context, msg_id: MsgId, new_text: String) -> Result<()> {
+    let mut original_msg = Message::load_from_db(context, msg_id).await?;
+    ensure!(
+        original_msg.from_id == ContactId::SELF,
+        "Can edit only own messages"
+    );
+    ensure!(!original_msg.is_info(), "Cannot edit info messages");
+    ensure!(!original_msg.has_html(), "Cannot edit HTML messages");
+    ensure!(
+        original_msg.viewtype != Viewtype::VideochatInvitation,
+        "Cannot edit videochat invitations"
+    );
+    ensure!(
+        !original_msg.text.is_empty(), // avoid complexity in UI element changes. focus is typos and rewordings
+        "Cannot add text"
+    );
+    ensure!(!new_text.trim().is_empty(), "Edited text cannot be empty");
+    if original_msg.text == new_text {
+        info!(context, "Text unchanged.");
+        return Ok(());
+    }
+
+    save_text_edit_to_db(context, &mut original_msg, &new_text).await?;
+
+    let mut edit_msg = Message::new_text(EDITED_PREFIX.to_owned() + &new_text); // prefix only set for nicer display in Non-Delta-MUAs
+    edit_msg.set_quote(context, Some(&original_msg)).await?; // quote only set for nicer display in Non-Delta-MUAs
+    if original_msg.get_showpadlock() {
+        edit_msg.param.set_int(Param::GuaranteeE2ee, 1);
+    }
+    edit_msg
+        .param
+        .set(Param::TextEditFor, original_msg.rfc724_mid);
+    edit_msg.hidden = true;
+    send_msg(context, original_msg.chat_id, &mut edit_msg).await?;
+    Ok(())
+}
+
+pub(crate) async fn save_text_edit_to_db(
+    context: &Context,
+    original_msg: &mut Message,
+    new_text: &str,
+) -> Result<()> {
+    original_msg.param.set_int(Param::IsEdited, 1);
+    context
+        .sql
+        .execute(
+            "UPDATE msgs SET txt=?, txt_normalized=?, param=? WHERE id=?",
+            (
+                new_text,
+                message::normalize_text(new_text),
+                original_msg.param.to_string(),
+                original_msg.id,
+            ),
+        )
+        .await?;
+    context.emit_msgs_changed(original_msg.chat_id, original_msg.id);
+    Ok(())
+}
+
 /// Sends invitation to a videochat.
 pub async fn send_videochat_invitation(context: &Context, chat_id: ChatId) -> Result<MsgId> {
     ensure!(
@@ -3328,7 +3408,7 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()>
     } else {
         start_chat_ephemeral_timers(context, chat_id).await?;
 
-        if context
+        let noticed_msgs_count = context
             .sql
             .execute(
                 "UPDATE msgs
@@ -3338,9 +3418,36 @@ pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()>
             AND chat_id=?;",
                 (MessageState::InNoticed, MessageState::InFresh, chat_id),
             )
-            .await?
-            == 0
-        {
+            .await?;
+
+        // This is to trigger emitting `MsgsNoticed` on other devices when reactions are noticed
+        // locally (i.e. when the chat was opened locally).
+        let hidden_messages = context
+            .sql
+            .query_map(
+                "SELECT id, rfc724_mid FROM msgs
+                    WHERE state=?
+                      AND hidden=1
+                      AND chat_id=?
+                    ORDER BY id LIMIT 100", // LIMIT to 100 in order to avoid blocking the UI too long, usually there will be less than 100 messages anyway
+                (MessageState::InFresh, chat_id), // No need to check for InNoticed messages, because reactions are never InNoticed
+                |row| {
+                    let msg_id: MsgId = row.get(0)?;
+                    let rfc724_mid: String = row.get(1)?;
+                    Ok((msg_id, rfc724_mid))
+                },
+                |rows| {
+                    rows.collect::<std::result::Result<Vec<_>, _>>()
+                        .map_err(Into::into)
+                },
+            )
+            .await?;
+        for (msg_id, rfc724_mid) in &hidden_messages {
+            message::update_msg_state(context, *msg_id, MessageState::InSeen).await?;
+            imap::markseen_on_imap_table(context, rfc724_mid).await?;
+        }
+
+        if noticed_msgs_count == 0 {
             return Ok(());
         }
     }
@@ -3493,6 +3600,8 @@ pub async fn get_chat_contacts(context: &Context, chat_id: ChatId) -> Result<Vec
 }
 
 /// Returns a vector of contact IDs for given chat ID that are no longer part of the group.
+///
+/// Members that have been removed recently are in the beginning of the list.
 pub async fn get_past_chat_contacts(context: &Context, chat_id: ChatId) -> Result<Vec<ContactId>> {
     let now = time();
     let list = context
@@ -3505,7 +3614,7 @@ pub async fn get_past_chat_contacts(context: &Context, chat_id: ChatId) -> Resul
              WHERE cc.chat_id=?
              AND cc.add_timestamp < cc.remove_timestamp
              AND ? < cc.remove_timestamp
-             ORDER BY c.id=1, c.last_seen DESC, c.id DESC",
+             ORDER BY c.id=1, cc.remove_timestamp DESC, c.id DESC",
             (chat_id, now.saturating_sub(60 * 24 * 3600)),
             |row| row.get::<_, ContactId>(0),
             |ids| ids.collect::<Result<Vec<_>, _>>().map_err(Into::into),
@@ -4270,7 +4379,7 @@ pub async fn save_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             })
             .await?;
     }
-    context.send_sync_msg().await?;
+    context.scheduler.interrupt_inbox().await;
     Ok(())
 }
 
@@ -4748,6 +4857,9 @@ pub(crate) enum SyncId {
     Grpid(String),
     /// "Message-ID"-s, from oldest to latest. Used for ad-hoc groups.
     Msgids(Vec<String>),
+
+    // Special id for device chat.
+    Device,
 }
 
 /// An action synchronised to other devices.
@@ -4809,6 +4921,7 @@ impl Context {
                 ChatId::lookup_by_message(&msg)
                     .with_context(|| format!("No chat found for Message-IDs {msgids:?}"))?
             }
+            SyncId::Device => ChatId::get_for_contact(self, ContactId::DEVICE).await?,
         };
         match action {
             SyncAction::Block => chat_id.block_ex(self, Nosync).await,

src/chat/chat_tests.rs

@@ -695,16 +695,45 @@ async fn test_leave_group() -> Result<()> {
 
     assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2);
 
+    // Clear events so that we can later check
+    // that the 'Group left' message didn't trigger IncomingMsg:
+    alice.evtracker.clear_events();
+
+    // Shift the time so that we can later check the 'Group left' message's timestamp:
+    SystemTime::shift(Duration::from_secs(60));
+
     // Bob leaves the group.
     let bob_chat_id = bob_msg.chat_id;
     bob_chat_id.accept(&bob).await?;
     remove_contact_from_chat(&bob, bob_chat_id, ContactId::SELF).await?;
 
     let leave_msg = bob.pop_sent_msg().await;
-    alice.recv_msg(&leave_msg).await;
+    let rcvd_leave_msg = alice.recv_msg(&leave_msg).await;
 
     assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 1);
 
+    assert_eq!(rcvd_leave_msg.state, MessageState::InSeen);
+
+    alice.emit_event(EventType::Test);
+    alice
+        .evtracker
+        .get_matching(|ev| match ev {
+            EventType::Test => true,
+            EventType::IncomingMsg { .. } => panic!("'Group left' message should be silent"),
+            EventType::MsgsNoticed(..) => {
+                panic!("'Group left' message shouldn't clear notifications")
+            }
+            _ => false,
+        })
+        .await;
+
+    // The 'Group left' message timestamp should be the same as the previous message in the chat
+    // so that the chat is not popped up in the chatlist:
+    assert_eq!(
+        sent_msg.load_from_db().await.timestamp_sort,
+        rcvd_leave_msg.timestamp_sort
+    );
+
     Ok(())
 }
 
@@ -774,6 +803,21 @@ async fn test_self_talk() -> Result<()> {
     Ok(())
 }
 
+/// Tests that when BCC-self is disabled
+/// and no messages are actually sent
+/// in a self-chat, they have a padlock.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_self_talk_no_bcc_padlock() -> Result<()> {
+    let t = &TestContext::new_alice().await;
+    t.set_config_bool(Config::BccSelf, false).await?;
+    let chat = &t.get_self_chat().await;
+
+    let msg_id = send_text_msg(t, chat.id, "Foobar".to_string()).await?;
+    let msg = Message::load_from_db(t, msg_id).await?;
+    assert!(msg.get_showpadlock());
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_add_device_msg_unlabelled() {
     let t = TestContext::new().await;
@@ -3059,6 +3103,34 @@ async fn test_sync_visibility() -> Result<()> {
     Ok(())
 }
 
+/// Tests syncing of chat visibility on device message chat.
+///
+/// Previously due to a bug pinning "Device Messages"
+/// chat resulted in creation of `device@localhost` chat
+/// on another device.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sync_device_messages_visibility() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice0 = &tcm.alice().await;
+    let alice1 = &tcm.alice().await;
+    for a in [alice0, alice1] {
+        a.set_config_bool(Config::SyncMsgs, true).await?;
+    }
+
+    let device_chat_id0 = ChatId::get_for_contact(alice0, ContactId::DEVICE).await?;
+    device_chat_id0
+        .set_visibility(alice0, ChatVisibility::Pinned)
+        .await?;
+
+    sync(alice0, alice1).await;
+
+    let device_chat_id1 = ChatId::get_for_contact(alice1, ContactId::DEVICE).await?;
+    let device_chat1 = Chat::load_from_db(alice1, device_chat_id1).await?;
+    assert_eq!(device_chat1.get_visibility(), ChatVisibility::Pinned);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_sync_muted() -> Result<()> {
     let alice0 = &TestContext::new_alice().await;
@@ -3418,6 +3490,61 @@ async fn test_expire_past_members_after_60_days() -> Result<()> {
     Ok(())
 }
 
+/// Test that past members are ordered by the timestamp of their removal.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_past_members_order() -> Result<()> {
+    let t = &TestContext::new_alice().await;
+
+    let bob_contact_id = Contact::create(t, "Bob", "bob@example.net").await?;
+    let charlie_contact_id = Contact::create(t, "Charlie", "charlie@example.org").await?;
+    let fiona_contact_id = Contact::create(t, "Fiona", "fiona@example.net").await?;
+
+    let chat_id = create_group_chat(t, ProtectionStatus::Unprotected, "Group chat").await?;
+    add_contact_to_chat(t, chat_id, bob_contact_id).await?;
+    add_contact_to_chat(t, chat_id, charlie_contact_id).await?;
+    add_contact_to_chat(t, chat_id, fiona_contact_id).await?;
+    t.send_text(chat_id, "Hi! I created a group.").await;
+
+    assert_eq!(get_past_chat_contacts(t, chat_id).await?.len(), 0);
+
+    remove_contact_from_chat(t, chat_id, charlie_contact_id).await?;
+
+    let past_contacts = get_past_chat_contacts(t, chat_id).await?;
+    assert_eq!(past_contacts.len(), 1);
+    assert_eq!(past_contacts[0], charlie_contact_id);
+
+    SystemTime::shift(Duration::from_secs(5));
+    remove_contact_from_chat(t, chat_id, bob_contact_id).await?;
+
+    let past_contacts = get_past_chat_contacts(t, chat_id).await?;
+    assert_eq!(past_contacts.len(), 2);
+    assert_eq!(past_contacts[0], bob_contact_id);
+    assert_eq!(past_contacts[1], charlie_contact_id);
+
+    SystemTime::shift(Duration::from_secs(5));
+    remove_contact_from_chat(t, chat_id, fiona_contact_id).await?;
+
+    let past_contacts = get_past_chat_contacts(t, chat_id).await?;
+    assert_eq!(past_contacts.len(), 3);
+    assert_eq!(past_contacts[0], fiona_contact_id);
+    assert_eq!(past_contacts[1], bob_contact_id);
+    assert_eq!(past_contacts[2], charlie_contact_id);
+
+    // Adding and removing Bob
+    // moves him to the top of past member list.
+    SystemTime::shift(Duration::from_secs(5));
+    add_contact_to_chat(t, chat_id, bob_contact_id).await?;
+    remove_contact_from_chat(t, chat_id, bob_contact_id).await?;
+
+    let past_contacts = get_past_chat_contacts(t, chat_id).await?;
+    assert_eq!(past_contacts.len(), 3);
+    assert_eq!(past_contacts[0], bob_contact_id);
+    assert_eq!(past_contacts[1], fiona_contact_id);
+    assert_eq!(past_contacts[2], charlie_contact_id);
+
+    Ok(())
+}
+
 /// Test the case when Alice restores a backup older than 60 days
 /// with outdated member list.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -3547,3 +3674,172 @@ async fn test_one_to_one_chat_no_group_member_timestamps() {
     let payload = sent.payload;
     assert!(!payload.contains("Chat-Group-Member-Timestamps:"));
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_send_edit_request() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_chat = alice.create_chat(bob).await;
+
+    // Alice sends a message with typos, followed by a correction message
+    let sent1 = alice.send_text(alice_chat.id, "zext me in delra.cat").await;
+    let alice_msg = sent1.load_from_db().await;
+    assert_eq!(alice_msg.text, "zext me in delra.cat");
+
+    send_edit_request(alice, alice_msg.id, "Text me on Delta.Chat".to_string()).await?;
+    let sent2 = alice.pop_sent_msg().await;
+    let test = Message::load_from_db(alice, alice_msg.id).await?;
+    assert_eq!(test.text, "Text me on Delta.Chat");
+
+    // Bob receives both messages and has the correct text at the end
+    let bob_msg = bob.recv_msg(&sent1).await;
+    assert_eq!(bob_msg.text, "zext me in delra.cat");
+
+    bob.recv_msg_opt(&sent2).await;
+    let test = Message::load_from_db(bob, bob_msg.id).await?;
+    assert_eq!(test.text, "Text me on Delta.Chat");
+
+    // alice has another device, and sees the correction also there
+    let alice2 = tcm.alice().await;
+    let alice2_msg = alice2.recv_msg(&sent1).await;
+    assert_eq!(alice2_msg.text, "zext me in delra.cat");
+
+    alice2.recv_msg_opt(&sent2).await;
+    let test = Message::load_from_db(&alice2, alice2_msg.id).await?;
+    assert_eq!(test.text, "Text me on Delta.Chat");
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_edit_request_after_removal() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_chat = alice.create_chat(bob).await;
+
+    // Alice sends a messag with typos, followed by a correction message
+    let sent1 = alice.send_text(alice_chat.id, "zext me in delra.cat").await;
+    let alice_msg = sent1.load_from_db().await;
+    send_edit_request(alice, alice_msg.id, "Text me on Delta.Chat".to_string()).await?;
+    let sent2 = alice.pop_sent_msg().await;
+
+    // Bob receives first message, deletes it and then ignores the correction
+    let bob_msg = bob.recv_msg(&sent1).await;
+    let bob_chat_id = bob_msg.chat_id;
+    assert_eq!(bob_msg.text, "zext me in delra.cat");
+    assert_eq!(bob_chat_id.get_msg_cnt(bob).await?, 1);
+
+    delete_msgs(bob, &[bob_msg.id]).await?;
+    assert_eq!(bob_chat_id.get_msg_cnt(bob).await?, 0);
+
+    bob.recv_msg_trash(&sent2).await;
+    assert_eq!(bob_chat_id.get_msg_cnt(bob).await?, 0);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_cannot_send_edit_request() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let chat_id = alice
+        .create_group_with_members(ProtectionStatus::Unprotected, "My Group", &[bob])
+        .await;
+
+    // Alice can edit her message
+    let sent1 = alice.send_text(chat_id, "foo").await;
+    send_edit_request(alice, sent1.sender_msg_id, "bar".to_string()).await?;
+
+    // Bob cannot edit Alice's message
+    let msg = bob.recv_msg(&sent1).await;
+    assert!(send_edit_request(bob, msg.id, "bar".to_string())
+        .await
+        .is_err());
+
+    // HTML messages cannot be edited
+    let mut msg = Message::new_text("plain text".to_string());
+    msg.set_html(Some("<b>html</b> text".to_string()));
+    let sent2 = alice.send_msg(chat_id, &mut msg).await;
+    assert!(msg.has_html());
+    assert!(
+        send_edit_request(alice, sent2.sender_msg_id, "foo".to_string())
+            .await
+            .is_err()
+    );
+
+    // Info messages cannot be edited
+    set_chat_name(alice, chat_id, "bar").await?;
+    let msg = alice.get_last_msg().await;
+    assert!(msg.is_info());
+    assert_eq!(msg.from_id, ContactId::SELF);
+    assert!(send_edit_request(alice, msg.id, "bar".to_string())
+        .await
+        .is_err());
+
+    // Videochat invitations cannot be edited
+    alice
+        .set_config(Config::WebrtcInstance, Some("https://foo.bar"))
+        .await?;
+    let msg_id = send_videochat_invitation(alice, chat_id).await?;
+    assert!(send_edit_request(alice, msg_id, "bar".to_string())
+        .await
+        .is_err());
+
+    // If not text was given initally, there is nothing to edit
+    // (this also avoids complexity in UI element changes; focus is typos and rewordings)
+    let mut msg = Message::new(Viewtype::File);
+    msg.make_vcard(alice, &[ContactId::SELF]).await?;
+    let sent3 = alice.send_msg(chat_id, &mut msg).await;
+    assert!(msg.text.is_empty());
+    assert!(
+        send_edit_request(alice, sent3.sender_msg_id, "bar".to_string())
+            .await
+            .is_err()
+    );
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_send_delete_request() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_chat = alice.create_chat(bob).await;
+    let bob_chat = bob.create_chat(alice).await;
+
+    // Bobs sends a message to Alice, so Alice learns Bob's key
+    let sent0 = bob.send_text(bob_chat.id, "¡ola!").await;
+    alice.recv_msg(&sent0).await;
+
+    // Alice sends a message, then sends a deletion request
+    let sent1 = alice.send_text(alice_chat.id, "wtf").await;
+    let alice_msg = sent1.load_from_db().await;
+    assert_eq!(alice_chat.id.get_msg_cnt(alice).await?, 2);
+
+    message::delete_msgs_ex(alice, &[alice_msg.id], true).await?;
+    let sent2 = alice.pop_sent_msg().await;
+    assert_eq!(alice_chat.id.get_msg_cnt(alice).await?, 1);
+
+    // Bob receives both messages and has nothing the end
+    let bob_msg = bob.recv_msg(&sent1).await;
+    assert_eq!(bob_msg.text, "wtf");
+    assert_eq!(bob_msg.chat_id.get_msg_cnt(bob).await?, 2);
+
+    bob.recv_msg_opt(&sent2).await;
+    assert_eq!(bob_msg.chat_id.get_msg_cnt(bob).await?, 1);
+
+    // Alice has another device, and there is also nothing at the end
+    let alice2 = &tcm.alice().await;
+    alice2.recv_msg(&sent0).await;
+    let alice2_msg = alice2.recv_msg(&sent1).await;
+    assert_eq!(alice2_msg.chat_id.get_msg_cnt(alice2).await?, 2);
+
+    alice2.recv_msg_opt(&sent2).await;
+    assert_eq!(alice2_msg.chat_id.get_msg_cnt(alice2).await?, 1);
+
+    Ok(())
+}

src/config.rs

@@ -963,397 +963,4 @@ fn get_config_keys_string() -> String {
 }
 
 #[cfg(test)]
-mod tests {
-    use num_traits::FromPrimitive;
-
-    use super::*;
-    use crate::test_utils::{sync, TestContext, TestContextManager};
-
-    #[test]
-    fn test_to_string() {
-        assert_eq!(Config::MailServer.to_string(), "mail_server");
-        assert_eq!(Config::from_str("mail_server"), Ok(Config::MailServer));
-
-        assert_eq!(Config::SysConfigKeys.to_string(), "sys.config_keys");
-        assert_eq!(
-            Config::from_str("sys.config_keys"),
-            Ok(Config::SysConfigKeys)
-        );
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_set_config_addr() {
-        let t = TestContext::new().await;
-
-        // Test that uppercase address get lowercased.
-        assert!(t
-            .set_config(Config::Addr, Some("Foobar@eXample.oRg"))
-            .await
-            .is_ok());
-        assert_eq!(
-            t.get_config(Config::Addr).await.unwrap().unwrap(),
-            "foobar@example.org"
-        );
-    }
-
-    /// Tests that "bot" config can only be set to "0" or "1".
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_set_config_bot() {
-        let t = TestContext::new().await;
-
-        assert!(t.set_config(Config::Bot, None).await.is_ok());
-        assert!(t.set_config(Config::Bot, Some("0")).await.is_ok());
-        assert!(t.set_config(Config::Bot, Some("1")).await.is_ok());
-        assert!(t.set_config(Config::Bot, Some("2")).await.is_err());
-        assert!(t.set_config(Config::Bot, Some("Foobar")).await.is_err());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_media_quality_config_option() {
-        let t = TestContext::new().await;
-        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
-        assert_eq!(media_quality, 0);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Balanced);
-
-        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
-
-        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
-        assert_eq!(media_quality, 1);
-        assert_eq!(constants::MediaQuality::Worse as i32, 1);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Worse);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ui_config() -> Result<()> {
-        let t = TestContext::new().await;
-
-        assert_eq!(t.get_ui_config("ui.desktop.linux.systray").await?, None);
-
-        t.set_ui_config("ui.android.screen_security", Some("safe"))
-            .await?;
-        assert_eq!(
-            t.get_ui_config("ui.android.screen_security").await?,
-            Some("safe".to_string())
-        );
-
-        t.set_ui_config("ui.android.screen_security", None).await?;
-        assert_eq!(t.get_ui_config("ui.android.screen_security").await?, None);
-
-        assert!(t.set_ui_config("configured", Some("bar")).await.is_err());
-
-        Ok(())
-    }
-
-    /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_set_config_bool() -> Result<()> {
-        let t = TestContext::new().await;
-
-        // We need some config that defaults to true
-        let c = Config::E2eeEnabled;
-        assert_eq!(t.get_config_bool(c).await?, true);
-        t.set_config_bool(c, false).await?;
-        assert_eq!(t.get_config_bool(c).await?, false);
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_self_addrs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        assert!(alice.is_self_addr("alice@example.org").await?);
-        assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
-        assert!(!alice.is_self_addr("alice@alice.com").await?);
-
-        // Test adding the same primary address
-        alice.set_primary_self_addr("alice@example.org").await?;
-        alice.set_primary_self_addr("Alice@Example.Org").await?;
-        assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
-
-        // Test adding a new (primary) self address
-        // The address is trimmed during configure by `LoginParam::from_database()`,
-        // so `set_primary_self_addr()` doesn't have to trim it.
-        alice.set_primary_self_addr("Alice@alice.com").await?;
-        assert!(alice.is_self_addr("aliCe@example.org").await?);
-        assert!(alice.is_self_addr("alice@alice.com").await?);
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["Alice@alice.com", "Alice@Example.Org"]
-        );
-
-        // Check that the entry is not duplicated
-        alice.set_primary_self_addr("alice@alice.com").await?;
-        alice.set_primary_self_addr("alice@alice.com").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@alice.com", "Alice@Example.Org"]
-        );
-
-        // Test switching back
-        alice.set_primary_self_addr("alice@example.org").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@example.org", "alice@alice.com"]
-        );
-
-        // Test setting a new primary self address, the previous self address
-        // should be kept as a secondary self address
-        alice.set_primary_self_addr("alice@alice.xyz").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
-        );
-        assert!(alice.is_self_addr("alice@example.org").await?);
-        assert!(alice.is_self_addr("alice@alice.com").await?);
-        assert!(alice.is_self_addr("Alice@alice.xyz").await?);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_mdns_default_behaviour() -> Result<()> {
-        let t = &TestContext::new_alice().await;
-        assert!(t.should_request_mdns().await?);
-        assert!(t.should_send_mdns().await?);
-        assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
-        // The setting should be displayed correctly.
-        assert!(t.get_config_bool(Config::MdnsEnabled).await?);
-
-        t.set_config_bool(Config::Bot, true).await?;
-        assert!(!t.should_request_mdns().await?);
-        assert!(t.should_send_mdns().await?);
-        assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
-        assert!(t.get_config_bool(Config::MdnsEnabled).await?);
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_delete_server_after_default() -> Result<()> {
-        let t = &TestContext::new_alice().await;
-
-        // Check that the settings are displayed correctly.
-        assert_eq!(t.get_config(Config::BccSelf).await?, Some("1".to_string()));
-        assert_eq!(
-            t.get_config(Config::DeleteServerAfter).await?,
-            Some("0".to_string())
-        );
-
-        // Leaving emails on the server even w/o `BccSelf` is a good default at least because other
-        // MUAs do so even if the server doesn't save sent messages to some sentbox (like Gmail
-        // does).
-        t.set_config_bool(Config::BccSelf, false).await?;
-        assert_eq!(
-            t.get_config(Config::DeleteServerAfter).await?,
-            Some("0".to_string())
-        );
-        Ok(())
-    }
-
-    const SAVED_MESSAGES_DEDUPLICATED_FILE: &str = "969142cb84015bc135767bc2370934a.png";
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_sync() -> Result<()> {
-        let alice0 = TestContext::new_alice().await;
-        let alice1 = TestContext::new_alice().await;
-        for a in [&alice0, &alice1] {
-            a.set_config_bool(Config::SyncMsgs, true).await?;
-        }
-
-        let mdns_enabled = alice0.get_config_bool(Config::MdnsEnabled).await?;
-        // Alice1 has a different config value.
-        alice1
-            .set_config_bool(Config::MdnsEnabled, !mdns_enabled)
-            .await?;
-        // This changes nothing, but still sends a sync message.
-        alice0
-            .set_config_bool(Config::MdnsEnabled, mdns_enabled)
-            .await?;
-        sync(&alice0, &alice1).await;
-        assert_eq!(
-            alice1.get_config_bool(Config::MdnsEnabled).await?,
-            mdns_enabled
-        );
-
-        // Reset to default. Test that it's not synced because defaults may differ across client
-        // versions.
-        alice0.set_config(Config::MdnsEnabled, None).await?;
-        alice0.set_config_bool(Config::MdnsEnabled, false).await?;
-        sync(&alice0, &alice1).await;
-        assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
-
-        for key in [Config::ShowEmails, Config::MvboxMove] {
-            let val = alice0.get_config_bool(key).await?;
-            alice0.set_config_bool(key, !val).await?;
-            sync(&alice0, &alice1).await;
-            assert_eq!(alice1.get_config_bool(key).await?, !val);
-        }
-
-        // `Config::SyncMsgs` mustn't be synced.
-        alice0.set_config_bool(Config::SyncMsgs, false).await?;
-        alice0.set_config_bool(Config::SyncMsgs, true).await?;
-        alice0.set_config_bool(Config::MdnsEnabled, true).await?;
-        sync(&alice0, &alice1).await;
-        assert!(alice1.get_config_bool(Config::MdnsEnabled).await?);
-
-        // Usual sync scenario.
-        async fn test_config_str(
-            alice0: &TestContext,
-            alice1: &TestContext,
-            key: Config,
-            val: &str,
-        ) -> Result<()> {
-            alice0.set_config(key, Some(val)).await?;
-            sync(alice0, alice1).await;
-            assert_eq!(alice1.get_config(key).await?, Some(val.to_string()));
-            Ok(())
-        }
-        test_config_str(&alice0, &alice1, Config::Displayname, "Alice Sync").await?;
-        test_config_str(&alice0, &alice1, Config::Selfstatus, "My status").await?;
-
-        assert!(alice0.get_config(Config::Selfavatar).await?.is_none());
-        let file = alice0.dir.path().join("avatar.png");
-        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        tokio::fs::write(&file, bytes).await?;
-        alice0
-            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
-            .await?;
-        sync(&alice0, &alice1).await;
-        // There was a bug that a sync message creates the self-chat with the user avatar instead of
-        // the special icon and that remains so when the self-chat becomes user-visible. Let's check
-        // this.
-        let self_chat = alice0.get_self_chat().await;
-        let self_chat_avatar_path = self_chat.get_profile_image(&alice0).await?.unwrap();
-        assert_eq!(
-            self_chat_avatar_path,
-            alice0.get_blobdir().join(SAVED_MESSAGES_DEDUPLICATED_FILE)
-        );
-        assert!(alice1
-            .get_config(Config::Selfavatar)
-            .await?
-            .filter(|path| path.ends_with(".png"))
-            .is_some());
-        alice0.set_config(Config::Selfavatar, None).await?;
-        sync(&alice0, &alice1).await;
-        assert!(alice1.get_config(Config::Selfavatar).await?.is_none());
-
-        Ok(())
-    }
-
-    /// Sync message mustn't be sent if self-{status,avatar} is changed by a self-sent message.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_no_sync_on_self_sent_msg() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice0 = &tcm.alice().await;
-        let alice1 = &tcm.alice().await;
-        for a in [alice0, alice1] {
-            a.set_config_bool(Config::SyncMsgs, true).await?;
-        }
-
-        let status = "Synced via usual message";
-        alice0.set_config(Config::Selfstatus, Some(status)).await?;
-        alice0.send_sync_msg().await?;
-        alice0.pop_sent_sync_msg().await;
-        let status1 = "Synced via sync message";
-        alice1.set_config(Config::Selfstatus, Some(status1)).await?;
-        tcm.send_recv(alice0, alice1, "hi Alice!").await;
-        assert_eq!(
-            alice1.get_config(Config::Selfstatus).await?,
-            Some(status.to_string())
-        );
-        sync(alice1, alice0).await;
-        assert_eq!(
-            alice0.get_config(Config::Selfstatus).await?,
-            Some(status1.to_string())
-        );
-
-        // Need a chat with another contact to send self-avatar.
-        let bob = &tcm.bob().await;
-        let a0b_chat_id = tcm.send_recv_accept(bob, alice0, "hi").await.chat_id;
-        let file = alice0.dir.path().join("avatar.png");
-        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        tokio::fs::write(&file, bytes).await?;
-        alice0
-            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
-            .await?;
-        alice0.send_sync_msg().await?;
-        alice0.pop_sent_sync_msg().await;
-        let file = alice1.dir.path().join("avatar.jpg");
-        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        tokio::fs::write(&file, bytes).await?;
-        alice1
-            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
-            .await?;
-        let sent_msg = alice0.send_text(a0b_chat_id, "hi").await;
-        alice1.recv_msg(&sent_msg).await;
-        assert!(alice1
-            .get_config(Config::Selfavatar)
-            .await?
-            .filter(|path| path.ends_with(".png"))
-            .is_some());
-        sync(alice1, alice0).await;
-        assert!(alice0
-            .get_config(Config::Selfavatar)
-            .await?
-            .filter(|path| path.ends_with(".jpg"))
-            .is_some());
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_event_config_synced() -> Result<()> {
-        let alice0 = TestContext::new_alice().await;
-        let alice1 = TestContext::new_alice().await;
-        for a in [&alice0, &alice1] {
-            a.set_config_bool(Config::SyncMsgs, true).await?;
-        }
-
-        alice0
-            .set_config(Config::Displayname, Some("Alice Sync"))
-            .await?;
-        alice0
-            .evtracker
-            .get_matching(|e| {
-                matches!(
-                    e,
-                    EventType::ConfigSynced {
-                        key: Config::Displayname
-                    }
-                )
-            })
-            .await;
-        sync(&alice0, &alice1).await;
-        assert_eq!(
-            alice1.get_config(Config::Displayname).await?,
-            Some("Alice Sync".to_string())
-        );
-        alice1
-            .evtracker
-            .get_matching(|e| {
-                matches!(
-                    e,
-                    EventType::ConfigSynced {
-                        key: Config::Displayname
-                    }
-                )
-            })
-            .await;
-
-        alice0.set_config(Config::Displayname, None).await?;
-        alice0
-            .evtracker
-            .get_matching(|e| {
-                matches!(
-                    e,
-                    EventType::ConfigSynced {
-                        key: Config::Displayname
-                    }
-                )
-            })
-            .await;
-
-        Ok(())
-    }
-}
+mod config_tests;

src/config/config_tests.rs

@@ -0,0 +1,392 @@
+use num_traits::FromPrimitive;
+
+use super::*;
+use crate::test_utils::{sync, TestContext, TestContextManager};
+
+#[test]
+fn test_to_string() {
+    assert_eq!(Config::MailServer.to_string(), "mail_server");
+    assert_eq!(Config::from_str("mail_server"), Ok(Config::MailServer));
+
+    assert_eq!(Config::SysConfigKeys.to_string(), "sys.config_keys");
+    assert_eq!(
+        Config::from_str("sys.config_keys"),
+        Ok(Config::SysConfigKeys)
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_set_config_addr() {
+    let t = TestContext::new().await;
+
+    // Test that uppercase address get lowercased.
+    assert!(t
+        .set_config(Config::Addr, Some("Foobar@eXample.oRg"))
+        .await
+        .is_ok());
+    assert_eq!(
+        t.get_config(Config::Addr).await.unwrap().unwrap(),
+        "foobar@example.org"
+    );
+}
+
+/// Tests that "bot" config can only be set to "0" or "1".
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_set_config_bot() {
+    let t = TestContext::new().await;
+
+    assert!(t.set_config(Config::Bot, None).await.is_ok());
+    assert!(t.set_config(Config::Bot, Some("0")).await.is_ok());
+    assert!(t.set_config(Config::Bot, Some("1")).await.is_ok());
+    assert!(t.set_config(Config::Bot, Some("2")).await.is_err());
+    assert!(t.set_config(Config::Bot, Some("Foobar")).await.is_err());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_media_quality_config_option() {
+    let t = TestContext::new().await;
+    let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
+    assert_eq!(media_quality, 0);
+    let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+    assert_eq!(media_quality, constants::MediaQuality::Balanced);
+
+    t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
+
+    let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
+    assert_eq!(media_quality, 1);
+    assert_eq!(constants::MediaQuality::Worse as i32, 1);
+    let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+    assert_eq!(media_quality, constants::MediaQuality::Worse);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ui_config() -> Result<()> {
+    let t = TestContext::new().await;
+
+    assert_eq!(t.get_ui_config("ui.desktop.linux.systray").await?, None);
+
+    t.set_ui_config("ui.android.screen_security", Some("safe"))
+        .await?;
+    assert_eq!(
+        t.get_ui_config("ui.android.screen_security").await?,
+        Some("safe".to_string())
+    );
+
+    t.set_ui_config("ui.android.screen_security", None).await?;
+    assert_eq!(t.get_ui_config("ui.android.screen_security").await?, None);
+
+    assert!(t.set_ui_config("configured", Some("bar")).await.is_err());
+
+    Ok(())
+}
+
+/// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_set_config_bool() -> Result<()> {
+    let t = TestContext::new().await;
+
+    // We need some config that defaults to true
+    let c = Config::E2eeEnabled;
+    assert_eq!(t.get_config_bool(c).await?, true);
+    t.set_config_bool(c, false).await?;
+    assert_eq!(t.get_config_bool(c).await?, false);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_self_addrs() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+
+    assert!(alice.is_self_addr("alice@example.org").await?);
+    assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
+    assert!(!alice.is_self_addr("alice@alice.com").await?);
+
+    // Test adding the same primary address
+    alice.set_primary_self_addr("alice@example.org").await?;
+    alice.set_primary_self_addr("Alice@Example.Org").await?;
+    assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
+
+    // Test adding a new (primary) self address
+    // The address is trimmed during configure by `LoginParam::from_database()`,
+    // so `set_primary_self_addr()` doesn't have to trim it.
+    alice.set_primary_self_addr("Alice@alice.com").await?;
+    assert!(alice.is_self_addr("aliCe@example.org").await?);
+    assert!(alice.is_self_addr("alice@alice.com").await?);
+    assert_eq!(
+        alice.get_all_self_addrs().await?,
+        vec!["Alice@alice.com", "Alice@Example.Org"]
+    );
+
+    // Check that the entry is not duplicated
+    alice.set_primary_self_addr("alice@alice.com").await?;
+    alice.set_primary_self_addr("alice@alice.com").await?;
+    assert_eq!(
+        alice.get_all_self_addrs().await?,
+        vec!["alice@alice.com", "Alice@Example.Org"]
+    );
+
+    // Test switching back
+    alice.set_primary_self_addr("alice@example.org").await?;
+    assert_eq!(
+        alice.get_all_self_addrs().await?,
+        vec!["alice@example.org", "alice@alice.com"]
+    );
+
+    // Test setting a new primary self address, the previous self address
+    // should be kept as a secondary self address
+    alice.set_primary_self_addr("alice@alice.xyz").await?;
+    assert_eq!(
+        alice.get_all_self_addrs().await?,
+        vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
+    );
+    assert!(alice.is_self_addr("alice@example.org").await?);
+    assert!(alice.is_self_addr("alice@alice.com").await?);
+    assert!(alice.is_self_addr("Alice@alice.xyz").await?);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_mdns_default_behaviour() -> Result<()> {
+    let t = &TestContext::new_alice().await;
+    assert!(t.should_request_mdns().await?);
+    assert!(t.should_send_mdns().await?);
+    assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
+    // The setting should be displayed correctly.
+    assert!(t.get_config_bool(Config::MdnsEnabled).await?);
+
+    t.set_config_bool(Config::Bot, true).await?;
+    assert!(!t.should_request_mdns().await?);
+    assert!(t.should_send_mdns().await?);
+    assert!(t.get_config_bool_opt(Config::MdnsEnabled).await?.is_none());
+    assert!(t.get_config_bool(Config::MdnsEnabled).await?);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_delete_server_after_default() -> Result<()> {
+    let t = &TestContext::new_alice().await;
+
+    // Check that the settings are displayed correctly.
+    assert_eq!(t.get_config(Config::BccSelf).await?, Some("1".to_string()));
+    assert_eq!(
+        t.get_config(Config::DeleteServerAfter).await?,
+        Some("0".to_string())
+    );
+
+    // Leaving emails on the server even w/o `BccSelf` is a good default at least because other
+    // MUAs do so even if the server doesn't save sent messages to some sentbox (like Gmail
+    // does).
+    t.set_config_bool(Config::BccSelf, false).await?;
+    assert_eq!(
+        t.get_config(Config::DeleteServerAfter).await?,
+        Some("0".to_string())
+    );
+    Ok(())
+}
+
+const SAVED_MESSAGES_DEDUPLICATED_FILE: &str = "969142cb84015bc135767bc2370934a.png";
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sync() -> Result<()> {
+    let alice0 = TestContext::new_alice().await;
+    let alice1 = TestContext::new_alice().await;
+    for a in [&alice0, &alice1] {
+        a.set_config_bool(Config::SyncMsgs, true).await?;
+    }
+
+    let mdns_enabled = alice0.get_config_bool(Config::MdnsEnabled).await?;
+    // Alice1 has a different config value.
+    alice1
+        .set_config_bool(Config::MdnsEnabled, !mdns_enabled)
+        .await?;
+    // This changes nothing, but still sends a sync message.
+    alice0
+        .set_config_bool(Config::MdnsEnabled, mdns_enabled)
+        .await?;
+    sync(&alice0, &alice1).await;
+    assert_eq!(
+        alice1.get_config_bool(Config::MdnsEnabled).await?,
+        mdns_enabled
+    );
+
+    // Reset to default. Test that it's not synced because defaults may differ across client
+    // versions.
+    alice0.set_config(Config::MdnsEnabled, None).await?;
+    alice0.set_config_bool(Config::MdnsEnabled, false).await?;
+    sync(&alice0, &alice1).await;
+    assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
+
+    for key in [Config::ShowEmails, Config::MvboxMove] {
+        let val = alice0.get_config_bool(key).await?;
+        alice0.set_config_bool(key, !val).await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(alice1.get_config_bool(key).await?, !val);
+    }
+
+    // `Config::SyncMsgs` mustn't be synced.
+    alice0.set_config_bool(Config::SyncMsgs, false).await?;
+    alice0.set_config_bool(Config::SyncMsgs, true).await?;
+    alice0.set_config_bool(Config::MdnsEnabled, true).await?;
+    sync(&alice0, &alice1).await;
+    assert!(alice1.get_config_bool(Config::MdnsEnabled).await?);
+
+    // Usual sync scenario.
+    async fn test_config_str(
+        alice0: &TestContext,
+        alice1: &TestContext,
+        key: Config,
+        val: &str,
+    ) -> Result<()> {
+        alice0.set_config(key, Some(val)).await?;
+        sync(alice0, alice1).await;
+        assert_eq!(alice1.get_config(key).await?, Some(val.to_string()));
+        Ok(())
+    }
+    test_config_str(&alice0, &alice1, Config::Displayname, "Alice Sync").await?;
+    test_config_str(&alice0, &alice1, Config::Selfstatus, "My status").await?;
+
+    assert!(alice0.get_config(Config::Selfavatar).await?.is_none());
+    let file = alice0.dir.path().join("avatar.png");
+    let bytes = include_bytes!("../../test-data/image/avatar64x64.png");
+    tokio::fs::write(&file, bytes).await?;
+    alice0
+        .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+        .await?;
+    sync(&alice0, &alice1).await;
+    // There was a bug that a sync message creates the self-chat with the user avatar instead of
+    // the special icon and that remains so when the self-chat becomes user-visible. Let's check
+    // this.
+    let self_chat = alice0.get_self_chat().await;
+    let self_chat_avatar_path = self_chat.get_profile_image(&alice0).await?.unwrap();
+    assert_eq!(
+        self_chat_avatar_path,
+        alice0.get_blobdir().join(SAVED_MESSAGES_DEDUPLICATED_FILE)
+    );
+    assert!(alice1
+        .get_config(Config::Selfavatar)
+        .await?
+        .filter(|path| path.ends_with(".png"))
+        .is_some());
+    alice0.set_config(Config::Selfavatar, None).await?;
+    sync(&alice0, &alice1).await;
+    assert!(alice1.get_config(Config::Selfavatar).await?.is_none());
+
+    Ok(())
+}
+
+/// Sync message mustn't be sent if self-{status,avatar} is changed by a self-sent message.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_no_sync_on_self_sent_msg() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice0 = &tcm.alice().await;
+    let alice1 = &tcm.alice().await;
+    for a in [alice0, alice1] {
+        a.set_config_bool(Config::SyncMsgs, true).await?;
+    }
+
+    let status = "Synced via usual message";
+    alice0.set_config(Config::Selfstatus, Some(status)).await?;
+    alice0.send_sync_msg().await?;
+    alice0.pop_sent_sync_msg().await;
+    let status1 = "Synced via sync message";
+    alice1.set_config(Config::Selfstatus, Some(status1)).await?;
+    tcm.send_recv(alice0, alice1, "hi Alice!").await;
+    assert_eq!(
+        alice1.get_config(Config::Selfstatus).await?,
+        Some(status.to_string())
+    );
+    sync(alice1, alice0).await;
+    assert_eq!(
+        alice0.get_config(Config::Selfstatus).await?,
+        Some(status1.to_string())
+    );
+
+    // Need a chat with another contact to send self-avatar.
+    let bob = &tcm.bob().await;
+    let a0b_chat_id = tcm.send_recv_accept(bob, alice0, "hi").await.chat_id;
+    let file = alice0.dir.path().join("avatar.png");
+    let bytes = include_bytes!("../../test-data/image/avatar64x64.png");
+    tokio::fs::write(&file, bytes).await?;
+    alice0
+        .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+        .await?;
+    alice0.send_sync_msg().await?;
+    alice0.pop_sent_sync_msg().await;
+    let file = alice1.dir.path().join("avatar.jpg");
+    let bytes = include_bytes!("../../test-data/image/avatar1000x1000.jpg");
+    tokio::fs::write(&file, bytes).await?;
+    alice1
+        .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+        .await?;
+    let sent_msg = alice0.send_text(a0b_chat_id, "hi").await;
+    alice1.recv_msg(&sent_msg).await;
+    assert!(alice1
+        .get_config(Config::Selfavatar)
+        .await?
+        .filter(|path| path.ends_with(".png"))
+        .is_some());
+    sync(alice1, alice0).await;
+    assert!(alice0
+        .get_config(Config::Selfavatar)
+        .await?
+        .filter(|path| path.ends_with(".jpg"))
+        .is_some());
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_event_config_synced() -> Result<()> {
+    let alice0 = TestContext::new_alice().await;
+    let alice1 = TestContext::new_alice().await;
+    for a in [&alice0, &alice1] {
+        a.set_config_bool(Config::SyncMsgs, true).await?;
+    }
+
+    alice0
+        .set_config(Config::Displayname, Some("Alice Sync"))
+        .await?;
+    alice0
+        .evtracker
+        .get_matching(|e| {
+            matches!(
+                e,
+                EventType::ConfigSynced {
+                    key: Config::Displayname
+                }
+            )
+        })
+        .await;
+    sync(&alice0, &alice1).await;
+    assert_eq!(
+        alice1.get_config(Config::Displayname).await?,
+        Some("Alice Sync".to_string())
+    );
+    alice1
+        .evtracker
+        .get_matching(|e| {
+            matches!(
+                e,
+                EventType::ConfigSynced {
+                    key: Config::Displayname
+                }
+            )
+        })
+        .await;
+
+    alice0.set_config(Config::Displayname, None).await?;
+    alice0
+        .evtracker
+        .get_matching(|e| {
+            matches!(
+                e,
+                EventType::ConfigSynced {
+                    key: Config::Displayname
+                }
+            )
+        })
+        .await;
+
+    Ok(())
+}

src/configure.rs

@@ -38,7 +38,7 @@ use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::sync::Sync::*;
 use crate::tools::time;
-use crate::{chat, e2ee, provider};
+use crate::{chat, provider};
 use crate::{stock_str, EventType};
 use deltachat_contact_tools::addr_cmp;
 
@@ -473,9 +473,6 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
 
     progress!(ctx, 920);
 
-    e2ee::ensure_secret_key_exists(ctx).await?;
-    info!(ctx, "key generation completed");
-
     ctx.set_config_internal(Config::FetchedExistingMsgs, config::from_bool(false))
         .await?;
     ctx.scheduler.interrupt_inbox().await;

src/constants.rs

@@ -234,6 +234,10 @@ pub(crate) const TIMESTAMP_SENT_TOLERANCE: i64 = 60;
 /// on mobile devices. See also [`crate::chat::CantSendReason::SecurejoinWait`].
 pub(crate) const SECUREJOIN_WAIT_TIMEOUT: u64 = 15;
 
+// To make text edits clearer for Non-Delta-MUA or old Delta Chats, edited text will be prefixed by EDITED_PREFIX.
+// Newer Delta Chats will remove the prefix as needed.
+pub(crate) const EDITED_PREFIX: &str = "✏️";
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/contact.rs

@@ -247,7 +247,16 @@ pub async fn make_vcard(context: &Context, contacts: &[ContactId]) -> Result<Str
             timestamp: Ok(now),
         });
     }
-    Ok(contact_tools::make_vcard(&vcard_contacts))
+
+    // XXX: newline at the end of vCard is trimmed
+    // for compatibility with core <=1.155.3
+    // Newer core should be able to deal with
+    // trailing CRLF as the fix
+    // <https://github.com/deltachat/deltachat-core-rust/pull/6522>
+    // is merged.
+    Ok(contact_tools::make_vcard(&vcard_contacts)
+        .trim_end()
+        .to_string())
 }
 
 /// Imports contacts from the given vCard.
@@ -496,7 +505,11 @@ pub enum Origin {
     /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling contact_is_verified() !
     SecurejoinInvited = 0x0100_0000,
 
-    /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling contact_is_verified() !
+    /// Set on Bob's side for contacts scanned from a QR code.
+    /// Only means the contact has been scanned from the QR code,
+    /// but does not mean that securejoin succeeded
+    /// or the key has not changed since the last scan.
+    /// Getting the current key verification status requires calling contact_is_verified() !
     SecurejoinJoined = 0x0200_0000,
 
     /// contact added manually by create_contact(), this should be the largest origin as otherwise the user cannot modify the names
@@ -1422,7 +1435,7 @@ impl Contact {
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
         if self.id == ContactId::SELF {
             if let Some(p) = context.get_config(Config::Selfavatar).await? {
-                return Ok(Some(PathBuf::from(p)));
+                return Ok(Some(PathBuf::from(p))); // get_config() calls get_abs_path() internally already
             }
         } else if let Some(image_rel) = self.param.get(Param::ProfileImage) {
             if !image_rel.is_empty() {

src/debug_logging.rs

@@ -9,7 +9,6 @@ use crate::tools::time;
 use crate::webxdc::StatusUpdateItem;
 use async_channel::{self as channel, Receiver, Sender};
 use serde_json::json;
-use std::path::PathBuf;
 use tokio::task;
 
 #[derive(Debug)]
@@ -100,9 +99,7 @@ pub async fn maybe_set_logging_xdc(
         context,
         msg.get_viewtype(),
         chat_id,
-        msg.param
-            .get_path(Param::Filename, context)
-            .unwrap_or_default(),
+        msg.param.get(Param::Filename),
         msg.get_id(),
     )
     .await?;
@@ -115,18 +112,16 @@ pub async fn maybe_set_logging_xdc_inner(
     context: &Context,
     viewtype: Viewtype,
     chat_id: ChatId,
-    filename: Option<PathBuf>,
+    filename: Option<&str>,
     msg_id: MsgId,
 ) -> anyhow::Result<()> {
     if viewtype == Viewtype::Webxdc {
-        if let Some(file) = filename {
-            if let Some(file_name) = file.file_name().and_then(|name| name.to_str()) {
-                if file_name.starts_with("debug_logging")
-                    && file_name.ends_with(".xdc")
-                    && chat_id.is_self_talk(context).await?
-                {
-                    set_debug_logging_xdc(context, Some(msg_id)).await?;
-                }
+        if let Some(filename) = filename {
+            if filename.starts_with("debug_logging")
+                && filename.ends_with(".xdc")
+                && chat_id.is_self_talk(context).await?
+            {
+                set_debug_logging_xdc(context, Some(msg_id)).await?;
             }
         }
     }

src/e2ee.rs

@@ -1,6 +1,9 @@
 //! End-to-end encryption support.
 
+use std::io::Cursor;
+
 use anyhow::{format_err, Context as _, Result};
+use mail_builder::mime::MimePart;
 use num_traits::FromPrimitive;
 
 use crate::aheader::{Aheader, EncryptPreference};
@@ -95,7 +98,7 @@ impl EncryptHelper {
         self,
         context: &Context,
         verified: bool,
-        mail_to_encrypt: lettre_email::PartBuilder,
+        mail_to_encrypt: MimePart<'static>,
         peerstates: Vec<(Option<Peerstate>, String)>,
         compress: bool,
     ) -> Result<String> {
@@ -136,7 +139,9 @@ impl EncryptHelper {
 
         let sign_key = load_self_secret_key(context).await?;
 
-        let raw_message = mail_to_encrypt.build().as_string().into_bytes();
+        let mut raw_message = Vec::new();
+        let cursor = Cursor::new(&mut raw_message);
+        mail_to_encrypt.clone().write_part(cursor).ok();
 
         let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key), compress).await?;
 
@@ -145,15 +150,13 @@ impl EncryptHelper {
 
     /// Signs the passed-in `mail` using the private key from `context`.
     /// Returns the payload and the signature.
-    pub async fn sign(
-        self,
-        context: &Context,
-        mail: lettre_email::PartBuilder,
-    ) -> Result<(lettre_email::MimeMessage, String)> {
+    pub async fn sign(self, context: &Context, mail: &MimePart<'static>) -> Result<String> {
         let sign_key = load_self_secret_key(context).await?;
-        let mime_message = mail.build();
-        let signature = pgp::pk_calc_signature(mime_message.as_string().as_bytes(), &sign_key)?;
-        Ok((mime_message, signature))
+        let mut buffer = Vec::new();
+        let cursor = Cursor::new(&mut buffer);
+        mail.clone().write_part(cursor).ok();
+        let signature = pgp::pk_calc_signature(&buffer, &sign_key)?;
+        Ok(signature)
     }
 }
 

src/ephemeral/ephemeral_tests.rs

@@ -1,7 +1,11 @@
 use super::*;
-use crate::chat::{marknoticed_chat, set_muted, ChatVisibility, MuteDuration};
+use crate::chat::{
+    add_contact_to_chat, marknoticed_chat, remove_contact_from_chat, set_muted, ChatVisibility,
+    MuteDuration,
+};
 use crate::config::Config;
 use crate::constants::DC_CHAT_ID_ARCHIVED_LINK;
+use crate::contact::Contact;
 use crate::download::DownloadState;
 use crate::location;
 use crate::message::markseen_msgs;
@@ -779,3 +783,39 @@ async fn test_archived_ephemeral_timer() -> Result<()> {
 
     Ok(())
 }
+
+/// Tests that non-members cannot change ephemeral timer settings.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_timer_non_member() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let alice_bob_contact_id = Contact::create(alice, "Bob", "bob@example.net").await?;
+    let alice_chat_id =
+        create_group_chat(alice, ProtectionStatus::Unprotected, "Group name").await?;
+    add_contact_to_chat(alice, alice_chat_id, alice_bob_contact_id).await?;
+    send_text_msg(alice, alice_chat_id, "Hi!".to_string()).await?;
+
+    let sent = alice.pop_sent_msg().await;
+    let bob_chat_id = bob.recv_msg(&sent).await.chat_id;
+
+    // Bob wants to modify the timer.
+    bob_chat_id.accept(bob).await?;
+    bob_chat_id
+        .set_ephemeral_timer(bob, Timer::Enabled { duration: 60 })
+        .await?;
+    let sent_ephemeral_timer_change = bob.pop_sent_msg().await;
+
+    // Alice removes Bob before receiving the timer change.
+    remove_contact_from_chat(alice, alice_chat_id, alice_bob_contact_id).await?;
+    alice.recv_msg(&sent_ephemeral_timer_change).await;
+
+    // Timer is not changed because Bob is not a member.
+    assert_eq!(
+        alice_chat_id.get_ephemeral_timer(alice).await?,
+        Timer::Disabled
+    );
+
+    Ok(())
+}

src/headerdef.rs

@@ -57,6 +57,7 @@ pub enum HeaderDef {
     ChatGroupId,
     ChatGroupName,
     ChatGroupNameChanged,
+    ChatGroupNameTimestamp,
     ChatVerified,
     ChatGroupAvatar,
     ChatUserAvatar,
@@ -80,6 +81,12 @@ pub enum HeaderDef {
     ChatDispositionNotificationTo,
     ChatWebrtcRoom,
 
+    /// This message deletes the messages listed in the value by rfc724_mid.
+    ChatDelete,
+
+    /// This message obsoletes the text of the message defined here by rfc724_mid.
+    ChatEdit,
+
     /// [Autocrypt](https://autocrypt.org/) header.
     Autocrypt,
     AutocryptGossip,

src/html.rs

@@ -11,9 +11,8 @@ use std::mem;
 
 use anyhow::{Context as _, Result};
 use base64::Engine as _;
-use lettre_email::mime::Mime;
-use lettre_email::PartBuilder;
 use mailparse::ParsedContentType;
+use mime::Mime;
 
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
@@ -277,16 +276,6 @@ impl MsgId {
     }
 }
 
-/// Wraps HTML text into a new text/html mimepart structure.
-///
-/// Used on forwarding messages to avoid leaking the original mime structure
-/// and also to avoid sending too much, maybe large data.
-pub fn new_html_mimepart(html: String) -> PartBuilder {
-    PartBuilder::new()
-        .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
-        .body(html)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

src/imex.rs

@@ -15,15 +15,15 @@ use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
 use tokio_tar::Archive;
 
 use crate::blob::BlobDirContents;
-use crate::chat::{self, delete_and_reset_all_device_msgs};
+use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
 use crate::context::Context;
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::log::LogExt;
-use crate::message::{Message, Viewtype};
 use crate::pgp;
+use crate::qr::DCBACKUP_VERSION;
 use crate::sql;
 use crate::tools::{
     create_folder, delete_file, get_filesuffix_lc, read_file, time, write_file, TempPathGuard,
@@ -139,20 +139,7 @@ pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
     }
 }
 
-async fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
-    if !context.sql.get_raw_config_bool("bcc_self").await? {
-        let mut msg = Message::new(Viewtype::Text);
-        // TODO: define this as a stockstring once the wording is settled.
-        msg.text = "It seems you are using multiple devices with Delta Chat. Great!\n\n\
-             If you also want to synchronize outgoing messages across all devices, \
-             go to \"Settings → Advanced\" and enable \"Send Copy to Self\"."
-            .to_string();
-        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg)).await?;
-    }
-    Ok(())
-}
-
-async fn set_self_key(context: &Context, armored: &str, set_default: bool) -> Result<()> {
+async fn set_self_key(context: &Context, armored: &str) -> Result<()> {
     // try hard to only modify key-state
     let (private_key, header) = SignedSecretKey::from_asc(armored)?;
     let public_key = private_key.split_public_key()?;
@@ -184,16 +171,7 @@ async fn set_self_key(context: &Context, armored: &str, set_default: bool) -> Re
         public: public_key,
         secret: private_key,
     };
-    key::store_self_keypair(
-        context,
-        &keypair,
-        if set_default {
-            key::KeyPairUse::Default
-        } else {
-            key::KeyPairUse::ReadOnly
-        },
-    )
-    .await?;
+    key::store_self_keypair(context, &keypair).await?;
 
     info!(context, "stored self key: {:?}", keypair.secret.key_id());
     Ok(())
@@ -223,7 +201,7 @@ async fn imex_inner(
             .await
             .context("Cannot create private key or private key not available")?;
 
-        create_folder(context, &path).await?;
+        create_folder(context, path).await?;
     }
 
     match what {
@@ -415,6 +393,9 @@ async fn import_backup_stream_inner<R: tokio::io::AsyncRead + Unpin>(
             .await
             .context("cannot import unpacked database");
     }
+    if res.is_ok() {
+        res = check_backup_version(context).await;
+    }
     if res.is_ok() {
         res = adjust_bcc_self(context).await;
     }
@@ -613,10 +594,10 @@ where
 }
 
 /// Imports secret key from a file.
-async fn import_secret_key(context: &Context, path: &Path, set_default: bool) -> Result<()> {
-    let buf = read_file(context, &path).await?;
+async fn import_secret_key(context: &Context, path: &Path) -> Result<()> {
+    let buf = read_file(context, path).await?;
     let armored = std::string::String::from_utf8_lossy(&buf);
-    set_self_key(context, &armored, set_default).await?;
+    set_self_key(context, &armored).await?;
     Ok(())
 }
 
@@ -638,8 +619,7 @@ async fn import_self_keys(context: &Context, path: &Path) -> Result<()> {
             "Importing secret key from {} as the default key.",
             path.display()
         );
-        let set_default = true;
-        import_secret_key(context, path, set_default).await?;
+        import_secret_key(context, path).await?;
         return Ok(());
     }
 
@@ -657,14 +637,13 @@ async fn import_self_keys(context: &Context, path: &Path) -> Result<()> {
         } else {
             continue;
         };
-        let set_default = !name_f.contains("legacy");
         info!(
             context,
             "Considering key file: {}.",
             path_plus_name.display()
         );
 
-        if let Err(err) = import_secret_key(context, &path_plus_name, set_default).await {
+        if let Err(err) = import_secret_key(context, &path_plus_name).await {
             warn!(
                 context,
                 "Failed to import secret key from {}: {:#}.",
@@ -801,6 +780,10 @@ async fn export_database(
         .sql
         .set_raw_config_int("backup_time", timestamp)
         .await?;
+    context
+        .sql
+        .set_raw_config_int("backup_version", DCBACKUP_VERSION)
+        .await?;
     sql::housekeeping(context).await.log_err(context).ok();
     context
         .sql
@@ -838,6 +821,15 @@ async fn adjust_bcc_self(context: &Context) -> Result<()> {
     Ok(())
 }
 
+async fn check_backup_version(context: &Context) -> Result<()> {
+    let version = (context.sql.get_raw_config_int("backup_version").await?).unwrap_or(2);
+    ensure!(
+        version <= DCBACKUP_VERSION,
+        "Backup too new, please update Delta Chat"
+    );
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use std::time::Duration;
@@ -885,7 +877,7 @@ mod tests {
 
         assert_eq!(bytes, key.to_asc(None).into_bytes());
 
-        let alice = &TestContext::new_alice().await;
+        let alice = &TestContext::new().await;
         if let Err(err) = imex(alice, ImexMode::ImportSelfKeys, Path::new(&filename), None).await {
             panic!("got error on import: {err:#}");
         }
@@ -907,7 +899,7 @@ mod tests {
             panic!("got error on export: {err:#}");
         }
 
-        let context2 = TestContext::new_alice().await;
+        let context2 = TestContext::new().await;
         if let Err(err) = imex(
             &context2.ctx,
             ImexMode::ImportSelfKeys,
@@ -934,15 +926,18 @@ mod tests {
         let alice = &TestContext::new_alice().await;
         let old_key = key::load_self_secret_key(alice).await?;
 
-        imex(alice, ImexMode::ImportSelfKeys, export_dir.path(), None).await?;
-
-        let new_key = key::load_self_secret_key(alice).await?;
-        assert_ne!(new_key, old_key);
-        assert_eq!(
-            key::load_self_secret_keyring(alice).await?,
-            vec![new_key, old_key]
+        assert!(
+            imex(alice, ImexMode::ImportSelfKeys, export_dir.path(), None)
+                .await
+                .is_err()
         );
 
+        // Importing a second key is not allowed anymore,
+        // even as a non-default key.
+        assert_eq!(key::load_self_secret_key(alice).await?, old_key);
+
+        assert_eq!(key::load_self_secret_keyring(alice).await?, vec![old_key]);
+
         let msg = alice.recv_msg(&sent).await;
         assert!(msg.get_showpadlock());
         assert_eq!(msg.chat_id, alice.get_self_chat().await.id);

src/imex/key_transfer.rs

@@ -8,7 +8,6 @@ use crate::chat::{self, ChatId};
 use crate::config::Config;
 use crate::contact::ContactId;
 use crate::context::Context;
-use crate::imex::maybe_add_bcc_self_device_msg;
 use crate::imex::set_self_key;
 use crate::key::{load_self_secret_key, DcKey};
 use crate::message::{Message, MsgId, Viewtype};
@@ -48,11 +47,10 @@ pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
     msg.param.set_int(Param::SkipAutocrypt, 1);
 
     chat::send_msg(context, chat_id, &mut msg).await?;
-    // no maybe_add_bcc_self_device_msg() here.
-    // the ui shows the dialog with the setup code on this device,
-    // it would be too much noise to have two things popping up at the same time.
-    // maybe_add_bcc_self_device_msg() is called on the other device
-    // once the transfer is completed.
+
+    // Enable BCC-self, because transferring a key
+    // means we have a multi-device setup.
+    context.set_config_bool(Config::BccSelf, true).await?;
     Ok(setup_code)
 }
 
@@ -77,8 +75,8 @@ pub async fn continue_key_transfer(
         let file = open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
         let armored_key = decrypt_setup_file(&sc, file).await?;
-        set_self_key(context, &armored_key, true).await?;
-        maybe_add_bcc_self_device_msg(context).await?;
+        set_self_key(context, &armored_key).await?;
+        context.set_config_bool(Config::BccSelf, true).await?;
 
         Ok(())
     } else {
@@ -301,8 +299,12 @@ mod tests {
     async fn test_key_transfer() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
+        alice.set_config(Config::BccSelf, Some("0")).await?;
         let setup_code = initiate_key_transfer(&alice).await?;
 
+        // Test that sending Autocrypt Setup Message enables `bcc_self`.
+        assert_eq!(alice.get_config_bool(Config::BccSelf).await?, true);
+
         // Get Autocrypt Setup Message.
         let sent = alice.pop_sent_msg().await;
 
@@ -313,21 +315,24 @@ mod tests {
         alice2.recv_msg(&sent).await;
         let msg = alice2.get_last_msg().await;
         assert!(msg.is_setupmessage());
-
-        // Send a message that cannot be decrypted because the keys are
-        // not synchronized yet.
-        let sent = alice2.send_text(msg.chat_id, "Test").await;
-        let trashed_message = alice.recv_msg_opt(&sent).await;
-        assert!(trashed_message.is_none());
-        assert_ne!(alice.get_last_msg().await.get_text(), "Test");
+        assert_eq!(
+            crate::key::load_self_secret_keyring(&alice2).await?.len(),
+            0
+        );
 
         // Transfer the key.
+        alice2.set_config(Config::BccSelf, Some("0")).await?;
         continue_key_transfer(&alice2, msg.id, &setup_code).await?;
+        assert_eq!(alice2.get_config_bool(Config::BccSelf).await?, true);
+        assert_eq!(
+            crate::key::load_self_secret_keyring(&alice2).await?.len(),
+            1
+        );
 
         // Alice sends a message to self from the new device.
         let sent = alice2.send_text(msg.chat_id, "Test").await;
-        alice.recv_msg(&sent).await;
-        assert_eq!(alice.get_last_msg().await.get_text(), "Test");
+        let rcvd_msg = alice.recv_msg(&sent).await;
+        assert_eq!(rcvd_msg.get_text(), "Test");
 
         Ok(())
     }

src/key.rs

@@ -289,7 +289,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
                 .spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
                 .await??;
 
-            store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
+            store_self_keypair(context, &keypair).await?;
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
@@ -326,18 +326,6 @@ pub(crate) async fn load_keypair(context: &Context) -> Result<Option<KeyPair>> {
     })
 }
 
-/// Use of a key pair for encryption or decryption.
-///
-/// This is used by `store_self_keypair` to know what kind of key is
-/// being saved.
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum KeyPairUse {
-    /// The default key used to encrypt new messages.
-    Default,
-    /// Only used to decrypt existing message.
-    ReadOnly,
-}
-
 /// Store the keypair as an owned keypair for addr in the database.
 ///
 /// This will save the keypair as keys for the given address.  The
@@ -350,11 +338,7 @@ pub enum KeyPairUse {
 /// same key again overwrites it.
 ///
 /// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub(crate) async fn store_self_keypair(
-    context: &Context,
-    keypair: &KeyPair,
-    default: KeyPairUse,
-) -> Result<()> {
+pub(crate) async fn store_self_keypair(context: &Context, keypair: &KeyPair) -> Result<()> {
     let mut config_cache_lock = context.sql.config_cache.write().await;
     let new_key_id = context
         .sql
@@ -362,29 +346,28 @@ pub(crate) async fn store_self_keypair(
             let public_key = DcKey::to_bytes(&keypair.public);
             let secret_key = DcKey::to_bytes(&keypair.secret);
 
-            let is_default = match default {
-                KeyPairUse::Default => true,
-                KeyPairUse::ReadOnly => false,
-            };
-
+            // private_key and public_key columns
+            // are UNIQUE since migration 107,
+            // so this fails if we already have this key.
             transaction
                 .execute(
-                    "INSERT OR REPLACE INTO keypairs (public_key, private_key)
+                    "INSERT INTO keypairs (public_key, private_key)
                      VALUES (?,?)",
                     (&public_key, &secret_key),
                 )
                 .context("Failed to insert keypair")?;
 
-            if is_default {
-                let new_key_id = transaction.last_insert_rowid();
-                transaction.execute(
-                    "INSERT OR REPLACE INTO config (keyname, value) VALUES ('key_id', ?)",
-                    (new_key_id,),
-                )?;
-                Ok(Some(new_key_id))
-            } else {
-                Ok(None)
-            }
+            let new_key_id = transaction.last_insert_rowid();
+
+            // This will fail if we already have `key_id`.
+            //
+            // Setting default key is only possible if we don't
+            // have a key already.
+            transaction.execute(
+                "INSERT INTO config (keyname, value) VALUES ('key_id', ?)",
+                (new_key_id,),
+            )?;
+            Ok(Some(new_key_id))
         })
         .await?;
 
@@ -405,7 +388,7 @@ pub async fn preconfigure_keypair(context: &Context, secret_data: &str) -> Resul
     let secret = SignedSecretKey::from_asc(secret_data)?.0;
     let public = secret.split_public_key()?;
     let keypair = KeyPair { public, secret };
-    store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
+    store_self_keypair(context, &keypair).await?;
     Ok(())
 }
 
@@ -700,6 +683,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
+    /// Tests that setting a default key second time is not allowed.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
@@ -714,13 +698,13 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
                 .unwrap()
         };
         assert_eq!(nrows().await, 0);
-        store_self_keypair(&ctx, &KEYPAIR, KeyPairUse::Default)
-            .await
-            .unwrap();
+        store_self_keypair(&ctx, &KEYPAIR).await.unwrap();
         assert_eq!(nrows().await, 1);
-        store_self_keypair(&ctx, &KEYPAIR, KeyPairUse::Default)
-            .await
-            .unwrap();
+
+        // Saving a second key fails.
+        let res = store_self_keypair(&ctx, &KEYPAIR).await;
+        assert!(res.is_err());
+
         assert_eq!(nrows().await, 1);
     }
 

src/message.rs

@@ -1,6 +1,7 @@
 //! # Messages and their identifiers.
 
 use std::collections::BTreeSet;
+use std::collections::HashSet;
 use std::path::{Path, PathBuf};
 use std::str;
 
@@ -11,7 +12,7 @@ use serde::{Deserialize, Serialize};
 use tokio::{fs, io};
 
 use crate::blob::BlobObject;
-use crate::chat::{Chat, ChatId, ChatIdBlocked, ChatVisibility};
+use crate::chat::{send_msg, Chat, ChatId, ChatIdBlocked, ChatVisibility};
 use crate::chatlist_events;
 use crate::config::Config;
 use crate::constants::{
@@ -31,6 +32,7 @@ use crate::pgp::split_armored_data;
 use crate::reaction::get_msg_reactions;
 use crate::sql;
 use crate::summary::Summary;
+use crate::sync::SyncData;
 use crate::tools::{
     buf_compress, buf_decompress, get_filebytes, get_filemeta, gm2local_offset, read_file,
     sanitize_filename, time, timestamp_to_str, truncate,
@@ -636,7 +638,7 @@ impl Message {
 
     /// Returns the full path to the file associated with a message.
     pub fn get_file(&self, context: &Context) -> Option<PathBuf> {
-        self.param.get_path(Param::File, context).unwrap_or(None)
+        self.param.get_file_path(context).unwrap_or(None)
     }
 
     /// Returns vector of vcards if the file has a vCard attachment.
@@ -669,7 +671,7 @@ impl Message {
     /// If message is an image or gif, set Param::Width and Param::Height
     pub(crate) async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> {
         if self.viewtype.has_file() {
-            let file_param = self.param.get_path(Param::File, context)?;
+            let file_param = self.param.get_file_path(context)?;
             if let Some(path_and_filename) = file_param {
                 if (self.viewtype == Viewtype::Image || self.viewtype == Viewtype::Gif)
                     && !self.param.exists(Param::Width)
@@ -817,7 +819,7 @@ impl Message {
 
     /// Returns the size of the file in bytes, if applicable.
     pub async fn get_filebytes(&self, context: &Context) -> Result<Option<u64>> {
-        if let Some(path) = self.param.get_path(Param::File, context)? {
+        if let Some(path) = self.param.get_file_path(context)? {
             Ok(Some(get_filebytes(context, &path).await.with_context(
                 || format!("failed to get {} size in bytes", path.display()),
             )?))
@@ -931,6 +933,11 @@ impl Message {
         0 != self.param.get_int(Param::Forwarded).unwrap_or_default()
     }
 
+    /// Returns true if the message is edited.
+    pub fn is_edited(&self) -> bool {
+        self.param.get_bool(Param::IsEdited).unwrap_or_default()
+    }
+
     /// Returns true if the message is an informational message.
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
@@ -968,7 +975,7 @@ impl Message {
         }
 
         if let Some(filename) = self.get_file(context) {
-            if let Ok(ref buf) = read_file(context, filename).await {
+            if let Ok(ref buf) = read_file(context, &filename).await {
                 if let Ok((typ, headers, _)) = split_armored_data(buf) {
                     if typ == pgp::armor::BlockType::Message {
                         return headers.get(crate::pgp::HEADER_SETUPCODE).cloned();
@@ -1070,21 +1077,6 @@ impl Message {
         self.subject = subject;
     }
 
-    /// Sets the file associated with a message.
-    ///
-    /// This function does not use the file or check if it exists,
-    /// the file will only be used when the message is prepared
-    /// for sending.
-    pub fn set_file(&mut self, file: impl ToString, filemime: Option<&str>) {
-        if let Some(name) = Path::new(&file.to_string()).file_name() {
-            if let Some(name) = name.to_str() {
-                self.param.set(Param::Filename, name);
-            }
-        }
-        self.param.set(Param::File, file);
-        self.param.set_optional(Param::MimeType, filemime);
-    }
-
     /// Sets the file associated with a message, deduplicating files with the same name.
     ///
     /// If `name` is Some, it is used as the file name
@@ -1370,7 +1362,7 @@ impl Message {
     /// * Lack of valid signature on an e2ee message, usually for received messages.
     /// * Failure to decrypt an e2ee message, usually for received messages.
     /// * When a message could not be delivered to one or more recipients the non-delivery
-    ///    notification text can be stored in the error status.
+    ///   notification text can be stored in the error status.
     pub fn error(&self) -> Option<String> {
         self.error.clone()
     }
@@ -1661,34 +1653,90 @@ pub async fn get_mime_headers(context: &Context, msg_id: MsgId) -> Result<Vec<u8
     Ok(headers)
 }
 
-/// Deletes requested messages
-/// by moving them to the trash chat
-/// and scheduling for deletion on IMAP.
+/// Delete a single message from the database, including references in other tables.
+/// This may be called in batches; the final events are emitted in delete_msgs_locally_done() then.
+pub(crate) async fn delete_msg_locally(context: &Context, msg: &Message) -> Result<()> {
+    if msg.location_id > 0 {
+        delete_poi_location(context, msg.location_id).await?;
+    }
+    let on_server = true;
+    msg.id
+        .trash(context, on_server)
+        .await
+        .with_context(|| format!("Unable to trash message {}", msg.id))?;
+
+    context.emit_event(EventType::MsgDeleted {
+        chat_id: msg.chat_id,
+        msg_id: msg.id,
+    });
+
+    if msg.viewtype == Viewtype::Webxdc {
+        context.emit_event(EventType::WebxdcInstanceDeleted { msg_id: msg.id });
+    }
+
+    let logging_xdc_id = context
+        .debug_logging
+        .read()
+        .expect("RwLock is poisoned")
+        .as_ref()
+        .map(|dl| dl.msg_id);
+    if let Some(id) = logging_xdc_id {
+        if id == msg.id {
+            set_debug_logging_xdc(context, None).await?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Do final events and jobs after batch deletion using calls to delete_msg_locally().
+/// To avoid additional database queries, collecting data is up to the caller.
+pub(crate) async fn delete_msgs_locally_done(
+    context: &Context,
+    msg_ids: &[MsgId],
+    modified_chat_ids: HashSet<ChatId>,
+) -> Result<()> {
+    for modified_chat_id in modified_chat_ids {
+        context.emit_msgs_changed_without_msg_id(modified_chat_id);
+        chatlist_events::emit_chatlist_item_changed(context, modified_chat_id);
+    }
+    if !msg_ids.is_empty() {
+        context.emit_msgs_changed_without_ids();
+        chatlist_events::emit_chatlist_changed(context);
+        // Run housekeeping to delete unused blobs.
+        context
+            .set_config_internal(Config::LastHousekeeping, None)
+            .await?;
+    }
+    Ok(())
+}
+
+/// Delete messages on all devices and on IMAP.
 pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
-    let mut modified_chat_ids = BTreeSet::new();
+    delete_msgs_ex(context, msg_ids, false).await
+}
+
+/// Delete messages on all devices, on IMAP and optionally for all chat members.
+/// Deleted messages are moved to the trash chat and scheduling for deletion on IMAP.
+/// When deleting messages for others, all messages must be self-sent and in the same chat.
+pub async fn delete_msgs_ex(
+    context: &Context,
+    msg_ids: &[MsgId],
+    delete_for_all: bool,
+) -> Result<()> {
+    let mut modified_chat_ids = HashSet::new();
+    let mut deleted_rfc724_mid = Vec::new();
     let mut res = Ok(());
 
     for &msg_id in msg_ids {
         let msg = Message::load_from_db(context, msg_id).await?;
-        if msg.location_id > 0 {
-            delete_poi_location(context, msg.location_id).await?;
-        }
-        let on_server = true;
-        msg_id
-            .trash(context, on_server)
-            .await
-            .with_context(|| format!("Unable to trash message {msg_id}"))?;
-
-        context.emit_event(EventType::MsgDeleted {
-            chat_id: msg.chat_id,
-            msg_id,
-        });
-
-        if msg.viewtype == Viewtype::Webxdc {
-            context.emit_event(EventType::WebxdcInstanceDeleted { msg_id });
-        }
+        ensure!(
+            !delete_for_all || msg.from_id == ContactId::SELF,
+            "Can delete only own messages for others"
+        );
 
         modified_chat_ids.insert(msg.chat_id);
+        deleted_rfc724_mid.push(msg.rfc724_mid.clone());
 
         let target = context.get_delete_msgs_target().await?;
         let update_db = |trans: &mut rusqlite::Transaction| {
@@ -1704,38 +1752,39 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             res = Err(e);
             continue;
         }
-
-        let logging_xdc_id = context
-            .debug_logging
-            .read()
-            .expect("RwLock is poisoned")
-            .as_ref()
-            .map(|dl| dl.msg_id);
-
-        if let Some(id) = logging_xdc_id {
-            if id == msg_id {
-                set_debug_logging_xdc(context, None).await?;
-            }
-        }
     }
     res?;
 
-    for modified_chat_id in modified_chat_ids {
-        context.emit_msgs_changed_without_msg_id(modified_chat_id);
-        chatlist_events::emit_chatlist_item_changed(context, modified_chat_id);
-    }
-
-    if !msg_ids.is_empty() {
-        context.emit_msgs_changed_without_ids();
-        chatlist_events::emit_chatlist_changed(context);
-        // Run housekeeping to delete unused blobs.
+    if delete_for_all {
+        ensure!(
+            modified_chat_ids.len() == 1,
+            "Can delete only from same chat."
+        );
+        if let Some(chat_id) = modified_chat_ids.iter().next() {
+            let mut msg = Message::new_text("🚮".to_owned());
+            msg.param.set_int(Param::GuaranteeE2ee, 1);
+            msg.param
+                .set(Param::DeleteRequestFor, deleted_rfc724_mid.join(" "));
+            msg.hidden = true;
+            send_msg(context, *chat_id, &mut msg).await?;
+        }
+    } else {
         context
-            .set_config_internal(Config::LastHousekeeping, None)
+            .add_sync_item(SyncData::DeleteMessages {
+                msgs: deleted_rfc724_mid,
+            })
             .await?;
     }
 
-    // Interrupt Inbox loop to start message deletion and run housekeeping.
+    for &msg_id in msg_ids {
+        let msg = Message::load_from_db(context, msg_id).await?;
+        delete_msg_locally(context, &msg).await?;
+    }
+    delete_msgs_locally_done(context, msg_ids, modified_chat_ids).await?;
+
+    // Interrupt Inbox loop to start message deletion, run housekeeping and call send_sync_msg().
     context.scheduler.interrupt_inbox().await;
+
     Ok(())
 }
 
@@ -2162,12 +2211,12 @@ pub enum Viewtype {
     /// Image message.
     /// If the image is a GIF and has the appropriate extension, the viewtype is auto-changed to
     /// `Gif` when sending the message.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension
-    /// and retrieved via dc_msg_set_file(), dc_msg_set_dimension().
+    /// File, width and height are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_dimension()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_height(), dc_msg_get_width().
     Image = 20,
 
     /// Animated GIF message.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension()
+    /// File, width and height are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_dimension()
     /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
     Gif = 21,
 
@@ -2180,26 +2229,26 @@ pub enum Viewtype {
     Sticker = 23,
 
     /// Message containing an Audio file.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+    /// File and duration are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_duration()
     /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
     Audio = 40,
 
     /// A voice message that was directly recorded by the user.
     /// For all other audio messages, the type #DC_MSG_AUDIO should be used.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+    /// File and duration are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_duration()
     /// and retrieved via dc_msg_get_file(), dc_msg_get_duration()
     Voice = 41,
 
     /// Video messages.
     /// File, width, height and durarion
-    /// are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
+    /// are set via dc_msg_set_file_and_deduplicate(), dc_msg_set_dimension(), dc_msg_set_duration()
     /// and retrieved via
     /// dc_msg_get_file(), dc_msg_get_width(),
     /// dc_msg_get_height(), dc_msg_get_duration().
     Video = 50,
 
     /// Message containing any file, eg. a PDF.
-    /// The file is set via dc_msg_set_file()
+    /// The file is set via dc_msg_set_file_and_deduplicate()
     /// and retrieved via dc_msg_get_file().
     File = 60,
 

src/message/message_tests.rs

@@ -9,7 +9,7 @@ use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::reaction::send_reaction;
 use crate::receive_imf::receive_imf;
-use crate::test_utils as test;
+use crate::test_utils;
 use crate::test_utils::{TestContext, TestContextManager};
 
 #[test]
@@ -106,7 +106,7 @@ async fn test_create_webrtc_instance_noroom() {
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_get_width_height() {
-    let t = test::TestContext::new().await;
+    let t = TestContext::new().await;
 
     // test that get_width() and get_height() are returning some dimensions for images;
     // (as the device-chat contains a welcome-images, we check that)
@@ -136,7 +136,7 @@ async fn test_get_width_height() {
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_quote() {
-    let d = test::TestContext::new().await;
+    let d = TestContext::new().await;
     let ctx = &d.ctx;
 
     ctx.set_config(Config::ConfiguredAddr, Some("self@example.com"))
@@ -756,6 +756,37 @@ async fn test_delete_msgs_offline() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_delete_msgs_sync() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let alice2 = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_chat_id = alice.create_chat(bob).await.id;
+
+    alice.set_config_bool(Config::SyncMsgs, true).await?;
+    alice2.set_config_bool(Config::SyncMsgs, true).await?;
+    bob.set_config_bool(Config::SyncMsgs, true).await?;
+
+    // Alice sends a messsage and receives it on the other device
+    let sent1 = alice.send_text(alice_chat_id, "foo").await;
+    assert_eq!(alice_chat_id.get_msg_cnt(alice).await?, 1);
+
+    let msg = alice2.recv_msg(&sent1).await;
+    let alice2_chat_id = msg.chat_id;
+    assert_eq!(alice2.get_last_msg_in(alice2_chat_id).await.id, msg.id);
+    assert_eq!(alice2_chat_id.get_msg_cnt(alice2).await?, 1);
+
+    // Alice deletes the message; this should happen on both devices as well
+    delete_msgs(alice, &[sent1.sender_msg_id]).await?;
+    assert_eq!(alice_chat_id.get_msg_cnt(alice).await?, 0);
+
+    test_utils::sync(alice, alice2).await;
+    assert_eq!(alice2_chat_id.get_msg_cnt(alice2).await?, 0);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_sanitize_filename_message() -> Result<()> {
     let t = &TestContext::new().await;
@@ -768,7 +799,7 @@ async fn test_sanitize_filename_message() -> Result<()> {
     msg.set_file_from_bytes(t, "/\\:ee.tx*T ", b"hallo", None)?;
     assert_eq!(msg.get_filename().unwrap(), "ee.txT");
 
-    let blob = msg.param.get_blob(Param::File, t).await?.unwrap();
+    let blob = msg.param.get_file_blob(t)?.unwrap();
     assert_eq!(blob.suffix().unwrap(), "txt");
 
     // The filename shouldn't be empty if there were only illegal characters:

src/mimefactory/mimefactory_tests.rs

@@ -0,0 +1,863 @@
+use deltachat_contact_tools::ContactAddress;
+use mail_builder::headers::Header;
+use mailparse::{addrparse_header, MailHeaderMap};
+use std::str;
+
+use super::*;
+use crate::chat::{
+    add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg, ChatId,
+    ProtectionStatus,
+};
+use crate::chatlist::Chatlist;
+use crate::constants;
+use crate::contact::Origin;
+use crate::headerdef::HeaderDef;
+use crate::mimeparser::MimeMessage;
+use crate::receive_imf::receive_imf;
+use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
+
+fn render_email_address(display_name: &str, addr: &str) -> String {
+    let mut output = Vec::<u8>::new();
+    new_address_with_name(display_name, addr.to_string())
+        .unwrap_address()
+        .write_header(&mut output, 0)
+        .unwrap();
+
+    String::from_utf8(output).unwrap()
+}
+
+#[test]
+fn test_render_email_address() {
+    let display_name = "ä space";
+    let addr = "x@y.org";
+
+    assert!(!display_name.is_ascii());
+    assert!(!display_name
+        .chars()
+        .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+    let s = render_email_address(display_name, addr);
+
+    println!("{s}");
+
+    assert_eq!(s, r#""=?utf-8?B?w6Qgc3BhY2U=?=" <x@y.org>"#);
+}
+
+#[test]
+fn test_render_email_address_noescape() {
+    let display_name = "a space";
+    let addr = "x@y.org";
+
+    assert!(display_name.is_ascii());
+    assert!(display_name
+        .chars()
+        .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+    let s = render_email_address(display_name, addr);
+
+    // Addresses should not be unnecessarily be encoded, see <https://github.com/deltachat/deltachat-core-rust/issues/1575>:
+    assert_eq!(s, r#""a space" <x@y.org>"#);
+}
+
+#[test]
+fn test_render_email_address_duplicated_as_name() {
+    let addr = "x@y.org";
+    let s = render_email_address(addr, addr);
+    assert_eq!(s, "<x@y.org>");
+}
+
+#[test]
+fn test_render_rfc724_mid() {
+    assert_eq!(
+        render_rfc724_mid("kqjwle123@qlwe"),
+        "<kqjwle123@qlwe>".to_string()
+    );
+    assert_eq!(
+        render_rfc724_mid("  kqjwle123@qlwe "),
+        "<kqjwle123@qlwe>".to_string()
+    );
+    assert_eq!(
+        render_rfc724_mid("<kqjwle123@qlwe>"),
+        "<kqjwle123@qlwe>".to_string()
+    );
+}
+
+fn render_header_text(text: &str) -> String {
+    let mut output = Vec::<u8>::new();
+    mail_builder::headers::text::Text::new(text.to_string())
+        .write_header(&mut output, 0)
+        .unwrap();
+
+    String::from_utf8(output).unwrap()
+}
+
+#[test]
+fn test_header_encoding() {
+    assert_eq!(render_header_text("foobar"), "foobar\r\n");
+    assert_eq!(render_header_text("-_.~%"), "-_.~%\r\n");
+    assert_eq!(render_header_text("äöü"), "=?utf-8?B?w6TDtsO8?=\r\n");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_manually_set_subject() -> Result<()> {
+    let t = TestContext::new_alice().await;
+    let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+    let mut msg = Message::new(Viewtype::Text);
+    msg.set_subject("Subjeeeeect".to_string());
+
+    let sent_msg = t.send_msg(chat.id, &mut msg).await;
+    let payload = sent_msg.payload();
+
+    assert_eq!(payload.match_indices("Subject: Subjeeeeect").count(), 1);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_subject_from_mua() {
+    // 1.: Receive a mail from an MUA
+    assert_eq!(
+        msg_to_subject_str(
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Bob <bob@example.com>\n\
+                To: alice@example.org\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+        )
+        .await,
+        "Re: Chat: hello"
+    );
+
+    assert_eq!(
+        msg_to_subject_str(
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Bob <bob@example.com>\n\
+                To: alice@example.org\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+        )
+        .await,
+        "Re: Infos: 42"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_subject_from_dc() {
+    // 2. Receive a message from Delta Chat
+    assert_eq!(
+        msg_to_subject_str(
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: bob@example.com\n\
+                To: alice@example.org\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+        )
+        .await,
+        "Re: Chat: hello"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_subject_outgoing() {
+    // 3. Send the first message to a new contact
+    let t = TestContext::new_alice().await;
+
+    assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
+
+    let t = TestContext::new_alice().await;
+    t.set_config(Config::Displayname, Some("Alice"))
+        .await
+        .unwrap();
+    assert_eq!(first_subject_str(t).await, "Message from Alice");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_subject_unicode() {
+    // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+    msg_to_subject_str(
+        "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+            From: bob@example.com\n\
+            To: alice@example.org\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+            .as_bytes(),
+    )
+    .await;
+
+    msg_to_subject_str(
+        "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+            From: bob@example.com\n\
+            To: alice@example.org\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+            .as_bytes(),
+    )
+    .await;
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_subject_mdn() {
+    // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
+    let t = TestContext::new_alice().await;
+    receive_imf(
+        &t,
+        b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+            From: alice@example.org\n\
+            To: bob@example.com\n\
+            Subject: Hello, Bob\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n",
+        false,
+    )
+    .await
+    .unwrap();
+    let mut new_msg = incoming_msg_to_reply_msg(
+        b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                 From: bob@example.com\n\
+                 To: alice@example.org\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;bob@example.com\n\
+                 Final-Recipient: rfc822;bob@example.com\n\
+                 Original-Message-ID: <2893@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n", &t).await;
+    chat::send_msg(&t, new_msg.chat_id, &mut new_msg)
+        .await
+        .unwrap();
+    let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap();
+    // The subject string should not be "Re: message opened"
+    assert_eq!("Re: Hello, Bob", mf.subject_str(&t).await.unwrap());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_mdn_create_encrypted() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    alice
+        .set_config(Config::Displayname, Some("Alice Exampleorg"))
+        .await?;
+    let bob = tcm.bob().await;
+    bob.set_config(Config::Displayname, Some("Bob Examplenet"))
+        .await?;
+    bob.set_config(Config::Selfstatus, Some("Bob Examplenet"))
+        .await?;
+    bob.set_config_bool(Config::MdnsEnabled, true).await?;
+
+    let mut msg = Message::new(Viewtype::Text);
+    msg.param.set_int(Param::SkipAutocrypt, 1);
+    let chat_alice = alice.create_chat(&bob).await.id;
+    let sent = alice.send_msg(chat_alice, &mut msg).await;
+
+    let rcvd = bob.recv_msg(&sent).await;
+    message::markseen_msgs(&bob, vec![rcvd.id]).await?;
+    let mimefactory =
+        MimeFactory::from_mdn(&bob, rcvd.from_id, rcvd.rfc724_mid.clone(), vec![]).await?;
+    let rendered_msg = mimefactory.render(&bob).await?;
+
+    assert!(!rendered_msg.is_encrypted);
+    assert!(!rendered_msg.message.contains("Bob Examplenet"));
+    assert!(!rendered_msg.message.contains("Alice Exampleorg"));
+    let bob_alice_contact = bob.add_or_lookup_contact(&alice).await;
+    assert_eq!(bob_alice_contact.get_authname(), "Alice Exampleorg");
+
+    let rcvd = tcm.send_recv(&alice, &bob, "Heyho").await;
+    message::markseen_msgs(&bob, vec![rcvd.id]).await?;
+
+    let mimefactory = MimeFactory::from_mdn(&bob, rcvd.from_id, rcvd.rfc724_mid, vec![]).await?;
+    let rendered_msg = mimefactory.render(&bob).await?;
+
+    // When encrypted, the MDN should be encrypted as well
+    assert!(rendered_msg.is_encrypted);
+    assert!(!rendered_msg.message.contains("Bob Examplenet"));
+    assert!(!rendered_msg.message.contains("Alice Exampleorg"));
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_subject_in_group() -> Result<()> {
+    async fn send_msg_get_subject(
+        t: &TestContext,
+        group_id: ChatId,
+        quote: Option<&Message>,
+    ) -> Result<String> {
+        let mut new_msg = Message::new_text("Hi".to_string());
+        if let Some(q) = quote {
+            new_msg.set_quote(t, Some(q)).await?;
+        }
+        let sent = t.send_msg(group_id, &mut new_msg).await;
+        get_subject(t, sent).await
+    }
+    async fn get_subject(
+        t: &TestContext,
+        sent: crate::test_utils::SentMessage<'_>,
+    ) -> Result<String> {
+        let parsed_subject = t.parse_msg(&sent).await.get_subject().unwrap();
+
+        let sent_msg = sent.load_from_db().await;
+        assert_eq!(parsed_subject, sent_msg.subject);
+
+        Ok(parsed_subject)
+    }
+
+    // 6. Test that in a group, replies also take the quoted message's subject, while non-replies use the group title as subject
+    let t = TestContext::new_alice().await;
+    let group_id = chat::create_group_chat(&t, chat::ProtectionStatus::Unprotected, "groupname") // TODO encodings, ä
+        .await
+        .unwrap();
+    let bob = Contact::create(&t, "", "bob@example.org").await?;
+    chat::add_contact_to_chat(&t, group_id, bob).await?;
+
+    let subject = send_msg_get_subject(&t, group_id, None).await?;
+    assert_eq!(subject, "groupname");
+
+    let subject = send_msg_get_subject(&t, group_id, None).await?;
+    assert_eq!(subject, "Re: groupname");
+
+    receive_imf(
+        &t,
+        format!(
+            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: bob@example.com\n\
+                To: alice@example.org\n\
+                Subject: Different subject\n\
+                In-Reply-To: {}\n\
+                Message-ID: <2893@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            t.get_last_msg().await.rfc724_mid
+        )
+        .as_bytes(),
+        false,
+    )
+    .await?;
+    let message_from_bob = t.get_last_msg().await;
+
+    let subject = send_msg_get_subject(&t, group_id, None).await?;
+    assert_eq!(subject, "Re: groupname");
+
+    let subject = send_msg_get_subject(&t, group_id, Some(&message_from_bob)).await?;
+    let outgoing_quoting_msg = t.get_last_msg().await;
+    assert_eq!(subject, "Re: Different subject");
+
+    let subject = send_msg_get_subject(&t, group_id, None).await?;
+    assert_eq!(subject, "Re: groupname");
+
+    let subject = send_msg_get_subject(&t, group_id, Some(&outgoing_quoting_msg)).await?;
+    assert_eq!(subject, "Re: Different subject");
+
+    chat::forward_msgs(&t, &[message_from_bob.id], group_id).await?;
+    let subject = get_subject(&t, t.pop_sent_msg().await).await?;
+    assert_eq!(subject, "Re: groupname");
+    Ok(())
+}
+
+async fn first_subject_str(t: TestContext) -> String {
+    let contact_id = Contact::add_or_lookup(
+        &t,
+        "Dave",
+        &ContactAddress::new("dave@example.com").unwrap(),
+        Origin::ManuallyCreated,
+    )
+    .await
+    .unwrap()
+    .0;
+
+    let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap();
+
+    let mut new_msg = Message::new_text("Hi".to_string());
+    new_msg.chat_id = chat_id;
+    chat::send_msg(&t, chat_id, &mut new_msg).await.unwrap();
+
+    let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap();
+
+    mf.subject_str(&t).await.unwrap()
+}
+
+// In `imf_raw`, From has to be bob@example.com, To has to be alice@example.org
+async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+    let subject_str = msg_to_subject_str_inner(imf_raw, false, false, false).await;
+
+    // Check that combinations of true and false reproduce the same subject_str:
+    assert_eq!(
+        subject_str,
+        msg_to_subject_str_inner(imf_raw, true, false, false).await
+    );
+    assert_eq!(
+        subject_str,
+        msg_to_subject_str_inner(imf_raw, false, true, false).await
+    );
+    assert_eq!(
+        subject_str,
+        msg_to_subject_str_inner(imf_raw, false, true, true).await
+    );
+    assert_eq!(
+        subject_str,
+        msg_to_subject_str_inner(imf_raw, true, true, false).await
+    );
+
+    // These two combinations are different: If `message_arrives_inbetween` is true, but
+    // `reply` is false, the core is actually expected to use the subject of the message
+    // that arrived in between.
+    assert_eq!(
+        "Re: Some other, completely unrelated subject",
+        msg_to_subject_str_inner(imf_raw, false, false, true).await
+    );
+    assert_eq!(
+        "Re: Some other, completely unrelated subject",
+        msg_to_subject_str_inner(imf_raw, true, false, true).await
+    );
+
+    // We leave away the combination (true, true, true) here:
+    // It would mean that the original message is quoted without sending the quoting message
+    // out yet, then the original message is deleted, then another unrelated message arrives
+    // and then the message with the quote is sent out. Not very realistic.
+
+    subject_str
+}
+
+async fn msg_to_subject_str_inner(
+    imf_raw: &[u8],
+    delete_original_msg: bool,
+    reply: bool,
+    message_arrives_inbetween: bool,
+) -> String {
+    let t = TestContext::new_alice().await;
+    let mut new_msg = incoming_msg_to_reply_msg(imf_raw, &t).await;
+    let incoming_msg = get_chat_msg(&t, new_msg.chat_id, 0, 1).await;
+
+    if delete_original_msg {
+        incoming_msg.id.trash(&t, false).await.unwrap();
+    }
+
+    if message_arrives_inbetween {
+        receive_imf(
+            &t,
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                    From: Bob <bob@example.com>\n\
+                    To: alice@example.org\n\
+                    Subject: Some other, completely unrelated subject\n\
+                    Message-ID: <3cl4@example.com>\n\
+                    Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                    \n\
+                    Some other, completely unrelated content\n",
+            false,
+        )
+        .await
+        .unwrap();
+
+        let arrived_msg = t.get_last_msg().await;
+        assert_eq!(arrived_msg.chat_id, incoming_msg.chat_id);
+    }
+
+    if reply {
+        new_msg.set_quote(&t, Some(&incoming_msg)).await.unwrap();
+    }
+
+    chat::send_msg(&t, new_msg.chat_id, &mut new_msg)
+        .await
+        .unwrap();
+    let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap();
+    mf.subject_str(&t).await.unwrap()
+}
+
+// Creates a `Message` that replies "Hi" to the incoming email in `imf_raw`.
+async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+    context
+        .set_config(Config::ShowEmails, Some("2"))
+        .await
+        .unwrap();
+
+    receive_imf(context, imf_raw, false).await.unwrap();
+
+    let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+    let chat_id = chats.get_chat_id(0).unwrap();
+    chat_id.accept(context).await.unwrap();
+
+    let mut new_msg = Message::new_text("Hi".to_string());
+    new_msg.chat_id = chat_id;
+
+    new_msg
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+// This test could still be extended
+async fn test_render_reply() {
+    let t = TestContext::new_alice().await;
+    let context = &t;
+
+    let mut msg = incoming_msg_to_reply_msg(
+        b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                From: Charlie <charlie@example.com>\n\
+                To: alice@example.org\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+        context,
+    )
+    .await;
+    chat::send_msg(&t, msg.chat_id, &mut msg).await.unwrap();
+
+    let mimefactory = MimeFactory::from_msg(&t, msg).await.unwrap();
+
+    let recipients = mimefactory.recipients();
+    assert_eq!(recipients, vec!["charlie@example.com"]);
+
+    let rendered_msg = mimefactory.render(context).await.unwrap();
+
+    let mail = mailparse::parse_mail(rendered_msg.message.as_bytes()).unwrap();
+    assert_eq!(
+        mail.headers
+            .iter()
+            .find(|h| h.get_key() == "MIME-Version")
+            .unwrap()
+            .get_value(),
+        "1.0"
+    );
+
+    let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes(), None)
+        .await
+        .unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_selfavatar_unencrypted() -> anyhow::Result<()> {
+    // create chat with bob, set selfavatar
+    let t = TestContext::new_alice().await;
+    let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+    let file = t.dir.path().join("avatar.png");
+    let bytes = include_bytes!("../../test-data/image/avatar64x64.png");
+    tokio::fs::write(&file, bytes).await?;
+    t.set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+        .await?;
+
+    // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
+    // make sure, `Subject:` stays in the outer header (imf header)
+    let mut msg = Message::new_text("this is the text!".to_string());
+
+    let sent_msg = t.send_msg(chat.id, &mut msg).await;
+    let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
+
+    let outer = payload.next().unwrap();
+    let inner = payload.next().unwrap();
+    let body = payload.next().unwrap();
+
+    assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+    assert_eq!(outer.match_indices("Message-ID:").count(), 1);
+    assert_eq!(outer.match_indices("Subject:").count(), 1);
+    assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
+    assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
+
+    assert_eq!(inner.match_indices("text/plain").count(), 1);
+    assert_eq!(inner.match_indices("Message-ID:").count(), 1);
+    assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 1);
+    assert_eq!(inner.match_indices("Subject:").count(), 0);
+
+    assert_eq!(body.match_indices("this is the text!").count(), 1);
+
+    // if another message is sent, that one must not contain the avatar
+    let sent_msg = t.send_msg(chat.id, &mut msg).await;
+    let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
+    let outer = payload.next().unwrap();
+    let inner = payload.next().unwrap();
+    let body = payload.next().unwrap();
+
+    assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+    assert_eq!(outer.match_indices("Message-ID:").count(), 1);
+    assert_eq!(outer.match_indices("Subject:").count(), 1);
+    assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
+    assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
+
+    assert_eq!(inner.match_indices("text/plain").count(), 1);
+    assert_eq!(inner.match_indices("Message-ID:").count(), 1);
+    assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 0);
+    assert_eq!(inner.match_indices("Subject:").count(), 0);
+
+    assert_eq!(body.match_indices("this is the text!").count(), 1);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_selfavatar_unencrypted_signed() {
+    // create chat with bob, set selfavatar
+    let t = TestContext::new_alice().await;
+    t.set_config(Config::SignUnencrypted, Some("1"))
+        .await
+        .unwrap();
+    let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+    let file = t.dir.path().join("avatar.png");
+    let bytes = include_bytes!("../../test-data/image/avatar64x64.png");
+    tokio::fs::write(&file, bytes).await.unwrap();
+    t.set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+        .await
+        .unwrap();
+
+    // send message to bob: that should get multipart/signed.
+    // `Subject:` is protected by copying it.
+    // make sure, `Subject:` stays in the outer header (imf header)
+    let mut msg = Message::new_text("this is the text!".to_string());
+
+    let sent_msg = t.send_msg(chat.id, &mut msg).await;
+    let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n");
+
+    let part = payload.next().unwrap();
+    assert_eq!(part.match_indices("multipart/signed").count(), 1);
+    assert_eq!(part.match_indices("From:").count(), 1);
+    assert_eq!(part.match_indices("Message-ID:").count(), 1);
+    assert_eq!(part.match_indices("Subject:").count(), 1);
+    assert_eq!(part.match_indices("Autocrypt:").count(), 1);
+    assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+
+    let part = payload.next().unwrap();
+    assert_eq!(
+        part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+            .count(),
+        1
+    );
+    assert_eq!(part.match_indices("From:").count(), 1);
+    assert_eq!(part.match_indices("Message-ID:").count(), 0);
+    assert_eq!(part.match_indices("Subject:").count(), 1);
+    assert_eq!(part.match_indices("Autocrypt:").count(), 0);
+    assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+
+    let part = payload.next().unwrap();
+    assert_eq!(part.match_indices("text/plain").count(), 1);
+    assert_eq!(part.match_indices("From:").count(), 0);
+    assert_eq!(part.match_indices("Message-ID:").count(), 1);
+    assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1);
+    assert_eq!(part.match_indices("Subject:").count(), 0);
+
+    let body = payload.next().unwrap();
+    assert_eq!(body.match_indices("this is the text!").count(), 1);
+
+    let bob = TestContext::new_bob().await;
+    bob.recv_msg(&sent_msg).await;
+    let alice_id = Contact::lookup_id_by_addr(&bob.ctx, "alice@example.org", Origin::Unknown)
+        .await
+        .unwrap()
+        .unwrap();
+    let alice_contact = Contact::get_by_id(&bob.ctx, alice_id).await.unwrap();
+    assert!(alice_contact
+        .get_profile_image(&bob.ctx)
+        .await
+        .unwrap()
+        .is_some());
+
+    // if another message is sent, that one must not contain the avatar
+    let sent_msg = t.send_msg(chat.id, &mut msg).await;
+    let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n");
+
+    let part = payload.next().unwrap();
+    assert_eq!(part.match_indices("multipart/signed").count(), 1);
+    assert_eq!(part.match_indices("From:").count(), 1);
+    assert_eq!(part.match_indices("Message-ID:").count(), 1);
+    assert_eq!(part.match_indices("Subject:").count(), 1);
+    assert_eq!(part.match_indices("Autocrypt:").count(), 1);
+    assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+
+    let part = payload.next().unwrap();
+    assert_eq!(
+        part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+            .count(),
+        1
+    );
+    assert_eq!(part.match_indices("From:").count(), 1);
+    assert_eq!(part.match_indices("Message-ID:").count(), 0);
+    assert_eq!(part.match_indices("Subject:").count(), 1);
+    assert_eq!(part.match_indices("Autocrypt:").count(), 0);
+    assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+
+    let part = payload.next().unwrap();
+    assert_eq!(part.match_indices("text/plain").count(), 1);
+    assert_eq!(body.match_indices("From:").count(), 0);
+    assert_eq!(part.match_indices("Message-ID:").count(), 1);
+    assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
+    assert_eq!(part.match_indices("Subject:").count(), 0);
+
+    let body = payload.next().unwrap();
+    assert_eq!(body.match_indices("this is the text!").count(), 1);
+
+    bob.recv_msg(&sent_msg).await;
+    let alice_contact = Contact::get_by_id(&bob.ctx, alice_id).await.unwrap();
+    assert!(alice_contact
+        .get_profile_image(&bob.ctx)
+        .await
+        .unwrap()
+        .is_some());
+}
+
+/// Test that removed member address does not go into the `To:` field.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_remove_member_bcc() -> Result<()> {
+    // Alice creates a group with Bob and Claire and then removes Bob.
+    let alice = TestContext::new_alice().await;
+
+    let claire_addr = "claire@foo.de";
+    let bob_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
+    let claire_id = Contact::create(&alice, "Claire", claire_addr).await?;
+
+    let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "foo").await?;
+    add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
+    add_contact_to_chat(&alice, alice_chat_id, claire_id).await?;
+    send_text_msg(&alice, alice_chat_id, "Creating a group".to_string()).await?;
+
+    remove_contact_from_chat(&alice, alice_chat_id, claire_id).await?;
+    let remove = alice.pop_sent_msg().await;
+    let remove_payload = remove.payload();
+    let parsed = mailparse::parse_mail(remove_payload.as_bytes())?;
+    let to = parsed
+        .headers
+        .get_first_header("To")
+        .context("no To: header parsed")?;
+    let to = addrparse_header(to)?;
+    for to_addr in to.iter() {
+        match to_addr {
+            mailparse::MailAddr::Single(ref info) => {
+                // Addresses should be of existing members (Alice and Bob) and not Claire.
+                assert_ne!(info.addr, claire_addr);
+            }
+            mailparse::MailAddr::Group(_) => {
+                panic!("Group addresses are not expected here");
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Tests that standard IMF header "From:" comes before non-standard "Autocrypt:" header.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_from_before_autocrypt() -> Result<()> {
+    // create chat with bob
+    let t = TestContext::new_alice().await;
+    let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+    // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
+    // make sure, `Subject:` stays in the outer header (imf header)
+    let mut msg = Message::new_text("this is the text!".to_string());
+
+    let sent_msg = t.send_msg(chat.id, &mut msg).await;
+    let payload = sent_msg.payload();
+
+    assert_eq!(payload.match_indices("Autocrypt:").count(), 1);
+    assert_eq!(payload.match_indices("From:").count(), 1);
+
+    assert!(payload.match_indices("From:").next() < payload.match_indices("Autocrypt:").next());
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_protected_headers_directive() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    let bob = tcm.bob().await;
+    let chat = tcm
+        .send_recv_accept(&alice, &bob, "alice->bob")
+        .await
+        .chat_id;
+
+    // Now Bob can send an encrypted message to Alice.
+    let mut msg = Message::new(Viewtype::File);
+    // Long messages are truncated and MimeMessage::decoded_data is set for them. We need
+    // decoded_data to check presence of the necessary headers.
+    msg.set_text("a".repeat(constants::DC_DESIRED_TEXT_LEN + 1));
+    msg.set_file_from_bytes(&bob, "foo.bar", "content".as_bytes(), None)?;
+    let sent = bob.send_msg(chat, &mut msg).await;
+    assert!(msg.get_showpadlock());
+    assert!(sent.payload.contains("\r\nSubject: [...]\r\n"));
+
+    let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?;
+    let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n");
+    let part = payload.next().unwrap();
+    assert_eq!(
+        part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+            .count(),
+        1
+    );
+    assert_eq!(part.match_indices("Subject:").count(), 1);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_dont_remove_self() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let first_group = alice
+        .create_group_with_members(ProtectionStatus::Unprotected, "First group", &[bob])
+        .await;
+    alice.send_text(first_group, "Hi! I created a group.").await;
+    remove_contact_from_chat(alice, first_group, ContactId::SELF).await?;
+    alice.pop_sent_msg().await;
+
+    let second_group = alice
+        .create_group_with_members(ProtectionStatus::Unprotected, "First group", &[bob])
+        .await;
+    let sent = alice
+        .send_text(second_group, "Hi! I created another group.")
+        .await;
+
+    println!("{}", sent.payload);
+    let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes(), None)
+        .await
+        .unwrap();
+    assert_eq!(
+        mime_message.get_header(HeaderDef::ChatGroupPastMembers),
+        None
+    );
+    assert_eq!(
+        mime_message.chat_group_member_timestamps().unwrap().len(),
+        1 // There is a timestamp for Bob, not for Alice
+    );
+
+    Ok(())
+}

src/mimeparser.rs

@@ -10,8 +10,8 @@ use anyhow::{bail, Context as _, Result};
 use deltachat_contact_tools::{addr_cmp, addr_normalize, sanitize_bidi_characters};
 use deltachat_derive::{FromSql, ToSql};
 use format_flowed::unformat_flowed;
-use lettre_email::mime::Mime;
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
+use mime::Mime;
 
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::authres::handle_authres;
@@ -290,7 +290,10 @@ impl MimeMessage {
 
                     // For now only avatar headers can be hidden.
                     if !headers.contains_key(&key)
-                        && (key == "chat-user-avatar" || key == "chat-group-avatar")
+                        && (key == "chat-user-avatar"
+                            || key == "chat-group-avatar"
+                            || key == "chat-delete"
+                            || key == "chat-edit")
                     {
                         headers.insert(key.to_string(), field.get_value());
                     }
@@ -443,11 +446,14 @@ impl MimeMessage {
                     HeaderDef::ChatGroupId,
                     HeaderDef::ChatGroupName,
                     HeaderDef::ChatGroupNameChanged,
+                    HeaderDef::ChatGroupNameTimestamp,
                     HeaderDef::ChatGroupAvatar,
                     HeaderDef::ChatGroupMemberRemoved,
                     HeaderDef::ChatGroupMemberAdded,
                     HeaderDef::ChatGroupMemberTimestamps,
                     HeaderDef::ChatGroupPastMembers,
+                    HeaderDef::ChatDelete,
+                    HeaderDef::ChatEdit,
                 ] {
                     headers.remove(h.get_headername());
                 }

src/mimeparser/mimeparser_tests.rs

@@ -312,6 +312,19 @@ fn test_mailparse_content_type() {
     );
 }
 
+/// Test to reproduce
+/// <https://github.com/staktrace/mailparse/issues/130>.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_mailparse_0_16_0_panic() {
+    let context = TestContext::new_alice().await;
+    let raw = include_bytes!("../../test-data/message/mailparse-0.16.0-panic.eml");
+
+    // There should be an error, but no panic.
+    assert!(MimeMessage::from_bytes(&context.ctx, &raw[..], None)
+        .await
+        .is_err());
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_parse_first_addr() {
     let context = TestContext::new().await;
@@ -785,7 +798,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
 
     // Make sure the file is there even though the html is wrong:
     let param = &message.parts[0].param;
-    let blob: BlobObject = param.get_blob(Param::File, &t).await.unwrap().unwrap();
+    let blob: BlobObject = param.get_file_blob(&t).unwrap().unwrap();
     let f = tokio::fs::File::open(blob.to_abs_path()).await.unwrap();
     let size = f.metadata().await.unwrap().len();
     assert_eq!(size, 154);
@@ -1871,8 +1884,7 @@ This is the epilogue.  It is also to be ignored.";
     assert_eq!(mimeparser.parts[0].typ, Viewtype::File);
     let blob: BlobObject = mimeparser.parts[0]
         .param
-        .get_blob(Param::File, &context)
-        .await
+        .get_file_blob(&context)
         .unwrap()
         .unwrap();
     assert_eq!(
@@ -1883,8 +1895,7 @@ This is the epilogue.  It is also to be ignored.";
     assert_eq!(mimeparser.parts[1].typ, Viewtype::File);
     let blob: BlobObject = mimeparser.parts[1]
         .param
-        .get_blob(Param::File, &context)
-        .await
+        .get_file_blob(&context)
         .unwrap()
         .unwrap();
     assert_eq!(

src/net/http.rs

@@ -167,7 +167,7 @@ async fn http_cache_get(context: &Context, url: &str) -> Result<Option<(Response
     };
     let is_stale = now > stale_timestamp;
 
-    let blob_object = BlobObject::from_name(context, blob_name)?;
+    let blob_object = BlobObject::from_name(context, &blob_name)?;
     let blob_abs_path = blob_object.to_abs_path();
     let blob = match fs::read(blob_abs_path)
         .await
@@ -253,7 +253,7 @@ async fn fetch_url(context: &Context, original_url: &str) -> Result<Response> {
                 .headers()
                 .get_all("location")
                 .iter()
-                .last()
+                .next_back()
                 .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
                 .to_str()?;
             info!(context, "Following redirect to {}", header);

src/param.rs

@@ -3,6 +3,7 @@ use std::fmt;
 use std::path::PathBuf;
 use std::str;
 
+use anyhow::ensure;
 use anyhow::{bail, Error, Result};
 use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
@@ -205,7 +206,15 @@ pub enum Param {
 
     /// For messages: Whether [crate::message::Viewtype::Sticker] should be forced.
     ForceSticker = b'X',
-    // 'L' was defined as ProtectionSettingsTimestamp for Chats, however, never used in production.
+
+    /// For messages: Message is a deletion request. The value is a list of rfc724_mid of the messages to delete.
+    DeleteRequestFor = b'M',
+
+    /// For messages: Message is a text edit message. the value of this parameter is the rfc724_mid of the original message.
+    TextEditFor = b'I',
+
+    /// For messages: Message text was edited.
+    IsEdited = b'L',
 }
 
 /// An object for handling key=value parameter lists.
@@ -290,6 +299,9 @@ impl Params {
 
     /// Set the given key to the passed in value.
     pub fn set(&mut self, key: Param, value: impl ToString) -> &mut Self {
+        if key == Param::File {
+            debug_assert!(value.to_string().starts_with("$BLOBDIR/"));
+        }
         self.inner.insert(key, value.to_string());
         self
     }
@@ -352,59 +364,20 @@ impl Params {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
-    /// Gets the given parameter and parse as [ParamsFile].
-    ///
-    /// See also [Params::get_blob] and [Params::get_path] which may
-    /// be more convenient.
-    pub fn get_file<'a>(&self, key: Param, context: &'a Context) -> Result<Option<ParamsFile<'a>>> {
-        let val = match self.get(key) {
-            Some(val) => val,
-            None => return Ok(None),
-        };
-        ParamsFile::from_param(context, val).map(Some)
-    }
-
-    /// Gets the parameter and returns a [BlobObject] for it.
-    ///
-    /// This parses the parameter value as a [ParamsFile] and than
-    /// tries to return a [BlobObject] for that file.  If the file is
-    /// not yet a valid blob, one will be created by copying the file.
-    ///
-    /// Note that in the [ParamsFile::FsPath] case the blob can be
-    /// created without copying if the path already refers to a valid
-    /// blob.  If so a [BlobObject] will be returned.
-    pub async fn get_blob<'a>(
-        &self,
-        key: Param,
-        context: &'a Context,
-    ) -> Result<Option<BlobObject<'a>>> {
-        let val = match self.get(key) {
-            Some(val) => val,
-            None => return Ok(None),
-        };
-        let file = ParamsFile::from_param(context, val)?;
-        let blob = match file {
-            ParamsFile::FsPath(path) => BlobObject::new_from_path(context, &path).await?,
-            ParamsFile::Blob(blob) => blob,
+    /// Returns a [BlobObject] for the [Param::File] parameter.
+    pub fn get_file_blob<'a>(&self, context: &'a Context) -> Result<Option<BlobObject<'a>>> {
+        let Some(val) = self.get(Param::File) else {
+            return Ok(None);
         };
+        ensure!(val.starts_with("$BLOBDIR/"));
+        let blob = BlobObject::from_name(context, val)?;
         Ok(Some(blob))
     }
 
-    /// Gets the parameter and returns a [PathBuf] for it.
-    ///
-    /// This parses the parameter value as a [ParamsFile] and returns
-    /// a [PathBuf] to the file.
-    pub fn get_path(&self, key: Param, context: &Context) -> Result<Option<PathBuf>> {
-        let val = match self.get(key) {
-            Some(val) => val,
-            None => return Ok(None),
-        };
-        let file = ParamsFile::from_param(context, val)?;
-        let path = match file {
-            ParamsFile::FsPath(path) => path,
-            ParamsFile::Blob(blob) => blob.to_abs_path(),
-        };
-        Ok(Some(path))
+    /// Returns a [PathBuf] for the [Param::File] parameter.
+    pub fn get_file_path(&self, context: &Context) -> Result<Option<PathBuf>> {
+        let blob = self.get_file_blob(context)?;
+        Ok(blob.map(|p| p.to_abs_path()))
     }
 
     /// Set the given parameter to the passed in `i32`.
@@ -426,48 +399,18 @@ impl Params {
     }
 }
 
-/// The value contained in [Param::File].
-///
-/// Because the only way to construct this object is from a valid
-/// UTF-8 string it is always safe to convert the value contained
-/// within the [ParamsFile::FsPath] back to a [String] or [&str].
-/// Despite the type itself does not guarantee this.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum ParamsFile<'a> {
-    FsPath(PathBuf),
-    Blob(BlobObject<'a>),
-}
-
-impl<'a> ParamsFile<'a> {
-    /// Parse the [Param::File] value into an object.
-    ///
-    /// If the value was stored into the [Params] correctly this
-    /// should not fail.
-    pub fn from_param(context: &'a Context, src: &str) -> Result<ParamsFile<'a>> {
-        let param = match src.starts_with("$BLOBDIR/") {
-            true => ParamsFile::Blob(BlobObject::from_name(context, src.to_string())?),
-            false => ParamsFile::FsPath(PathBuf::from(src)),
-        };
-        Ok(param)
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use std::path::Path;
     use std::str::FromStr;
 
-    use tokio::fs;
-
     use super::*;
-    use crate::test_utils::TestContext;
 
     #[test]
     fn test_dc_param() {
-        let mut p1: Params = "a=1\nf=2\nc=3".parse().unwrap();
+        let mut p1: Params = "a=1\nw=2\nc=3".parse().unwrap();
 
         assert_eq!(p1.get_int(Param::Forwarded), Some(1));
-        assert_eq!(p1.get_int(Param::File), Some(2));
+        assert_eq!(p1.get_int(Param::Width), Some(2));
         assert_eq!(p1.get_int(Param::Height), None);
         assert!(!p1.exists(Param::Height));
 
@@ -478,13 +421,13 @@ mod tests {
         let mut p1 = Params::new();
 
         p1.set(Param::Forwarded, "foo")
-            .set_int(Param::File, 2)
+            .set_int(Param::Width, 2)
             .remove(Param::GuaranteeE2ee)
             .set_int(Param::Duration, 4);
 
-        assert_eq!(p1.to_string(), "a=foo\nd=4\nf=2");
+        assert_eq!(p1.to_string(), "a=foo\nd=4\nw=2");
 
-        p1.remove(Param::File);
+        p1.remove(Param::Width);
 
         assert_eq!(p1.to_string(), "a=foo\nd=4",);
         assert_eq!(p1.len(), 2);
@@ -506,56 +449,6 @@ mod tests {
         assert_eq!(params.to_string().parse::<Params>().unwrap(), params);
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_params_file_fs_path() {
-        let t = TestContext::new().await;
-        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t, "/foo/bar/baz").unwrap() {
-            assert_eq!(p, Path::new("/foo/bar/baz"));
-        } else {
-            panic!("Wrong enum variant");
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_params_file_blob() {
-        let t = TestContext::new().await;
-        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t, "$BLOBDIR/foo").unwrap() {
-            assert_eq!(b.as_name(), "$BLOBDIR/foo");
-        } else {
-            panic!("Wrong enum variant");
-        }
-    }
-
-    // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_params_get_fileparam() {
-        let t = TestContext::new().await;
-        let fname = t.dir.path().join("foo");
-        let mut p = Params::new();
-        p.set(Param::File, fname.to_str().unwrap());
-
-        let file = p.get_file(Param::File, &t).unwrap().unwrap();
-        assert_eq!(file, ParamsFile::FsPath(fname.clone()));
-
-        let path: PathBuf = p.get_path(Param::File, &t).unwrap().unwrap();
-        assert_eq!(path, fname);
-
-        fs::write(fname, b"boo").await.unwrap();
-        let blob = p.get_blob(Param::File, &t).await.unwrap().unwrap();
-        assert!(blob.as_file_name().starts_with("foo"));
-
-        // Blob in blobdir, expect blob.
-        let bar_path = t.get_blobdir().join("bar");
-        p.set(Param::File, bar_path.to_str().unwrap());
-        let blob = p.get_blob(Param::File, &t).await.unwrap().unwrap();
-        assert_eq!(blob, BlobObject::from_name(&t, "bar".to_string()).unwrap());
-
-        p.remove(Param::File);
-        assert!(p.get_file(Param::File, &t).unwrap().is_none());
-        assert!(p.get_path(Param::File, &t).unwrap().is_none());
-        assert!(p.get_blob(Param::File, &t).await.unwrap().is_none());
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_params_unknown_key() -> Result<()> {
         // 'Z' is used as a key that is known to be unused; these keys should be ignored silently by definition.

src/peer_channels.rs

@@ -25,7 +25,6 @@
 
 use anyhow::{anyhow, bail, Context as _, Result};
 use data_encoding::BASE32_NOPAD;
-use email::Header;
 use futures_lite::StreamExt;
 use iroh::{Endpoint, NodeAddr, NodeId, PublicKey, RelayMap, RelayMode, RelayUrl, SecretKey};
 use iroh_gossip::net::{Event, Gossip, GossipEvent, JoinOptions, GOSSIP_ALPN};
@@ -40,7 +39,6 @@ use url::Url;
 use crate::chat::send_msg;
 use crate::config::Config;
 use crate::context::Context;
-use crate::headerdef::HeaderDef;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::EventType;
@@ -496,14 +494,11 @@ fn create_random_topic() -> TopicId {
 
 /// Creates `Iroh-Gossip-Header` with a new random topic
 /// and stores the topic for the message.
-pub(crate) async fn create_iroh_header(ctx: &Context, msg_id: MsgId) -> Result<Header> {
+pub(crate) async fn create_iroh_header(ctx: &Context, msg_id: MsgId) -> Result<String> {
     let topic = create_random_topic();
     insert_topic_stub(ctx, msg_id, topic).await?;
     let topic_string = BASE32_NOPAD.encode(topic.as_bytes()).to_ascii_lowercase();
-    Ok(Header::new(
-        HeaderDef::IrohGossipTopic.get_headername().to_string(),
-        topic_string,
-    ))
+    Ok(topic_string)
 }
 
 async fn subscribe_loop(

src/peerstate.rs

@@ -676,14 +676,7 @@ impl Peerstate {
                 let lastmsg = Message::load_from_db(context, *msg_id).await?;
                 lastmsg.timestamp_sort
             } else {
-                context
-                    .sql
-                    .query_get_value(
-                        "SELECT created_timestamp FROM chats WHERE id=?;",
-                        (chat_id,),
-                    )
-                    .await?
-                    .unwrap_or(0)
+                chat_id.created_timestamp(context).await?
             };
 
             if let PeerstateChange::Aeap(new_addr) = &change {

src/qr.rs

@@ -40,7 +40,11 @@ const HTTPS_SCHEME: &str = "https://";
 const SHADOWSOCKS_SCHEME: &str = "ss://";
 
 /// Backup transfer based on iroh-net.
-pub(crate) const DCBACKUP2_SCHEME: &str = "DCBACKUP2:";
+pub(crate) const DCBACKUP_SCHEME_PREFIX: &str = "DCBACKUP";
+
+/// Version written to Backups and Backup-QR-Codes.
+/// Imports will fail when they have a larger version.
+pub(crate) const DCBACKUP_VERSION: i32 = 2;
 
 /// Scanned QR code.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -118,6 +122,9 @@ pub enum Qr {
         auth_token: String,
     },
 
+    /// The QR code is a backup, but it is too new. The user has to update its Delta Chat.
+    BackupTooNew {},
+
     /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         /// Server domain name.
@@ -296,7 +303,7 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result<Qr> {
         decode_tg_socks_proxy(context, qr)?
     } else if qr.starts_with(SHADOWSOCKS_SCHEME) {
         decode_shadowsocks_proxy(qr)?
-    } else if starts_with_ignore_case(qr, DCBACKUP2_SCHEME) {
+    } else if starts_with_ignore_case(qr, DCBACKUP_SCHEME_PREFIX) {
         let qr_fixed = fix_add_second_device_qr(qr);
         decode_backup2(&qr_fixed)?
     } else if qr.starts_with(MAILTO_SCHEME) {
@@ -367,7 +374,9 @@ pub fn format_backup(qr: &Qr) -> Result<String> {
             ref auth_token,
         } => {
             let node_addr = serde_json::to_string(node_addr)?;
-            Ok(format!("{DCBACKUP2_SCHEME}{auth_token}&{node_addr}"))
+            Ok(format!(
+                "{DCBACKUP_SCHEME_PREFIX}{DCBACKUP_VERSION}:{auth_token}&{node_addr}"
+            ))
         }
         _ => Err(anyhow!("Not a backup QR code")),
     }
@@ -643,11 +652,19 @@ fn decode_shadowsocks_proxy(qr: &str) -> Result<Qr> {
     })
 }
 
-/// Decodes a [`DCBACKUP2_SCHEME`] QR code.
+/// Decodes a `DCBACKUP` QR code.
 fn decode_backup2(qr: &str) -> Result<Qr> {
-    let payload = qr
-        .strip_prefix(DCBACKUP2_SCHEME)
-        .ok_or_else(|| anyhow!("Invalid DCBACKUP2 scheme"))?;
+    let version_and_payload = qr
+        .strip_prefix(DCBACKUP_SCHEME_PREFIX)
+        .ok_or_else(|| anyhow!("Invalid DCBACKUP scheme"))?;
+    let (version, payload) = version_and_payload
+        .split_once(':')
+        .context("DCBACKUP scheme separator missing")?;
+    let version: i32 = version.parse().context("Not a valid number")?;
+    if version > DCBACKUP_VERSION {
+        return Ok(Qr::BackupTooNew {});
+    }
+
     let (auth_token, node_addr) = payload
         .split_once('&')
         .context("Backup QR code has no separator")?;

src/qr/qr_tests.rs

@@ -917,5 +917,11 @@ async fn test_decode_backup() -> Result<()> {
     let qr = check_qr(&ctx, r#"DCBACKUP2:AIvFjRFBt_aMiisSZ8P33JqY&{"node_id":"buzkyd4x76w66qtanjk5fm6ikeuo4quletajowsl3a3p7l6j23pa","info":{"relay_url":null,"direct_addresses":["192.168.1.5:12345"]}}"#).await?;
     assert!(matches!(qr, Qr::Backup2 { .. }));
 
+    let qr = check_qr(&ctx, r#"DCBACKUP9:from-the-future"#).await?;
+    assert!(matches!(qr, Qr::BackupTooNew { .. }));
+
+    let qr = check_qr(&ctx, r#"DCBACKUP99:far-from-the-future"#).await?;
+    assert!(matches!(qr, Qr::BackupTooNew { .. }));
+
     Ok(())
 }

src/reaction.rs

@@ -398,7 +398,7 @@ mod tests {
     use deltachat_contact_tools::ContactAddress;
 
     use super::*;
-    use crate::chat::{forward_msgs, get_chat_msgs, send_text_msg};
+    use crate::chat::{forward_msgs, get_chat_msgs, marknoticed_chat, send_text_msg};
     use crate::chatlist::Chatlist;
     use crate::config::Config;
     use crate::contact::{Contact, Origin};
@@ -623,7 +623,9 @@ Here's my footer -- bob@example.net"
             .get_matching_opt(t, |evt| {
                 matches!(
                     evt,
-                    EventType::IncomingReaction { .. } | EventType::IncomingMsg { .. }
+                    EventType::IncomingReaction { .. }
+                        | EventType::IncomingMsg { .. }
+                        | EventType::MsgsChanged { .. }
                 )
             })
             .await;
@@ -667,7 +669,8 @@ Here's my footer -- bob@example.net"
         assert_eq!(get_chat_msgs(&bob, bob_msg.chat_id).await?.len(), 2);
 
         let bob_reaction_msg = bob.pop_sent_msg().await;
-        alice.recv_msg_trash(&bob_reaction_msg).await;
+        let alice_reaction_msg = alice.recv_msg_hidden(&bob_reaction_msg).await;
+        assert_eq!(alice_reaction_msg.state, MessageState::InFresh);
         assert_eq!(get_chat_msgs(&alice, chat_alice.id).await?.len(), 2);
 
         let reactions = get_msg_reactions(&alice, alice_msg.sender_msg_id).await?;
@@ -691,6 +694,20 @@ Here's my footer -- bob@example.net"
         .await?;
         expect_no_unwanted_events(&alice).await;
 
+        marknoticed_chat(&alice, chat_alice.id).await?;
+        assert_eq!(
+            alice_reaction_msg.id.get_state(&alice).await?,
+            MessageState::InSeen
+        );
+        // Reactions don't request MDNs.
+        assert_eq!(
+            alice
+                .sql
+                .count("SELECT COUNT(*) FROM smtp_mdns", ())
+                .await?,
+            0
+        );
+
         // Alice reacts to own message.
         send_reaction(&alice, alice_msg.sender_msg_id, "👍 😀")
             .await
@@ -730,7 +747,7 @@ Here's my footer -- bob@example.net"
         bob_msg1.chat_id.accept(&bob).await?;
         send_reaction(&bob, bob_msg1.id, "👍").await?;
         let bob_send_reaction = bob.pop_sent_msg().await;
-        alice.recv_msg_trash(&bob_send_reaction).await;
+        alice.recv_msg_hidden(&bob_send_reaction).await;
         expect_incoming_reactions_event(
             &alice,
             alice_chat.id,
@@ -899,7 +916,7 @@ Here's my footer -- bob@example.net"
         let bob_reaction_msg = bob.pop_sent_msg().await;
 
         // Alice receives a reaction.
-        alice.recv_msg_trash(&bob_reaction_msg).await;
+        alice.recv_msg_hidden(&bob_reaction_msg).await;
 
         let reactions = get_msg_reactions(&alice, alice_msg_id).await?;
         assert_eq!(reactions.to_string(), "👍1");
@@ -951,7 +968,7 @@ Here's my footer -- bob@example.net"
         {
             send_reaction(&alice2, alice2_msg.id, "👍").await?;
             let msg = alice2.pop_sent_msg().await;
-            alice1.recv_msg_trash(&msg).await;
+            alice1.recv_msg_hidden(&msg).await;
         }
 
         // Check that the status is still the same.
@@ -973,7 +990,7 @@ Here's my footer -- bob@example.net"
         let alice1_msg = alice1.recv_msg(&alice0.pop_sent_msg().await).await;
 
         send_reaction(&alice0, alice0_msg_id, "👀").await?;
-        alice1.recv_msg_trash(&alice0.pop_sent_msg().await).await;
+        alice1.recv_msg_hidden(&alice0.pop_sent_msg().await).await;
 
         expect_reactions_changed_event(&alice0, chat_id, alice0_msg_id, ContactId::SELF).await?;
         expect_reactions_changed_event(&alice1, alice1_msg.chat_id, alice1_msg.id, ContactId::SELF)

src/receive_imf.rs

@@ -15,7 +15,7 @@ use regex::Regex;
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::{Blocked, Chattype, ShowEmails, DC_CHAT_ID_TRASH};
+use crate::constants::{Blocked, Chattype, ShowEmails, DC_CHAT_ID_TRASH, EDITED_PREFIX};
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
 use crate::debug_logging::maybe_set_logging_xdc_inner;
@@ -54,6 +54,9 @@ pub struct ReceivedMsg {
     /// Received message state.
     pub state: MessageState,
 
+    /// Whether the message is hidden.
+    pub hidden: bool,
+
     /// Message timestamp for sorting.
     pub sort_timestamp: i64,
 
@@ -192,6 +195,7 @@ pub(crate) async fn receive_imf_inner(
             return Ok(Some(ReceivedMsg {
                 chat_id: DC_CHAT_ID_TRASH,
                 state: MessageState::Undefined,
+                hidden: false,
                 sort_timestamp: 0,
                 msg_ids,
                 needs_delete_job: false,
@@ -364,7 +368,7 @@ pub(crate) async fn receive_imf_inner(
     if mime_parser.get_header(HeaderDef::SecureJoin).is_some() {
         let res;
         if mime_parser.incoming {
-            res = handle_securejoin_handshake(context, &mime_parser, from_id)
+            res = handle_securejoin_handshake(context, &mut mime_parser, from_id)
                 .await
                 .context("error in Secure-Join message handling")?;
 
@@ -385,6 +389,7 @@ pub(crate) async fn receive_imf_inner(
                 received_msg = Some(ReceivedMsg {
                     chat_id: DC_CHAT_ID_TRASH,
                     state: MessageState::InSeen,
+                    hidden: false,
                     sort_timestamp: mime_parser.timestamp_sent,
                     msg_ids: vec![msg_id],
                     needs_delete_job: res == securejoin::HandshakeMessage::Done,
@@ -619,7 +624,9 @@ pub(crate) async fn receive_imf_inner(
         }
     }
 
-    if let Some(replace_chat_id) = replace_chat_id {
+    if received_msg.hidden {
+        // No need to emit an event about the changed message
+    } else if let Some(replace_chat_id) = replace_chat_id {
         context.emit_msgs_changed_without_msg_id(replace_chat_id);
     } else if !chat_id.is_trash() {
         let fresh = received_msg.state == MessageState::InFresh;
@@ -723,7 +730,7 @@ async fn add_parts(
     let mut chat_id_blocked = Blocked::Not;
 
     let mut better_msg = None;
-    let mut group_changes_msgs = (Vec::new(), None);
+    let mut group_changes = GroupChangesInfo::default();
     if mime_parser.is_system_message == SystemMessage::LocationStreamingEnabled {
         better_msg = Some(stock_str::msg_location_enabled_by(context, from_id).await);
     }
@@ -778,7 +785,7 @@ async fn add_parts(
     // (of course, the user can add other chats manually later)
     let to_id: ContactId;
     let state: MessageState;
-    let mut hidden = false;
+    let mut hidden = is_reaction;
     let mut needs_delete_job = false;
     let mut restore_protection = false;
 
@@ -915,7 +922,7 @@ async fn add_parts(
                 }
             }
 
-            group_changes_msgs = apply_group_changes(
+            group_changes = apply_group_changes(
                 context,
                 mime_parser,
                 group_chat_id,
@@ -1036,8 +1043,9 @@ async fn add_parts(
         state = if seen
             || fetching_existing_messages
             || is_mdn
-            || is_reaction
             || chat_id_blocked == Blocked::Yes
+            || group_changes.silent
+        // No check for `hidden` because only reactions are such and they should be `InFresh`.
         {
             MessageState::InSeen
         } else {
@@ -1185,7 +1193,7 @@ async fn add_parts(
         }
 
         if let Some(chat_id) = chat_id {
-            group_changes_msgs = apply_group_changes(
+            group_changes = apply_group_changes(
                 context,
                 mime_parser,
                 chat_id,
@@ -1246,14 +1254,10 @@ async fn add_parts(
     }
 
     let orig_chat_id = chat_id;
-    let mut chat_id = if is_reaction {
+    let mut chat_id = chat_id.unwrap_or_else(|| {
+        info!(context, "No chat id for message (TRASH).");
         DC_CHAT_ID_TRASH
-    } else {
-        chat_id.unwrap_or_else(|| {
-            info!(context, "No chat id for message (TRASH).");
-            DC_CHAT_ID_TRASH
-        })
-    };
+    });
 
     // Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded.
     let mut ephemeral_timer = if is_partial_download.is_some() {
@@ -1292,8 +1296,18 @@ async fn add_parts(
         && !mime_parser.parts.is_empty()
         && chat_id.get_ephemeral_timer(context).await? != ephemeral_timer
     {
+        let chat_contacts =
+            HashSet::<ContactId>::from_iter(chat::get_chat_contacts(context, chat_id).await?);
+        let is_from_in_chat =
+            !chat_contacts.contains(&ContactId::SELF) || chat_contacts.contains(&from_id);
+
         info!(context, "Received new ephemeral timer value {ephemeral_timer:?} for chat {chat_id}, checking if it should be applied.");
-        if is_dc_message == MessengerMessage::Yes
+        if !is_from_in_chat {
+            warn!(
+                context,
+                "Ignoring ephemeral timer change to {ephemeral_timer:?} for chat {chat_id} because sender {from_id} is not a member.",
+            );
+        } else if is_dc_message == MessengerMessage::Yes
             && get_previous_message(context, mime_parser)
                 .await?
                 .map(|p| p.ephemeral_timer)
@@ -1382,18 +1396,14 @@ async fn add_parts(
         }
     }
 
-    // Ensure replies to messages are sorted after the parent message.
-    //
-    // This is useful in a case where sender clocks are not
-    // synchronized and parent message has a Date: header with a
-    // timestamp higher than reply timestamp.
-    //
-    // This does not help if parent message arrives later than the
-    // reply.
-    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
-    let sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
-        std::cmp::max(sort_timestamp, parent_timestamp)
-    });
+    let sort_timestamp = tweak_sort_timestamp(
+        context,
+        mime_parser,
+        group_changes.silent,
+        chat_id,
+        sort_timestamp,
+    )
+    .await?;
 
     // if the mime-headers should be saved, find out its size
     // (the mime-header ends with an empty line)
@@ -1437,24 +1447,23 @@ async fn add_parts(
 
     let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len());
 
-    if let Some(msg) = group_changes_msgs.1 {
+    if let Some(m) = group_changes.better_msg {
         match &better_msg {
-            None => better_msg = Some(msg),
+            None => better_msg = Some(m),
             Some(_) => {
-                if !msg.is_empty() {
-                    group_changes_msgs.0.push(msg)
+                if !m.is_empty() {
+                    group_changes.extra_msgs.push((m, is_system_message))
                 }
             }
         }
     }
 
-    for group_changes_msg in group_changes_msgs.0 {
-        // Currently all additional group changes messages are "Member added".
+    for (group_changes_msg, cmd) in group_changes.extra_msgs {
         chat::add_info_msg_with_cmd(
             context,
             chat_id,
             &group_changes_msg,
-            SystemMessage::MemberAddedToGroup,
+            cmd,
             sort_timestamp,
             None,
             None,
@@ -1490,6 +1499,73 @@ async fn add_parts(
         }
     }
 
+    if let Some(rfc724_mid) = mime_parser.get_header(HeaderDef::ChatEdit) {
+        chat_id = DC_CHAT_ID_TRASH;
+        if let Some((original_msg_id, _)) = rfc724_mid_exists(context, rfc724_mid).await? {
+            if let Some(mut original_msg) =
+                Message::load_from_db_optional(context, original_msg_id).await?
+            {
+                if original_msg.from_id == from_id {
+                    if let Some(part) = mime_parser.parts.first() {
+                        let edit_msg_showpadlock = part
+                            .param
+                            .get_bool(Param::GuaranteeE2ee)
+                            .unwrap_or_default();
+                        if edit_msg_showpadlock || !original_msg.get_showpadlock() {
+                            let new_text =
+                                part.msg.strip_prefix(EDITED_PREFIX).unwrap_or(&part.msg);
+                            chat::save_text_edit_to_db(context, &mut original_msg, new_text)
+                                .await?;
+                        } else {
+                            warn!(context, "Edit message: Not encrypted.");
+                        }
+                    }
+                } else {
+                    warn!(context, "Edit message: Bad sender.");
+                }
+            } else {
+                warn!(context, "Edit message: Database entry does not exist.");
+            }
+        } else {
+            warn!(
+                context,
+                "Edit message: rfc724_mid {rfc724_mid:?} not found."
+            );
+        }
+    } else if let Some(rfc724_mid_list) = mime_parser.get_header(HeaderDef::ChatDelete) {
+        chat_id = DC_CHAT_ID_TRASH;
+        if let Some(part) = mime_parser.parts.first() {
+            if part.param.get_bool(Param::GuaranteeE2ee).unwrap_or(false) {
+                let mut modified_chat_ids = HashSet::new();
+                let mut msg_ids = Vec::new();
+
+                let rfc724_mid_vec: Vec<&str> = rfc724_mid_list.split_whitespace().collect();
+                for rfc724_mid in rfc724_mid_vec {
+                    if let Some((msg_id, _)) =
+                        message::rfc724_mid_exists(context, rfc724_mid).await?
+                    {
+                        if let Some(msg) = Message::load_from_db_optional(context, msg_id).await? {
+                            if msg.from_id == from_id {
+                                message::delete_msg_locally(context, &msg).await?;
+                                msg_ids.push(msg.id);
+                                modified_chat_ids.insert(msg.chat_id);
+                            } else {
+                                warn!(context, "Delete message: Bad sender.");
+                            }
+                        } else {
+                            warn!(context, "Delete message: Database entry does not exist.");
+                        }
+                    } else {
+                        warn!(context, "Delete message: {rfc724_mid:?} not found.");
+                    }
+                }
+                message::delete_msgs_locally_done(context, &msg_ids, modified_chat_ids).await?;
+            } else {
+                warn!(context, "Delete message: Not encrypted.");
+            }
+        }
+    }
+
     let mut parts = mime_parser.parts.iter().peekable();
     while let Some(part) = parts.next() {
         if part.is_reaction {
@@ -1610,11 +1686,11 @@ RETURNING id
                     typ,
                     state,
                     is_dc_message,
-                    if trash { "" } else { msg },
-                    if trash { None } else { message::normalize_text(msg) },
-                    if trash { "" } else { &subject },
+                    if trash || hidden { "" } else { msg },
+                    if trash || hidden { None } else { message::normalize_text(msg) },
+                    if trash || hidden { "" } else { &subject },
                     // txt_raw might contain invalid utf8
-                    if trash { "" } else { &txt_raw },
+                    if trash || hidden { "" } else { &txt_raw },
                     if trash {
                         "".to_string()
                     } else {
@@ -1622,7 +1698,7 @@ RETURNING id
                     },
                     hidden,
                     part.bytes as isize,
-                    if (save_mime_headers || save_mime_modified) && !trash {
+                    if (save_mime_headers || save_mime_modified) && !(trash || hidden) {
                         mime_headers.clone()
                     } else {
                         Vec::new()
@@ -1682,9 +1758,7 @@ RETURNING id
             context,
             part.typ,
             chat_id,
-            part.param
-                .get_path(Param::File, context)
-                .unwrap_or_default(),
+            part.param.get(Param::Filename),
             *msg_id,
         )
         .await?;
@@ -1711,7 +1785,7 @@ RETURNING id
         "Message has {icnt} parts and is assigned to chat #{chat_id}."
     );
 
-    if !chat_id.is_trash() {
+    if !chat_id.is_trash() && !hidden {
         let mut chat = Chat::load_from_db(context, chat_id).await?;
 
         // In contrast to most other update-timestamps,
@@ -1749,6 +1823,7 @@ RETURNING id
     Ok(ReceivedMsg {
         chat_id,
         state,
+        hidden,
         sort_timestamp,
         msg_ids: created_db_entries,
         needs_delete_job,
@@ -1757,6 +1832,40 @@ RETURNING id
     })
 }
 
+async fn tweak_sort_timestamp(
+    context: &Context,
+    mime_parser: &mut MimeMessage,
+    silent: bool,
+    chat_id: ChatId,
+    sort_timestamp: i64,
+) -> Result<i64> {
+    // Ensure replies to messages are sorted after the parent message.
+    //
+    // This is useful in a case where sender clocks are not
+    // synchronized and parent message has a Date: header with a
+    // timestamp higher than reply timestamp.
+    //
+    // This does not help if parent message arrives later than the
+    // reply.
+    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
+    let mut sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
+        std::cmp::max(sort_timestamp, parent_timestamp)
+    });
+
+    // If the message should be silent,
+    // set the timestamp to be no more than the same as last message
+    // so that the chat is not sorted to the top of the chatlist.
+    if silent {
+        let last_msg_timestamp = if let Some(t) = chat_id.get_timestamp(context).await? {
+            t
+        } else {
+            chat_id.created_timestamp(context).await?
+        };
+        sort_timestamp = std::cmp::min(sort_timestamp, last_msg_timestamp);
+    }
+    Ok(sort_timestamp)
+}
+
 /// Saves attached locations to the database.
 ///
 /// Emits an event if at least one new location was added.
@@ -2205,11 +2314,23 @@ async fn update_chats_contacts_timestamps(
     Ok(modified)
 }
 
+/// The return type of [apply_group_changes].
+/// Contains information on which system messages
+/// should be shown in the chat.
+#[derive(Default)]
+struct GroupChangesInfo {
+    /// Optional: A better message that should replace the original system message.
+    /// If this is an empty string, the original system message should be trashed.
+    better_msg: Option<String>,
+    /// If true, the user should not be notified about the group change.
+    silent: bool,
+    /// A list of additional group changes messages that should be shown in the chat.
+    extra_msgs: Vec<(String, SystemMessage)>,
+}
+
 /// Apply group member list, name, avatar and protection status changes from the MIME message.
 ///
-/// Returns `Vec` of group changes messages and, optionally, a better message to replace the
-/// original system message. If the better message is empty, the original system message
-/// should be trashed.
+/// Returns [GroupChangesInfo].
 ///
 /// * `to_ids` - contents of the `To` and `Cc` headers.
 /// * `past_ids` - contents of the `Chat-Group-Past-Members` header.
@@ -2221,19 +2342,20 @@ async fn apply_group_changes(
     to_ids: &[ContactId],
     past_ids: &[ContactId],
     verified_encryption: &VerifiedEncryption,
-) -> Result<(Vec<String>, Option<String>)> {
+) -> Result<GroupChangesInfo> {
     if chat_id.is_special() {
         // Do not apply group changes to the trash chat.
-        return Ok((Vec::new(), None));
+        return Ok(GroupChangesInfo::default());
     }
     let mut chat = Chat::load_from_db(context, chat_id).await?;
     if chat.typ != Chattype::Group {
-        return Ok((Vec::new(), None));
+        return Ok(GroupChangesInfo::default());
     }
 
     let mut send_event_chat_modified = false;
     let (mut removed_id, mut added_id) = (None, None);
     let mut better_msg = None;
+    let mut silent = false;
 
     // True if a Delta Chat client has explicitly added our current primary address.
     let self_added =
@@ -2271,6 +2393,7 @@ async fn apply_group_changes(
         removed_id = Contact::lookup_id_by_addr(context, removed_addr, Origin::Unknown).await?;
         if let Some(id) = removed_id {
             better_msg = if id == from_id {
+                silent = true;
                 Some(stock_str::msg_group_left_local(context, from_id).await)
             } else {
                 Some(stock_str::msg_del_member_local(context, removed_addr, from_id).await)
@@ -2288,9 +2411,19 @@ async fn apply_group_changes(
         }
 
         better_msg = Some(stock_str::msg_add_member_local(context, added_addr, from_id).await);
-    } else if let Some(old_name) = mime_parser
+    }
+
+    let group_name_timestamp = mime_parser
+        .get_header(HeaderDef::ChatGroupNameTimestamp)
+        .and_then(|s| s.parse::<i64>().ok());
+    if let Some(old_name) = mime_parser
         .get_header(HeaderDef::ChatGroupNameChanged)
         .map(|s| s.trim())
+        .or(match group_name_timestamp {
+            Some(0) => None,
+            Some(_) => Some(chat.name.as_str()),
+            None => None,
+        })
     {
         if let Some(grpname) = mime_parser
             .get_header(HeaderDef::ChatGroupName)
@@ -2299,13 +2432,15 @@ async fn apply_group_changes(
         {
             let grpname = &sanitize_single_line(grpname);
             let old_name = &sanitize_single_line(old_name);
-            if chat_id
-                .update_timestamp(
-                    context,
-                    Param::GroupNameTimestamp,
-                    mime_parser.timestamp_sent,
-                )
-                .await?
+
+            let chat_group_name_timestamp =
+                chat.param.get_i64(Param::GroupNameTimestamp).unwrap_or(0);
+            let group_name_timestamp = group_name_timestamp.unwrap_or(mime_parser.timestamp_sent);
+            // To provide group name consistency, compare names if timestamps are equal.
+            if (chat_group_name_timestamp, grpname) < (group_name_timestamp, old_name)
+                && chat_id
+                    .update_timestamp(context, Param::GroupNameTimestamp, group_name_timestamp)
+                    .await?
             {
                 info!(context, "Updating grpname for chat {chat_id}.");
                 context
@@ -2314,10 +2449,18 @@ async fn apply_group_changes(
                     .await?;
                 send_event_chat_modified = true;
             }
-
-            better_msg = Some(stock_str::msg_grp_name(context, old_name, grpname, from_id).await);
+            if mime_parser
+                .get_header(HeaderDef::ChatGroupNameChanged)
+                .is_some()
+            {
+                better_msg.get_or_insert(
+                    stock_str::msg_grp_name(context, old_name, grpname, from_id).await,
+                );
+            }
         }
-    } else if let Some(value) = mime_parser.get_header(HeaderDef::ChatContent) {
+    }
+
+    if let (Some(value), None) = (mime_parser.get_header(HeaderDef::ChatContent), &better_msg) {
         if value == "group-avatar-changed" {
             if let Some(avatar_action) = &mime_parser.group_avatar {
                 // this is just an explicit message containing the group-avatar,
@@ -2497,7 +2640,11 @@ async fn apply_group_changes(
         context.emit_event(EventType::ChatModified(chat_id));
         chatlist_events::emit_chatlist_item_changed(context, chat_id);
     }
-    Ok((group_changes_msgs, better_msg))
+    Ok(GroupChangesInfo {
+        better_msg,
+        silent,
+        extra_msgs: group_changes_msgs,
+    })
 }
 
 /// Returns a list of strings that should be shown as info messages, informing about group membership changes.
@@ -2506,7 +2653,7 @@ async fn group_changes_msgs(
     added_ids: &HashSet<ContactId>,
     removed_ids: &HashSet<ContactId>,
     chat_id: ChatId,
-) -> Result<Vec<String>> {
+) -> Result<Vec<(String, SystemMessage)>> {
     let mut group_changes_msgs = Vec::new();
     if !added_ids.is_empty() {
         warn!(
@@ -2523,17 +2670,19 @@ async fn group_changes_msgs(
     group_changes_msgs.reserve(added_ids.len() + removed_ids.len());
     for contact_id in added_ids {
         let contact = Contact::get_by_id(context, *contact_id).await?;
-        group_changes_msgs.push(
+        group_changes_msgs.push((
             stock_str::msg_add_member_local(context, contact.get_addr(), ContactId::UNDEFINED)
                 .await,
-        );
+            SystemMessage::MemberAddedToGroup,
+        ));
     }
     for contact_id in removed_ids {
         let contact = Contact::get_by_id(context, *contact_id).await?;
-        group_changes_msgs.push(
+        group_changes_msgs.push((
             stock_str::msg_del_member_local(context, contact.get_addr(), ContactId::UNDEFINED)
                 .await,
-        );
+            SystemMessage::MemberRemovedFromGroup,
+        ));
     }
 
     Ok(group_changes_msgs)

src/receive_imf/receive_imf_tests.rs

@@ -1061,8 +1061,8 @@ async fn test_classic_mailing_list() -> Result<()> {
     let mime = sent.payload();
 
     println!("Sent mime message is:\n\n{mime}\n\n");
-    assert!(mime.contains("Content-Type: text/plain; charset=utf-8\r\n"));
-    assert!(mime.contains("Subject: =?utf-8?q?Re=3A_=5Bdelta-dev=5D_What=27s_up=3F?=\r\n"));
+    assert!(mime.contains("Content-Type: text/plain; charset=\"utf-8\"\r\n"));
+    assert!(mime.contains("Subject: Re: [delta-dev] What's up?\r\n"));
     assert!(mime.contains("MIME-Version: 1.0\r\n"));
     assert!(mime.contains("In-Reply-To: <38942@posteo.org>\r\n"));
     assert!(mime.contains("Chat-Version: 1.0\r\n"));
@@ -4705,8 +4705,10 @@ async fn test_outgoing_msg_forgery() -> Result<()> {
     // We need Bob only to encrypt the forged message to Alice's key, actually Bob doesn't
     // participate in the scenario.
     let bob = &TestContext::new().await;
+    assert_eq!(crate::key::load_self_secret_keyring(bob).await?.len(), 0);
     bob.configure_addr("bob@example.net").await;
     imex(bob, ImexMode::ImportSelfKeys, export_dir.path(), None).await?;
+    assert_eq!(crate::key::load_self_secret_keyring(bob).await?.len(), 1);
     let malice = &TestContext::new().await;
     malice.configure_addr(alice_addr).await;
 
@@ -4714,6 +4716,7 @@ async fn test_outgoing_msg_forgery() -> Result<()> {
         .send_recv_accept(bob, malice, "hi from bob")
         .await
         .chat_id;
+    assert_eq!(crate::key::load_self_secret_keyring(bob).await?.len(), 1);
 
     let sent_msg = malice.send_text(malice_chat_id, "hi from malice").await;
     let msg = alice.recv_msg(&sent_msg).await;
@@ -5425,6 +5428,40 @@ Hello!"
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_rename_chat_on_missing_message() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+    let chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "Group").await?;
+    add_to_chat_contacts_table(
+        &alice,
+        time(),
+        chat_id,
+        &[Contact::create(&alice, "bob", "bob@example.net").await?],
+    )
+    .await?;
+    send_text_msg(&alice, chat_id, "populate".to_string()).await?;
+    let bob_chat_id = bob.recv_msg(&alice.pop_sent_msg().await).await.chat_id;
+    bob_chat_id.accept(&bob).await?;
+
+    // Bob changes the group name. NB: If Bob does this too fast, it's not guaranteed that his group
+    // name wins because "Group-Name-Timestamp" may not increase.
+    SystemTime::shift(Duration::from_secs(3600));
+    chat::set_chat_name(&bob, bob_chat_id, "Renamed").await?;
+    bob.pop_sent_msg().await;
+
+    // Bob adds a new member.
+    let bob_orange = Contact::create(&bob, "orange", "orange@example.net").await?;
+    add_contact_to_chat(&bob, bob_chat_id, bob_orange).await?;
+    let add_msg = bob.pop_sent_msg().await;
+
+    // Alice only receives the member addition.
+    alice.recv_msg(&add_msg).await;
+    let chat = Chat::load_from_db(&alice, chat_id).await?;
+    assert_eq!(chat.get_name(), "Renamed");
+    Ok(())
+}
+
 /// Tests that creating a group
 /// is preferred over assigning message to existing
 /// chat based on `In-Reply-To` and `References`.
@@ -5612,7 +5649,7 @@ PGh0bWw+PGJvZHk+dGV4dDwvYm9keT5kYXRh
 
     assert_eq!(msg.get_filename().unwrap(), "test.HTML");
 
-    let blob = msg.param.get_blob(Param::File, alice).await?.unwrap();
+    let blob = msg.param.get_file_blob(alice)?.unwrap();
     assert_eq!(blob.suffix().unwrap(), "html");
 
     Ok(())

src/securejoin.rs

@@ -26,10 +26,8 @@ use crate::token;
 use crate::tools::time;
 
 mod bob;
-mod bobstate;
 mod qrinvite;
 
-pub(crate) use bobstate::BobState;
 use qrinvite::QrInvite;
 
 use crate::token::Namespace;
@@ -168,8 +166,7 @@ async fn securejoin(context: &Context, qr: &str) -> Result<ChatId> {
     bob::start_protocol(context, invite).await
 }
 
-/// Send handshake message from Alice's device;
-/// Bob's handshake messages are sent in `BobState::send_handshake_message()`.
+/// Send handshake message from Alice's device.
 async fn send_alice_handshake_msg(
     context: &Context,
     contact_id: ContactId,
@@ -259,7 +256,7 @@ pub(crate) enum HandshakeMessage {
     /// This leaves it on the IMAP server.  It means other devices on this account can
     /// receive and potentially process this message as well.  This is useful for example
     /// when the other device is running the protocol and has the relevant QR-code
-    /// information while this device does not have the joiner state ([`BobState`]).
+    /// information while this device does not have the joiner state.
     Ignore,
     /// The message should be further processed by incoming message handling.
     ///
@@ -281,7 +278,7 @@ pub(crate) enum HandshakeMessage {
 /// database; this is done by `receive_imf()` later on as needed.
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
-    mime_message: &MimeMessage,
+    mime_message: &mut MimeMessage,
     contact_id: ContactId,
 ) -> Result<HandshakeMessage> {
     if contact_id.is_special() {
@@ -479,15 +476,10 @@ pub(crate) async fn handle_securejoin_handshake(
         ====   Step 7 in "Setup verified contact" protocol   ====
         =======================================================*/
         "vc-contact-confirm" => {
-            if let Some(mut bobstate) = BobState::from_db(&context.sql).await? {
-                if !bobstate.is_msg_expected(context, step) {
-                    warn!(context, "Unexpected vc-contact-confirm.");
-                    return Ok(HandshakeMessage::Ignore);
-                }
-
-                bobstate.step_contact_confirm(context).await?;
-                bobstate.emit_progress(context, JoinerProgress::Succeeded);
-            }
+            context.emit_event(EventType::SecurejoinJoinerProgress {
+                contact_id,
+                progress: JoinerProgress::Succeeded.to_usize(),
+            });
             Ok(HandshakeMessage::Ignore)
         }
         "vg-member-added" => {
@@ -506,15 +498,22 @@ pub(crate) async fn handle_securejoin_handshake(
                 );
                 return Ok(HandshakeMessage::Propagate);
             }
-            if let Some(mut bobstate) = BobState::from_db(&context.sql).await? {
-                if !bobstate.is_msg_expected(context, step) {
-                    warn!(context, "Unexpected vg-member-added.");
-                    return Ok(HandshakeMessage::Propagate);
-                }
 
-                bobstate.step_contact_confirm(context).await?;
-                bobstate.emit_progress(context, JoinerProgress::Succeeded);
+            // Mark peer as backward verified.
+            //
+            // This is needed for the case when we join a non-protected group
+            // because in this case `Chat-Verified` header that otherwise
+            // sets backward verification is not sent.
+            if let Some(peerstate) = &mut mime_message.peerstate {
+                peerstate.backward_verified_key_id =
+                    Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
+                peerstate.save_to_db(&context.sql).await?;
             }
+
+            context.emit_event(EventType::SecurejoinJoinerProgress {
+                contact_id,
+                progress: JoinerProgress::Succeeded.to_usize(),
+            });
             Ok(HandshakeMessage::Propagate)
         }
 

src/securejoin/bob.rs

@@ -1,31 +1,40 @@
-//! Bob's side of SecureJoin handling.
-//!
-//! This are some helper functions around [`BobState`] which augment the state changes with
-//! the required user interactions.
+//! Bob's side of SecureJoin handling, the joiner-side.
 
 use anyhow::{Context as _, Result};
 
-use super::bobstate::{BobHandshakeStage, BobState};
 use super::qrinvite::QrInvite;
 use super::HandshakeMessage;
-use crate::chat::{is_contact_in_chat, ChatId, ProtectionStatus};
+use crate::chat::{self, is_contact_in_chat, ChatId, ProtectionStatus};
 use crate::constants::{self, Blocked, Chattype};
-use crate::contact::Contact;
+use crate::contact::Origin;
 use crate::context::Context;
 use crate::events::EventType;
+use crate::key::{load_self_public_key, DcKey};
+use crate::message::{Message, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
+use crate::param::Param;
+use crate::securejoin::{encrypted_and_signed, verify_sender_by_fingerprint, ContactId};
+use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::tools::{create_smeared_timestamp, time};
-use crate::{chat, stock_str};
 
 /// Starts the securejoin protocol with the QR `invite`.
 ///
-/// This will try to start the securejoin protocol for the given QR `invite`.  If it
-/// succeeded the protocol state will be tracked in `self`.
+/// This will try to start the securejoin protocol for the given QR `invite`.
+///
+/// If Bob already has Alice's key, he sends `AUTH` token
+/// and forgets about the invite.
+/// If Bob does not yet have Alice's key, he sends `vc-request`
+/// or `vg-request` message and stores a row in the `bobstate` table
+/// so he can check Alice's key against the fingerprint
+/// and send `AUTH` token later.
 ///
 /// This function takes care of handling multiple concurrent joins and handling errors while
 /// starting the protocol.
 ///
+/// # Bob - the joiner's side
+/// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
+///
 /// # Returns
 ///
 /// The [`ChatId`] of the created chat is returned, for a SetupContact QR this is the 1:1
@@ -42,22 +51,47 @@ pub(super) async fn start_protocol(context: &Context, invite: QrInvite) -> Resul
         .await
         .with_context(|| format!("can't create chat for contact {}", invite.contact_id()))?;
 
-    // Now start the protocol and initialise the state
-    let (state, stage, aborted_states) =
-        BobState::start_protocol(context, invite.clone(), chat_id).await?;
-    for state in aborted_states {
-        error!(context, "Aborting previously unfinished QR Join process.");
-        state.notify_aborted(context, "New QR code scanned").await?;
-        state.emit_progress(context, JoinerProgress::Error);
-    }
-    if matches!(stage, BobHandshakeStage::RequestWithAuthSent) {
-        state.emit_progress(context, JoinerProgress::RequestWithAuthSent);
+    ContactId::scaleup_origin(context, &[invite.contact_id()], Origin::SecurejoinJoined).await?;
+    context.emit_event(EventType::ContactsChanged(None));
+
+    // Now start the protocol and initialise the state.
+    {
+        let peer_verified =
+            verify_sender_by_fingerprint(context, invite.fingerprint(), invite.contact_id())
+                .await?;
+
+        if peer_verified {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+
+            // Mark 1:1 chat as verified already.
+            chat_id
+                .set_protection(
+                    context,
+                    ProtectionStatus::Protected,
+                    time(),
+                    Some(invite.contact_id()),
+                )
+                .await?;
+
+            context.emit_event(EventType::SecurejoinJoinerProgress {
+                contact_id: invite.contact_id(),
+                progress: JoinerProgress::RequestWithAuthSent.to_usize(),
+            });
+        } else {
+            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
+
+            insert_new_db_entry(context, invite.clone(), chat_id).await?;
+        }
     }
+
     match invite {
         QrInvite::Group { .. } => {
             // For a secure-join we need to create the group and add the contact.  The group will
             // only become usable once the protocol is finished.
-            let group_chat_id = state.joining_chat_id(context).await?;
+            let group_chat_id = joining_chat_id(context, &invite, chat_id).await?;
             if !is_contact_in_chat(context, group_chat_id, invite.contact_id()).await? {
                 chat::add_to_chat_contacts_table(
                     context,
@@ -74,7 +108,6 @@ pub(super) async fn start_protocol(context: &Context, invite: QrInvite) -> Resul
         QrInvite::Contact { .. } => {
             // For setup-contact the BobState already ensured the 1:1 chat exists because it
             // uses it to send the handshake messages.
-            let chat_id = state.alice_chat();
             // Calculate the sort timestamp before checking the chat protection status so that if we
             // race with its change, we don't add our message below the protection message.
             let sort_to_bottom = true;
@@ -102,6 +135,19 @@ pub(super) async fn start_protocol(context: &Context, invite: QrInvite) -> Resul
     }
 }
 
+/// Inserts a new entry in the bobstate table.
+///
+/// Returns the ID of the newly inserted entry.
+async fn insert_new_db_entry(context: &Context, invite: QrInvite, chat_id: ChatId) -> Result<i64> {
+    context
+        .sql
+        .insert(
+            "INSERT INTO bobstate (invite, next_step, chat_id) VALUES (?, ?, ?);",
+            (invite, 0, chat_id),
+        )
+        .await
+}
+
 /// Handles `vc-auth-required` and `vg-auth-required` handshake messages.
 ///
 /// # Bob - the joiner's side
@@ -110,121 +156,214 @@ pub(super) async fn handle_auth_required(
     context: &Context,
     message: &MimeMessage,
 ) -> Result<HandshakeMessage> {
-    let Some(mut bobstate) = BobState::from_db(&context.sql).await? else {
-        return Ok(HandshakeMessage::Ignore);
-    };
+    // Load all Bob states that expect `vc-auth-required` or `vg-auth-required`.
+    let bob_states: Vec<(i64, QrInvite, ChatId)> = context
+        .sql
+        .query_map(
+            "SELECT id, invite, chat_id FROM bobstate",
+            (),
+            |row| {
+                let row_id: i64 = row.get(0)?;
+                let invite: QrInvite = row.get(1)?;
+                let chat_id: ChatId = row.get(2)?;
+                Ok((row_id, invite, chat_id))
+            },
+            |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
+        .await?;
 
-    match bobstate.handle_auth_required(context, message).await? {
-        Some(BobHandshakeStage::Terminated(why)) => {
-            bobstate.notify_aborted(context, why).await?;
-            Ok(HandshakeMessage::Done)
+    info!(
+        context,
+        "Bob Step 4 - handling {{vc,vg}}-auth-required message."
+    );
+
+    let mut auth_sent = false;
+    for (bobstate_row_id, invite, chat_id) in bob_states {
+        if !encrypted_and_signed(context, message, invite.fingerprint()) {
+            continue;
         }
-        Some(_stage) => {
-            if bobstate.is_join_group() {
+
+        if !verify_sender_by_fingerprint(context, invite.fingerprint(), invite.contact_id()).await?
+        {
+            continue;
+        }
+
+        info!(context, "Fingerprint verified.",);
+        send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth).await?;
+        context
+            .sql
+            .execute("DELETE FROM bobstate WHERE id=?", (bobstate_row_id,))
+            .await?;
+
+        match invite {
+            QrInvite::Contact { .. } => {}
+            QrInvite::Group { .. } => {
                 // The message reads "Alice replied, waiting to be added to the group…",
                 // so only show it on secure-join and not on setup-contact.
-                let contact_id = bobstate.invite().contact_id();
+                let contact_id = invite.contact_id();
                 let msg = stock_str::secure_join_replies(context, contact_id).await;
-                let chat_id = bobstate.joining_chat_id(context).await?;
+                let chat_id = joining_chat_id(context, &invite, chat_id).await?;
                 chat::add_info_msg(context, chat_id, &msg, time()).await?;
             }
-            bobstate
-                .set_peer_verified(context, message.timestamp_sent)
-                .await?;
-            bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
-            Ok(HandshakeMessage::Done)
         }
-        None => Ok(HandshakeMessage::Ignore),
-    }
-}
 
-/// Private implementations for user interactions about this [`BobState`].
-impl BobState {
-    fn is_join_group(&self) -> bool {
-        match self.invite() {
-            QrInvite::Contact { .. } => false,
-            QrInvite::Group { .. } => true,
-        }
-    }
-
-    pub(crate) fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
-        let contact_id = self.invite().contact_id();
-        context.emit_event(EventType::SecurejoinJoinerProgress {
-            contact_id,
-            progress: progress.into(),
-        });
-    }
-
-    /// Returns the [`ChatId`] of the chat being joined.
-    ///
-    /// This is the chat in which you want to notify the user as well.
-    ///
-    /// When joining a group this is the [`ChatId`] of the group chat, when verifying a
-    /// contact this is the [`ChatId`] of the 1:1 chat.  The 1:1 chat is assumed to exist
-    /// because a [`BobState`] can not exist without, the group chat will be created if it
-    /// does not yet exist.
-    async fn joining_chat_id(&self, context: &Context) -> Result<ChatId> {
-        match self.invite() {
-            QrInvite::Contact { .. } => Ok(self.alice_chat()),
-            QrInvite::Group {
-                ref grpid,
-                ref name,
-                ..
-            } => {
-                let group_chat_id = match chat::get_chat_id_by_grpid(context, grpid).await? {
-                    Some((chat_id, _protected, _blocked)) => {
-                        chat_id.unblock_ex(context, Nosync).await?;
-                        chat_id
-                    }
-                    None => {
-                        ChatId::create_multiuser_record(
-                            context,
-                            Chattype::Group,
-                            grpid,
-                            name,
-                            Blocked::Not,
-                            ProtectionStatus::Unprotected, // protection is added later as needed
-                            None,
-                            create_smeared_timestamp(context),
-                        )
-                        .await?
-                    }
-                };
-                Ok(group_chat_id)
-            }
-        }
-    }
-
-    /// Notifies the user that the SecureJoin was aborted.
-    ///
-    /// This creates an info message in the chat being joined.
-    async fn notify_aborted(&self, context: &Context, why: &str) -> Result<()> {
-        let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        let mut msg = stock_str::contact_not_verified(context, &contact).await;
-        msg += " (";
-        msg += why;
-        msg += ")";
-        let chat_id = self.joining_chat_id(context).await?;
-        chat::add_info_msg(context, chat_id, &msg, time()).await?;
-        warn!(
-            context,
-            "StockMessage::ContactNotVerified posted to joining chat ({})", why
-        );
-        Ok(())
-    }
-
-    /// Turns 1:1 chat with SecureJoin peer into protected chat.
-    pub(crate) async fn set_peer_verified(&self, context: &Context, timestamp: i64) -> Result<()> {
-        let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        self.alice_chat()
+        chat_id
             .set_protection(
                 context,
                 ProtectionStatus::Protected,
-                timestamp,
-                Some(contact.id),
+                message.timestamp_sent,
+                Some(invite.contact_id()),
             )
             .await?;
-        Ok(())
+
+        context.emit_event(EventType::SecurejoinJoinerProgress {
+            contact_id: invite.contact_id(),
+            progress: JoinerProgress::RequestWithAuthSent.to_usize(),
+        });
+
+        auth_sent = true;
+    }
+
+    if auth_sent {
+        // Delete the message from IMAP server.
+        Ok(HandshakeMessage::Done)
+    } else {
+        // We have not found any corresponding AUTH codes,
+        // maybe another Bob device has scanned the QR code.
+        // Leave the message on IMAP server and let the other device
+        // process it.
+        Ok(HandshakeMessage::Ignore)
+    }
+}
+
+/// Sends the requested handshake message to Alice.
+pub(crate) async fn send_handshake_message(
+    context: &Context,
+    invite: &QrInvite,
+    chat_id: ChatId,
+    step: BobHandshakeMsg,
+) -> Result<()> {
+    let mut msg = Message {
+        viewtype: Viewtype::Text,
+        text: step.body_text(invite),
+        hidden: true,
+        ..Default::default()
+    };
+    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
+
+    // Sends the step in Secure-Join header.
+    msg.param.set(Param::Arg, step.securejoin_header(invite));
+
+    match step {
+        BobHandshakeMsg::Request => {
+            // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
+            msg.param.set(Param::Arg2, invite.invitenumber());
+            msg.force_plaintext();
+        }
+        BobHandshakeMsg::RequestWithAuth => {
+            // Sends the Secure-Join-Auth header in mimefactory.rs.
+            msg.param.set(Param::Arg2, invite.authcode());
+            msg.param.set_int(Param::GuaranteeE2ee, 1);
+
+            // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+            let bob_fp = load_self_public_key(context).await?.dc_fingerprint();
+            msg.param.set(Param::Arg3, bob_fp.hex());
+
+            // Sends the grpid in the Secure-Join-Group header.
+            //
+            // `Secure-Join-Group` header is deprecated,
+            // but old Delta Chat core requires that Alice receives it.
+            //
+            // Previous Delta Chat core also sent `Secure-Join-Group` header
+            // in `vg-request` messages,
+            // but it was not used on the receiver.
+            if let QrInvite::Group { ref grpid, .. } = invite {
+                msg.param.set(Param::Arg4, grpid);
+            }
+        }
+    };
+
+    chat::send_msg(context, chat_id, &mut msg).await?;
+    Ok(())
+}
+
+/// Identifies the SecureJoin handshake messages Bob can send.
+pub(crate) enum BobHandshakeMsg {
+    /// vc-request or vg-request
+    Request,
+    /// vc-request-with-auth or vg-request-with-auth
+    RequestWithAuth,
+}
+
+impl BobHandshakeMsg {
+    /// Returns the text to send in the body of the handshake message.
+    ///
+    /// This text has no significance to the protocol, but would be visible if users see
+    /// this email message directly, e.g. when accessing their email without using
+    /// DeltaChat.
+    fn body_text(&self, invite: &QrInvite) -> String {
+        format!("Secure-Join: {}", self.securejoin_header(invite))
+    }
+
+    /// Returns the `Secure-Join` header value.
+    ///
+    /// This identifies the step this message is sending information about.  Most protocol
+    /// steps include additional information into other headers, see
+    /// [`send_handshake_message`] for these.
+    fn securejoin_header(&self, invite: &QrInvite) -> &'static str {
+        match self {
+            Self::Request => match invite {
+                QrInvite::Contact { .. } => "vc-request",
+                QrInvite::Group { .. } => "vg-request",
+            },
+            Self::RequestWithAuth => match invite {
+                QrInvite::Contact { .. } => "vc-request-with-auth",
+                QrInvite::Group { .. } => "vg-request-with-auth",
+            },
+        }
+    }
+}
+
+/// Returns the [`ChatId`] of the chat being joined.
+///
+/// This is the chat in which you want to notify the user as well.
+///
+/// When joining a group this is the [`ChatId`] of the group chat, when verifying a
+/// contact this is the [`ChatId`] of the 1:1 chat.
+/// The group chat will be created if it does not yet exist.
+async fn joining_chat_id(
+    context: &Context,
+    invite: &QrInvite,
+    alice_chat_id: ChatId,
+) -> Result<ChatId> {
+    match invite {
+        QrInvite::Contact { .. } => Ok(alice_chat_id),
+        QrInvite::Group {
+            ref grpid,
+            ref name,
+            ..
+        } => {
+            let group_chat_id = match chat::get_chat_id_by_grpid(context, grpid).await? {
+                Some((chat_id, _protected, _blocked)) => {
+                    chat_id.unblock_ex(context, Nosync).await?;
+                    chat_id
+                }
+                None => {
+                    ChatId::create_multiuser_record(
+                        context,
+                        Chattype::Group,
+                        grpid,
+                        name,
+                        Blocked::Not,
+                        ProtectionStatus::Unprotected, // protection is added later as needed
+                        None,
+                        create_smeared_timestamp(context),
+                    )
+                    .await?
+                }
+            };
+            Ok(group_chat_id)
+        }
     }
 }
 
@@ -233,8 +372,6 @@ impl BobState {
 /// This has an `From<JoinerProgress> for usize` impl yielding numbers between 0 and a 1000
 /// which can be shown as a progress bar.
 pub(crate) enum JoinerProgress {
-    /// An error occurred.
-    Error,
     /// vg-vc-request-with-auth sent.
     ///
     /// Typically shows as "alice@addr verified, introducing myself."
@@ -243,10 +380,10 @@ pub(crate) enum JoinerProgress {
     Succeeded,
 }
 
-impl From<JoinerProgress> for usize {
-    fn from(progress: JoinerProgress) -> Self {
-        match progress {
-            JoinerProgress::Error => 0,
+impl JoinerProgress {
+    #[expect(clippy::wrong_self_convention)]
+    pub(crate) fn to_usize(self) -> usize {
+        match self {
             JoinerProgress::RequestWithAuthSent => 400,
             JoinerProgress::Succeeded => 1000,
         }

src/securejoin/bobstate.rs

@@ -1,517 +0,0 @@
-//! Secure-Join protocol state machine for Bob, the joiner-side.
-//!
-//! This module contains the state machine to run the Secure-Join handshake for Bob and does
-//! not do any user interaction required by the protocol.  Instead the state machine
-//! provides all the information to its driver so it can perform the correct interactions.
-//!
-//! The [`BobState`] is only directly used to initially create it when starting the
-//! protocol.
-
-use anyhow::Result;
-use rusqlite::Connection;
-
-use super::qrinvite::QrInvite;
-use super::{encrypted_and_signed, verify_sender_by_fingerprint};
-use crate::chat::{self, ChatId};
-use crate::config::Config;
-use crate::contact::{ContactId, Origin};
-use crate::context::Context;
-use crate::events::EventType;
-use crate::headerdef::HeaderDef;
-use crate::key::{load_self_public_key, DcKey};
-use crate::message::{Message, Viewtype};
-use crate::mimeparser::{MimeMessage, SystemMessage};
-use crate::param::Param;
-use crate::securejoin::Peerstate;
-use crate::sql::Sql;
-use crate::tools::time;
-
-/// The stage of the [`BobState`] securejoin handshake protocol state machine.
-///
-/// This does not concern itself with user interactions, only represents what happened to
-/// the protocol state machine from handling this message.
-#[derive(Clone, Copy, Debug, Display)]
-pub enum BobHandshakeStage {
-    /// Step 2 completed: (vc|vg)-request message sent.
-    RequestSent,
-    /// Step 4 completed: (vc|vg)-request-with-auth message sent.
-    RequestWithAuthSent,
-    /// The protocol prematurely terminated with given reason.
-    Terminated(&'static str),
-}
-
-/// The securejoin state kept while Bob is joining.
-///
-/// This is stored in the database and loaded from there using [`BobState::from_db`].  To
-/// create a new one use [`BobState::start_protocol`].
-///
-/// This purposefully has nothing optional, the state is always fully valid.  However once a
-/// terminal state is reached in [`BobState::next`] the entry in the database will already
-/// have been deleted.
-///
-/// # Conducting the securejoin handshake
-///
-/// The methods on this struct allow you to interact with the state and thus conduct the
-/// securejoin handshake for Bob.  The methods only concern themselves with the protocol
-/// state and explicitly avoid performing any user interactions required by securejoin.
-/// This simplifies the concerns and logic required in both the callers and in the state
-/// management.  The return values can be used to understand what user interactions need to
-/// happen.
-///
-/// [`Bob`]: super::Bob
-/// [`Bob::state`]: super::Bob::state
-#[derive(Debug, Clone)]
-pub struct BobState {
-    /// Database primary key.
-    id: i64,
-    /// The QR Invite code.
-    invite: QrInvite,
-    /// The next expected message from Alice.
-    next: SecureJoinStep,
-    /// The [`ChatId`] of the 1:1 chat with Alice, matching [`QrInvite::contact_id`].
-    chat_id: ChatId,
-}
-
-impl BobState {
-    /// Starts the securejoin protocol and creates a new [`BobState`].
-    ///
-    /// The `chat_id` needs to be the ID of the 1:1 chat with Alice, this chat will be used
-    /// to exchange the SecureJoin handshake messages as well as for showing error messages.
-    ///
-    /// # Bob - the joiner's side
-    /// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    ///
-    /// This currently aborts any other securejoin process if any did not yet complete.  The
-    /// ChatIds of the relevant 1:1 chat of any aborted handshakes are returned so that you
-    /// can report the aboreted handshake in the chat.  (Yes, there can only ever be one
-    /// ChatId in that Vec, the database doesn't care though.)
-    pub async fn start_protocol(
-        context: &Context,
-        invite: QrInvite,
-        chat_id: ChatId,
-    ) -> Result<(Self, BobHandshakeStage, Vec<Self>)> {
-        let peer_verified =
-            verify_sender_by_fingerprint(context, invite.fingerprint(), invite.contact_id())
-                .await?;
-
-        let (stage, next);
-        if peer_verified {
-            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
-            info!(context, "Taking securejoin protocol shortcut");
-            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
-                .await?;
-
-            stage = BobHandshakeStage::RequestWithAuthSent;
-            next = SecureJoinStep::ContactConfirm;
-        } else {
-            send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
-
-            stage = BobHandshakeStage::RequestSent;
-            next = SecureJoinStep::AuthRequired;
-        };
-
-        let (id, aborted_states) =
-            Self::insert_new_db_entry(context, next, invite.clone(), chat_id).await?;
-        let state = Self {
-            id,
-            invite,
-            next,
-            chat_id,
-        };
-
-        if peer_verified {
-            // Mark 1:1 chat as verified already.
-            state.set_peer_verified(context, time()).await?;
-        }
-
-        Ok((state, stage, aborted_states))
-    }
-
-    /// Inserts a new entry in the bobstate table, deleting all previous entries.
-    ///
-    /// Returns the ID of the newly inserted entry and all the aborted states.
-    async fn insert_new_db_entry(
-        context: &Context,
-        next: SecureJoinStep,
-        invite: QrInvite,
-        chat_id: ChatId,
-    ) -> Result<(i64, Vec<Self>)> {
-        context
-            .sql
-            .transaction(move |transaction| {
-                // We need to start a write transaction right away, so that we have the
-                // database locked and no one else can write to this table while we read the
-                // rows that we will delete.  So start with a dummy UPDATE.
-                transaction.execute(
-                    r#"UPDATE bobstate SET next_step=?;"#,
-                    (SecureJoinStep::Terminated,),
-                )?;
-                let mut stmt = transaction.prepare("SELECT id FROM bobstate;")?;
-                let mut aborted = Vec::new();
-                for id in stmt.query_map((), |row| row.get::<_, i64>(0))? {
-                    let id = id?;
-                    let state = BobState::from_db_id(transaction, id)?;
-                    aborted.push(state);
-                }
-
-                // Finally delete everything and insert new row.
-                transaction.execute("DELETE FROM bobstate;", ())?;
-                transaction.execute(
-                    "INSERT INTO bobstate (invite, next_step, chat_id) VALUES (?, ?, ?);",
-                    (invite, next, chat_id),
-                )?;
-                let id = transaction.last_insert_rowid();
-                Ok((id, aborted))
-            })
-            .await
-    }
-
-    /// Load [`BobState`] from the database.
-    pub async fn from_db(sql: &Sql) -> Result<Option<Self>> {
-        // Because of how Self::start_protocol() updates the database we are currently
-        // guaranteed to only have one row.
-        sql.query_row_optional(
-            "SELECT id, invite, next_step, chat_id FROM bobstate;",
-            (),
-            |row| {
-                let s = BobState {
-                    id: row.get(0)?,
-                    invite: row.get(1)?,
-                    next: row.get(2)?,
-                    chat_id: row.get(3)?,
-                };
-                Ok(s)
-            },
-        )
-        .await
-    }
-
-    fn from_db_id(connection: &Connection, id: i64) -> rusqlite::Result<Self> {
-        connection.query_row(
-            "SELECT invite, next_step, chat_id FROM bobstate WHERE id=?;",
-            (id,),
-            |row| {
-                let s = BobState {
-                    id,
-                    invite: row.get(0)?,
-                    next: row.get(1)?,
-                    chat_id: row.get(2)?,
-                };
-                Ok(s)
-            },
-        )
-    }
-
-    /// Returns the [`QrInvite`] used to create this [`BobState`].
-    pub fn invite(&self) -> &QrInvite {
-        &self.invite
-    }
-
-    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
-    pub fn alice_chat(&self) -> ChatId {
-        self.chat_id
-    }
-
-    /// Updates the [`BobState::next`] field in memory and the database.
-    ///
-    /// If the next state is a terminal state it will remove this [`BobState`] from the
-    /// database.
-    ///
-    /// If a user scanned a new QR code after this [`BobState`] was loaded this update will
-    /// fail currently because starting a new joiner process currently kills any previously
-    /// running processes.  This is a limitation which will go away in the future.
-    async fn update_next(&mut self, sql: &Sql, next: SecureJoinStep) -> Result<()> {
-        // TODO: write test verifying how this would fail.
-        match next {
-            SecureJoinStep::AuthRequired | SecureJoinStep::ContactConfirm => {
-                sql.execute(
-                    "UPDATE bobstate SET next_step=? WHERE id=?;",
-                    (next, self.id),
-                )
-                .await?;
-            }
-            SecureJoinStep::Terminated | SecureJoinStep::Completed => {
-                sql.execute("DELETE FROM bobstate WHERE id=?;", (self.id,))
-                    .await?;
-            }
-        }
-        self.next = next;
-        Ok(())
-    }
-
-    /// Handles {vc,vg}-auth-required message of the securejoin handshake for Bob.
-    ///
-    /// If the message was not used for this handshake `None` is returned, otherwise the new
-    /// stage is returned.  Once [`BobHandshakeStage::Terminated`] is reached this
-    /// [`BobState`] should be destroyed,
-    /// further calling it will just result in the messages being unused by this handshake.
-    pub(crate) async fn handle_auth_required(
-        &mut self,
-        context: &Context,
-        mime_message: &MimeMessage,
-    ) -> Result<Option<BobHandshakeStage>> {
-        let step = match mime_message.get_header(HeaderDef::SecureJoin) {
-            Some(step) => step,
-            None => {
-                warn!(
-                    context,
-                    "Message has no Secure-Join header: {}",
-                    mime_message.get_rfc724_mid().unwrap_or_default()
-                );
-                return Ok(None);
-            }
-        };
-        if !self.is_msg_expected(context, step) {
-            info!(context, "{} message out of sync for BobState", step);
-            return Ok(None);
-        }
-
-        info!(
-            context,
-            "Bob Step 4 - handling {{vc,vg}}-auth-required message."
-        );
-        if !encrypted_and_signed(context, mime_message, self.invite.fingerprint()) {
-            let reason = if mime_message.was_encrypted() {
-                "Valid signature missing"
-            } else {
-                "Required encryption missing"
-            };
-            self.update_next(&context.sql, SecureJoinStep::Terminated)
-                .await?;
-            return Ok(Some(BobHandshakeStage::Terminated(reason)));
-        }
-        if !verify_sender_by_fingerprint(
-            context,
-            self.invite.fingerprint(),
-            self.invite.contact_id(),
-        )
-        .await?
-        {
-            self.update_next(&context.sql, SecureJoinStep::Terminated)
-                .await?;
-            return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
-        }
-        info!(context, "Fingerprint verified.",);
-
-        self.update_next(&context.sql, SecureJoinStep::ContactConfirm)
-            .await?;
-        self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
-            .await?;
-        Ok(Some(BobHandshakeStage::RequestWithAuthSent))
-    }
-
-    /// Returns `true` if the message is expected according to the protocol.
-    pub(crate) fn is_msg_expected(&self, context: &Context, step: &str) -> bool {
-        let variant_matches = match self.invite {
-            QrInvite::Contact { .. } => step.starts_with("vc-"),
-            QrInvite::Group { .. } => step.starts_with("vg-"),
-        };
-        let step_matches = self.next.matches(context, step);
-        variant_matches && step_matches
-    }
-
-    /// Handles a *vc-contact-confirm* or *vg-member-added* message.
-    ///
-    /// # Bob - the joiner's side
-    /// ## Step 7 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    pub(crate) async fn step_contact_confirm(&mut self, context: &Context) -> Result<()> {
-        let fingerprint = self.invite.fingerprint();
-        let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, fingerprint).await?
-        else {
-            return Ok(());
-        };
-
-        // Mark peer as backward verified.
-        peerstate.backward_verified_key_id =
-            Some(context.get_config_i64(Config::KeyId).await?).filter(|&id| id > 0);
-        peerstate.save_to_db(&context.sql).await?;
-
-        ContactId::scaleup_origin(
-            context,
-            &[self.invite.contact_id()],
-            Origin::SecurejoinJoined,
-        )
-        .await?;
-        context.emit_event(EventType::ContactsChanged(None));
-
-        self.update_next(&context.sql, SecureJoinStep::Completed)
-            .await?;
-        Ok(())
-    }
-
-    /// Sends the requested handshake message to Alice.
-    ///
-    /// This takes care of adding the required headers for the step.
-    async fn send_handshake_message(&self, context: &Context, step: BobHandshakeMsg) -> Result<()> {
-        send_handshake_message(context, &self.invite, self.chat_id, step).await
-    }
-
-    /// Returns whether we are waiting for a SecureJoin message from Alice, i.e. the protocol hasn't
-    /// yet completed.
-    pub(crate) fn in_progress(&self) -> bool {
-        !matches!(
-            self.next,
-            SecureJoinStep::Terminated | SecureJoinStep::Completed
-        )
-    }
-}
-
-/// Sends the requested handshake message to Alice.
-///
-/// Same as [`BobState::send_handshake_message`] but this variation allows us to send this
-/// message before we create the state in [`BobState::start_protocol`].
-async fn send_handshake_message(
-    context: &Context,
-    invite: &QrInvite,
-    chat_id: ChatId,
-    step: BobHandshakeMsg,
-) -> Result<()> {
-    let mut msg = Message {
-        viewtype: Viewtype::Text,
-        text: step.body_text(invite),
-        hidden: true,
-        ..Default::default()
-    };
-    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
-
-    // Sends the step in Secure-Join header.
-    msg.param.set(Param::Arg, step.securejoin_header(invite));
-
-    match step {
-        BobHandshakeMsg::Request => {
-            // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
-            msg.param.set(Param::Arg2, invite.invitenumber());
-            msg.force_plaintext();
-        }
-        BobHandshakeMsg::RequestWithAuth => {
-            // Sends the Secure-Join-Auth header in mimefactory.rs.
-            msg.param.set(Param::Arg2, invite.authcode());
-            msg.param.set_int(Param::GuaranteeE2ee, 1);
-
-            // Sends our own fingerprint in the Secure-Join-Fingerprint header.
-            let bob_fp = load_self_public_key(context).await?.dc_fingerprint();
-            msg.param.set(Param::Arg3, bob_fp.hex());
-
-            // Sends the grpid in the Secure-Join-Group header.
-            //
-            // `Secure-Join-Group` header is deprecated,
-            // but old Delta Chat core requires that Alice receives it.
-            //
-            // Previous Delta Chat core also sent `Secure-Join-Group` header
-            // in `vg-request` messages,
-            // but it was not used on the receiver.
-            if let QrInvite::Group { ref grpid, .. } = invite {
-                msg.param.set(Param::Arg4, grpid);
-            }
-        }
-    };
-
-    chat::send_msg(context, chat_id, &mut msg).await?;
-    Ok(())
-}
-
-/// Identifies the SecureJoin handshake messages Bob can send.
-enum BobHandshakeMsg {
-    /// vc-request or vg-request
-    Request,
-    /// vc-request-with-auth or vg-request-with-auth
-    RequestWithAuth,
-}
-
-impl BobHandshakeMsg {
-    /// Returns the text to send in the body of the handshake message.
-    ///
-    /// This text has no significance to the protocol, but would be visible if users see
-    /// this email message directly, e.g. when accessing their email without using
-    /// DeltaChat.
-    fn body_text(&self, invite: &QrInvite) -> String {
-        format!("Secure-Join: {}", self.securejoin_header(invite))
-    }
-
-    /// Returns the `Secure-Join` header value.
-    ///
-    /// This identifies the step this message is sending information about.  Most protocol
-    /// steps include additional information into other headers, see
-    /// [`BobState::send_handshake_message`] for these.
-    fn securejoin_header(&self, invite: &QrInvite) -> &'static str {
-        match self {
-            Self::Request => match invite {
-                QrInvite::Contact { .. } => "vc-request",
-                QrInvite::Group { .. } => "vg-request",
-            },
-            Self::RequestWithAuth => match invite {
-                QrInvite::Contact { .. } => "vc-request-with-auth",
-                QrInvite::Group { .. } => "vg-request-with-auth",
-            },
-        }
-    }
-}
-
-/// The next message expected by [`BobState`] in the setup-contact/secure-join protocol.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SecureJoinStep {
-    /// Expecting the auth-required message.
-    ///
-    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
-    AuthRequired,
-    /// Expecting the contact-confirm message.
-    ///
-    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
-    /// 6b.
-    ContactConfirm,
-    /// The protocol terminated because of an error.
-    ///
-    /// The securejoin protocol terminated, this exists to ensure [`BobState`] can detect
-    /// when it earlier signalled that is should be terminated.  It is an error to call with
-    /// this state.
-    Terminated,
-    /// The protocol completed.
-    ///
-    /// This exists to ensure [`BobState`] can detect when it earlier signalled that it is
-    /// complete.  It is an error to call with this state.
-    Completed,
-}
-
-impl SecureJoinStep {
-    /// Compares the legacy string representation of a step to a [`SecureJoinStep`] variant.
-    fn matches(&self, context: &Context, step: &str) -> bool {
-        match self {
-            Self::AuthRequired => step == "vc-auth-required" || step == "vg-auth-required",
-            Self::ContactConfirm => step == "vc-contact-confirm" || step == "vg-member-added",
-            SecureJoinStep::Terminated => {
-                warn!(context, "Terminated state for next securejoin step");
-                false
-            }
-            SecureJoinStep::Completed => {
-                warn!(context, "Completed state for next securejoin step");
-                false
-            }
-        }
-    }
-}
-
-impl rusqlite::types::ToSql for SecureJoinStep {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
-        let num = match &self {
-            SecureJoinStep::AuthRequired => 0,
-            SecureJoinStep::ContactConfirm => 1,
-            SecureJoinStep::Terminated => 2,
-            SecureJoinStep::Completed => 3,
-        };
-        let val = rusqlite::types::Value::Integer(num);
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
-impl rusqlite::types::FromSql for SecureJoinStep {
-    fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| match val {
-            0 => Ok(SecureJoinStep::AuthRequired),
-            1 => Ok(SecureJoinStep::ContactConfirm),
-            2 => Ok(SecureJoinStep::Terminated),
-            3 => Ok(SecureJoinStep::Completed),
-            _ => Err(rusqlite::types::FromSqlError::OutOfRange(val)),
-        })
-    }
-}

src/securejoin/securejoin_tests.rs

@@ -209,7 +209,7 @@ async fn test_setup_contact_ex(case: SetupContactCase) {
             .insert(alice_addr.to_string(), wrong_pubkey)
             .unwrap();
         let contact_bob = alice.add_or_lookup_contact(&bob).await;
-        let handshake_msg = handle_securejoin_handshake(&alice, &msg, contact_bob.id)
+        let handshake_msg = handle_securejoin_handshake(&alice, &mut msg, contact_bob.id)
             .await
             .unwrap();
         assert_eq!(handshake_msg, HandshakeMessage::Ignore);
@@ -218,7 +218,7 @@ async fn test_setup_contact_ex(case: SetupContactCase) {
         msg.gossiped_keys
             .insert(alice_addr.to_string(), alice_pubkey)
             .unwrap();
-        let handshake_msg = handle_securejoin_handshake(&alice, &msg, contact_bob.id)
+        let handshake_msg = handle_securejoin_handshake(&alice, &mut msg, contact_bob.id)
             .await
             .unwrap();
         assert_eq!(handshake_msg, HandshakeMessage::Ignore);
@@ -839,3 +839,110 @@ async fn test_shared_bobs_key() -> Result<()> {
     assert_eq!(bob_ids.len(), 3);
     Ok(())
 }
+
+/// Tests Bob joining two groups by scanning two QR codes
+/// from the same Alice at the same time.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_parallel_securejoin() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let alice_chat1_id =
+        chat::create_group_chat(alice, ProtectionStatus::Protected, "First chat").await?;
+    let alice_chat2_id =
+        chat::create_group_chat(alice, ProtectionStatus::Protected, "Second chat").await?;
+
+    let qr1 = get_securejoin_qr(alice, Some(alice_chat1_id)).await?;
+    let qr2 = get_securejoin_qr(alice, Some(alice_chat2_id)).await?;
+
+    // Bob scans both QR codes.
+    let bob_chat1_id = join_securejoin(bob, &qr1).await?;
+    let sent_vg_request1 = bob.pop_sent_msg().await;
+
+    let bob_chat2_id = join_securejoin(bob, &qr2).await?;
+    let sent_vg_request2 = bob.pop_sent_msg().await;
+
+    // Alice receives two `vg-request` messages
+    // and sends back two `vg-auth-required` messages.
+    alice.recv_msg_trash(&sent_vg_request1).await;
+    let sent_vg_auth_required1 = alice.pop_sent_msg().await;
+
+    alice.recv_msg_trash(&sent_vg_request2).await;
+    let _sent_vg_auth_required2 = alice.pop_sent_msg().await;
+
+    // Bob receives first `vg-auth-required` message.
+    // Bob has two securejoin processes started,
+    // so he should send two `vg-request-with-auth` messages.
+    bob.recv_msg_trash(&sent_vg_auth_required1).await;
+
+    // Bob sends `vg-request-with-auth` messages.
+    let sent_vg_request_with_auth2 = bob.pop_sent_msg().await;
+    let sent_vg_request_with_auth1 = bob.pop_sent_msg().await;
+
+    // Alice receives both `vg-request-with-auth` messages.
+    alice.recv_msg_trash(&sent_vg_request_with_auth1).await;
+    let sent_vg_member_added1 = alice.pop_sent_msg().await;
+    let joined_chat_id1 = bob.recv_msg(&sent_vg_member_added1).await.chat_id;
+    assert_eq!(joined_chat_id1, bob_chat1_id);
+
+    alice.recv_msg_trash(&sent_vg_request_with_auth2).await;
+    let sent_vg_member_added2 = alice.pop_sent_msg().await;
+    let joined_chat_id2 = bob.recv_msg(&sent_vg_member_added2).await.chat_id;
+    assert_eq!(joined_chat_id2, bob_chat2_id);
+
+    Ok(())
+}
+
+/// Tests Bob scanning setup contact QR codes of Alice and Fiona
+/// concurrently.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_parallel_setup_contact() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let fiona = &tcm.fiona().await;
+
+    // Bob scans Alice's QR code,
+    // but Alice is offline and takes a while to respond.
+    let alice_qr = get_securejoin_qr(alice, None).await?;
+    join_securejoin(bob, &alice_qr).await?;
+    let sent_alice_vc_request = bob.pop_sent_msg().await;
+
+    // Bob scans Fiona's QR code while SecureJoin
+    // process with Alice is not finished.
+    let fiona_qr = get_securejoin_qr(fiona, None).await?;
+    join_securejoin(bob, &fiona_qr).await?;
+    let sent_fiona_vc_request = bob.pop_sent_msg().await;
+
+    fiona.recv_msg_trash(&sent_fiona_vc_request).await;
+    let sent_fiona_vc_auth_required = fiona.pop_sent_msg().await;
+
+    bob.recv_msg_trash(&sent_fiona_vc_auth_required).await;
+    let sent_fiona_vc_request_with_auth = bob.pop_sent_msg().await;
+
+    fiona.recv_msg_trash(&sent_fiona_vc_request_with_auth).await;
+    let sent_fiona_vc_contact_confirm = fiona.pop_sent_msg().await;
+
+    bob.recv_msg_trash(&sent_fiona_vc_contact_confirm).await;
+    let bob_fiona_contact_id = bob.add_or_lookup_contact_id(fiona).await;
+    let bob_fiona_contact = Contact::get_by_id(bob, bob_fiona_contact_id).await.unwrap();
+    assert_eq!(bob_fiona_contact.is_verified(bob).await.unwrap(), true);
+
+    // Alice gets online and previously started SecureJoin process finishes.
+    alice.recv_msg_trash(&sent_alice_vc_request).await;
+    let sent_alice_vc_auth_required = alice.pop_sent_msg().await;
+
+    bob.recv_msg_trash(&sent_alice_vc_auth_required).await;
+    let sent_alice_vc_request_with_auth = bob.pop_sent_msg().await;
+
+    alice.recv_msg_trash(&sent_alice_vc_request_with_auth).await;
+    let sent_alice_vc_contact_confirm = alice.pop_sent_msg().await;
+
+    bob.recv_msg_trash(&sent_alice_vc_contact_confirm).await;
+    let bob_alice_contact_id = bob.add_or_lookup_contact_id(alice).await;
+    let bob_alice_contact = Contact::get_by_id(bob, bob_alice_contact_id).await.unwrap();
+    assert_eq!(bob_alice_contact.is_verified(bob).await.unwrap(), true);
+
+    Ok(())
+}

src/sql.rs

@@ -251,7 +251,7 @@ impl Sql {
 
         if recode_avatar {
             if let Some(avatar) = context.get_config(Config::Selfavatar).await? {
-                let mut blob = BlobObject::new_from_path(context, avatar.as_ref()).await?;
+                let mut blob = BlobObject::from_path(context, Path::new(&avatar))?;
                 match blob.recode_to_avatar_size(context).await {
                     Ok(()) => {
                         if let Some(path) = blob.to_abs_path().to_str() {
@@ -458,7 +458,9 @@ impl Sql {
     ///   in parallel with other transactions. NB: Creating and modifying temporary tables are also
     ///   allowed with `query_only`, temporary tables aren't visible in other connections, but you
     ///   need to pass `PRAGMA query_only=0;` to SQLite before that:
-    ///     `pragma_update(None, "query_only", "0")`.
+    ///   ```text
+    ///   pragma_update(None, "query_only", "0")
+    ///   ```
     ///   Also temporary tables need to be dropped because the connection is returned to the pool
     ///   then.
     ///
@@ -912,42 +914,51 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> {
                         continue;
                     }
 
-                    if let Ok(stats) = tokio::fs::metadata(entry.path()).await {
-                        if stats.is_dir() {
-                            if let Err(e) = tokio::fs::remove_dir(entry.path()).await {
-                                // The dir could be created not by a user, but by a desktop
-                                // environment f.e. So, no warning.
-                                info!(
-                                    context,
-                                    "Housekeeping: Cannot rmdir {}: {:#}.",
-                                    entry.path().display(),
-                                    e
-                                );
-                            }
-                            continue;
-                        }
-                        unreferenced_count += 1;
-                        let recently_created =
-                            stats.created().is_ok_and(|t| t > keep_files_newer_than);
-                        let recently_modified =
-                            stats.modified().is_ok_and(|t| t > keep_files_newer_than);
-                        let recently_accessed =
-                            stats.accessed().is_ok_and(|t| t > keep_files_newer_than);
-
-                        if p == blobdir
-                            && (recently_created || recently_modified || recently_accessed)
-                        {
-                            info!(
+                    let stats = match tokio::fs::metadata(entry.path()).await {
+                        Err(err) => {
+                            warn!(
                                 context,
-                                "Housekeeping: Keeping new unreferenced file #{}: {:?}.",
-                                unreferenced_count,
-                                entry.file_name(),
+                                "Cannot get metadata for {}: {:#}.",
+                                entry.path().display(),
+                                err
                             );
                             continue;
                         }
-                    } else {
-                        unreferenced_count += 1;
+                        Ok(stats) => stats,
+                    };
+
+                    if stats.is_dir() {
+                        if let Err(e) = tokio::fs::remove_dir(entry.path()).await {
+                            // The dir could be created not by a user, but by a desktop
+                            // environment f.e. So, no warning.
+                            info!(
+                                context,
+                                "Housekeeping: Cannot rmdir {}: {:#}.",
+                                entry.path().display(),
+                                e
+                            );
+                        }
+                        continue;
                     }
+
+                    unreferenced_count += 1;
+                    let recently_created = stats.created().is_ok_and(|t| t > keep_files_newer_than);
+                    let recently_modified =
+                        stats.modified().is_ok_and(|t| t > keep_files_newer_than);
+                    let recently_accessed =
+                        stats.accessed().is_ok_and(|t| t > keep_files_newer_than);
+
+                    if p == blobdir && (recently_created || recently_modified || recently_accessed)
+                    {
+                        info!(
+                            context,
+                            "Housekeeping: Keeping new unreferenced file #{}: {:?}.",
+                            unreferenced_count,
+                            entry.file_name(),
+                        );
+                        continue;
+                    }
+
                     info!(
                         context,
                         "Housekeeping: Deleting unreferenced file #{}: {:?}.",

src/summary.rs

@@ -469,7 +469,8 @@ mod tests {
 
         let mut msg = Message::new(Viewtype::File);
         msg.set_text(some_text.clone());
-        msg.param.set(Param::File, "foo.bar");
+        msg.set_file_from_bytes(ctx, "foo.bar", b"data", None)
+            .unwrap();
         msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
         assert_summary_texts(&msg, ctx, "Autocrypt Setup Message").await; // file name is not added for autocrypt setup messages
     }

src/sync.rs

@@ -1,7 +1,7 @@
 //! # Synchronize items between devices.
 
 use anyhow::Result;
-use lettre_email::PartBuilder;
+use mail_builder::mime::MimePart;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, ChatId};
@@ -17,6 +17,7 @@ use crate::sync::SyncData::{AddQrToken, AlterChat, DeleteQrToken};
 use crate::token::Namespace;
 use crate::tools::time;
 use crate::{message, stock_str, token};
+use std::collections::HashSet;
 
 /// Whether to send device sync messages. Aimed for usage in the internal API.
 #[derive(Debug, PartialEq)]
@@ -66,6 +67,9 @@ pub(crate) enum SyncData {
         src: String,  // RFC724 id (i.e. "Message-Id" header)
         dest: String, // RFC724 id (i.e. "Message-Id" header)
     },
+    DeleteMessages {
+        msgs: Vec<String>, // RFC724 id (i.e. "Message-Id" header)
+    },
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -165,7 +169,7 @@ impl Context {
     ///
     /// Mustn't be called from multiple tasks in parallel to avoid sending the same sync items twice
     /// because sync items are removed from the db only after successful sending. We guarantee this
-    /// by calling `send_sync_msg()` only from the SMTP loop.
+    /// by calling `send_sync_msg()` only from the inbox loop.
     pub async fn send_sync_msg(&self) -> Result<Option<MsgId>> {
         if let Some((json, ids)) = self.build_sync_json().await? {
             let chat_id =
@@ -227,14 +231,8 @@ impl Context {
         }
     }
 
-    pub(crate) fn build_sync_part(&self, json: String) -> PartBuilder {
-        PartBuilder::new()
-            .content_type(&"application/json".parse::<mime::Mime>().unwrap())
-            .header((
-                "Content-Disposition",
-                "attachment; filename=\"multi-device-sync.json\"",
-            ))
-            .body(json)
+    pub(crate) fn build_sync_part(&self, json: String) -> MimePart<'static> {
+        MimePart::new("application/json", json).attachment("multi-device-sync.json")
     }
 
     /// Takes a JSON string created by `build_sync_json()`
@@ -264,6 +262,7 @@ impl Context {
                     AlterChat { id, action } => self.sync_alter_chat(id, action).await,
                     SyncData::Config { key, val } => self.sync_config(key, val).await,
                     SyncData::SaveMessage { src, dest } => self.save_message(src, dest).await,
+                    SyncData::DeleteMessages { msgs } => self.sync_message_deletion(msgs).await,
                 },
                 SyncDataOrUnknown::Unknown(data) => {
                     warn!(self, "Ignored unknown sync item: {data}.");
@@ -303,6 +302,26 @@ impl Context {
         }
         Ok(())
     }
+
+    async fn sync_message_deletion(&self, msgs: &Vec<String>) -> Result<()> {
+        let mut modified_chat_ids = HashSet::new();
+        let mut msg_ids = Vec::new();
+        for rfc724_mid in msgs {
+            if let Some((msg_id, _)) = message::rfc724_mid_exists(self, rfc724_mid).await? {
+                if let Some(msg) = Message::load_from_db_optional(self, msg_id).await? {
+                    message::delete_msg_locally(self, &msg).await?;
+                    msg_ids.push(msg.id);
+                    modified_chat_ids.insert(msg.chat_id);
+                } else {
+                    warn!(self, "Sync message delete: Database entry does not exist.");
+                }
+            } else {
+                warn!(self, "Sync message delete: {rfc724_mid:?} not found.");
+            }
+        }
+        message::delete_msgs_locally_done(self, &msg_ids, modified_chat_ids).await?;
+        Ok(())
+    }
 }
 
 #[cfg(test)]

src/test_utils.rs

@@ -33,7 +33,7 @@ use crate::contact::{Contact, ContactId, Modifier, Origin};
 use crate::context::Context;
 use crate::e2ee::EncryptHelper;
 use crate::events::{Event, EventEmitter, EventType, Events};
-use crate::key::{self, DcKey, KeyPairUse};
+use crate::key::{self, DcKey};
 use crate::message::{update_msg_state, Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::peerstate::Peerstate;
@@ -271,7 +271,7 @@ impl TestContextBuilder {
 
             let test_context = TestContext::new_internal(Some(name), self.log_sink).await;
             test_context.configure_addr(&addr).await;
-            key::store_self_keypair(&test_context, &key_pair, KeyPairUse::Default)
+            key::store_self_keypair(&test_context, &key_pair)
                 .await
                 .expect("Failed to save key");
             test_context
@@ -400,11 +400,11 @@ impl TestContext {
     /// Sets a name for this [`TestContext`] if one isn't yet set.
     ///
     /// This will show up in events logged in the test output.
-    pub fn set_name(&self, name: impl Into<String>) {
+    pub fn set_name(&self, name: &str) {
         let mut context_names = CONTEXT_NAMES.write().unwrap();
         context_names
             .entry(self.ctx.get_id())
-            .or_insert_with(|| name.into());
+            .or_insert_with(|| name.to_string());
     }
 
     /// Returns the name of this [`TestContext`].
@@ -607,6 +607,19 @@ impl TestContext {
         msg
     }
 
+    /// Receive a message using the `receive_imf()` pipeline. Panics if it's not hidden.
+    pub async fn recv_msg_hidden(&self, msg: &SentMessage<'_>) -> Message {
+        let received = self
+            .recv_msg_opt(msg)
+            .await
+            .expect("receive_imf() seems not to have added a new message to the db");
+        let msg = Message::load_from_db(self, *received.msg_ids.last().unwrap())
+            .await
+            .unwrap();
+        assert!(msg.hidden);
+        msg
+    }
+
     /// Receive a message using the `receive_imf()` pipeline. This is similar
     /// to `recv_msg()`, but doesn't assume that the message is shown in the chat.
     pub async fn recv_msg_opt(

src/tools.rs

@@ -34,6 +34,7 @@ use num_traits::PrimInt;
 use rand::{thread_rng, Rng};
 use tokio::{fs, io};
 use url::Url;
+use uuid::Uuid;
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
 use crate::config::Config;
@@ -275,12 +276,10 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time
     }
 }
 
-/* Message-ID tools */
-
 /// Generate an unique ID.
 ///
 /// The generated ID should be short but unique:
-/// - short, because it used in Message-ID and Chat-Group-ID headers and in QR codes
+/// - short, because it used in Chat-Group-ID headers and in QR codes
 /// - unique as two IDs generated on two devices should not be the same
 ///
 /// IDs generated by this function have 144 bits of entropy
@@ -312,7 +311,15 @@ pub(crate) fn validate_id(s: &str) -> bool {
 /// - the message ID should be globally unique
 /// - do not add a counter or any private data as this leaks information unnecessarily
 pub(crate) fn create_outgoing_rfc724_mid() -> String {
-    format!("{}@localhost", create_id())
+    // We use UUID similarly to iCloud web mail client
+    // because it seems their spam filter does not like Message-IDs
+    // without hyphens.
+    //
+    // However, we use `localhost` instead of the real domain to avoid
+    // leaking the domain when resent by otherwise anonymizing
+    // From-rewriting mailing lists and forwarders.
+    let uuid = Uuid::new_v4();
+    format!("{uuid}@localhost")
 }
 
 // the returned suffix is lower-case
@@ -341,14 +348,13 @@ pub(crate) fn get_abs_path(context: &Context, path: &Path) -> PathBuf {
     }
 }
 
-pub(crate) async fn get_filebytes(context: &Context, path: impl AsRef<Path>) -> Result<u64> {
-    let path_abs = get_abs_path(context, path.as_ref());
+pub(crate) async fn get_filebytes(context: &Context, path: &Path) -> Result<u64> {
+    let path_abs = get_abs_path(context, path);
     let meta = fs::metadata(&path_abs).await?;
     Ok(meta.len())
 }
 
-pub(crate) async fn delete_file(context: &Context, path: impl AsRef<Path>) -> Result<()> {
-    let path = path.as_ref();
+pub(crate) async fn delete_file(context: &Context, path: &Path) -> Result<()> {
     let path_abs = get_abs_path(context, path);
     if !path_abs.exists() {
         bail!("path {} does not exist", path_abs.display());
@@ -436,11 +442,8 @@ impl AsRef<Path> for TempPathGuard {
     }
 }
 
-pub(crate) async fn create_folder(
-    context: &Context,
-    path: impl AsRef<Path>,
-) -> Result<(), io::Error> {
-    let path_abs = get_abs_path(context, path.as_ref());
+pub(crate) async fn create_folder(context: &Context, path: &Path) -> Result<(), io::Error> {
+    let path_abs = get_abs_path(context, path);
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs).await {
             Ok(_) => Ok(()),
@@ -448,7 +451,7 @@ pub(crate) async fn create_folder(
                 warn!(
                     context,
                     "Cannot create directory \"{}\": {}",
-                    path.as_ref().display(),
+                    path.display(),
                     err
                 );
                 Err(err)
@@ -462,16 +465,16 @@ pub(crate) async fn create_folder(
 /// Write a the given content to provided file path.
 pub(crate) async fn write_file(
     context: &Context,
-    path: impl AsRef<Path>,
+    path: &Path,
     buf: &[u8],
 ) -> Result<(), io::Error> {
-    let path_abs = get_abs_path(context, path.as_ref());
+    let path_abs = get_abs_path(context, path);
     fs::write(&path_abs, buf).await.map_err(|err| {
         warn!(
             context,
             "Cannot write {} bytes to \"{}\": {}",
             buf.len(),
-            path.as_ref().display(),
+            path.display(),
             err
         );
         err
@@ -479,8 +482,8 @@ pub(crate) async fn write_file(
 }
 
 /// Reads the file and returns its context as a byte vector.
-pub async fn read_file(context: &Context, path: impl AsRef<Path>) -> Result<Vec<u8>> {
-    let path_abs = get_abs_path(context, path.as_ref());
+pub async fn read_file(context: &Context, path: &Path) -> Result<Vec<u8>> {
+    let path_abs = get_abs_path(context, path);
 
     match fs::read(&path_abs).await {
         Ok(bytes) => Ok(bytes),
@@ -488,7 +491,7 @@ pub async fn read_file(context: &Context, path: impl AsRef<Path>) -> Result<Vec<
             warn!(
                 context,
                 "Cannot read \"{}\" or file is empty: {}",
-                path.as_ref().display(),
+                path.display(),
                 err
             );
             Err(err.into())
@@ -496,8 +499,8 @@ pub async fn read_file(context: &Context, path: impl AsRef<Path>) -> Result<Vec<
     }
 }
 
-pub async fn open_file(context: &Context, path: impl AsRef<Path>) -> Result<fs::File> {
-    let path_abs = get_abs_path(context, path.as_ref());
+pub async fn open_file(context: &Context, path: &Path) -> Result<fs::File> {
+    let path_abs = get_abs_path(context, path);
 
     match fs::File::open(&path_abs).await {
         Ok(bytes) => Ok(bytes),
@@ -505,7 +508,7 @@ pub async fn open_file(context: &Context, path: impl AsRef<Path>) -> Result<fs::
             warn!(
                 context,
                 "Cannot read \"{}\" or file is empty: {}",
-                path.as_ref().display(),
+                path.display(),
                 err
             );
             Err(err.into())
@@ -733,566 +736,4 @@ pub(crate) fn inc_and_check<T: PrimInt + AddAssign + std::fmt::Debug>(
 }
 
 #[cfg(test)]
-mod tests {
-    use chrono::NaiveDate;
-    use proptest::prelude::*;
-
-    use super::*;
-    use crate::chatlist::Chatlist;
-    use crate::{chat, test_utils};
-    use crate::{receive_imf::receive_imf, test_utils::TestContext};
-
-    #[test]
-    fn test_parse_receive_headers() {
-        // Test `parse_receive_headers()` with some more-or-less random emails from the test-data
-        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let expected =
-            "Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000\n\
-             Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
-        check_parse_receive_headers(raw, expected);
-
-        let raw = include_bytes!("../test-data/message/wrong-html.eml");
-        let expected =
-            "Hop: From: oxbsltgw18.schlund.de; By: mrelayeu.kundenserver.de; Date: Thu, 6 Aug 2020 16:40:31 +0000\n\
-             Hop: From: mout.kundenserver.de; By: dd37930.kasserver.com; Date: Thu, 6 Aug 2020 16:40:32 +0000";
-        check_parse_receive_headers(raw, expected);
-
-        let raw = include_bytes!("../test-data/message/posteo_ndn.eml");
-        let expected =
-            "Hop: By: mout01.posteo.de; Date: Tue, 9 Jun 2020 18:44:22 +0000\n\
-             Hop: From: mout01.posteo.de; By: mx04.posteo.de; Date: Tue, 9 Jun 2020 18:44:22 +0000\n\
-             Hop: From: mx04.posteo.de; By: mailin06.posteo.de; Date: Tue, 9 Jun 2020 18:44:23 +0000\n\
-             Hop: From: mailin06.posteo.de; By: proxy02.posteo.de; Date: Tue, 9 Jun 2020 18:44:23 +0000\n\
-             Hop: From: proxy02.posteo.de; By: proxy02.posteo.name; Date: Tue, 9 Jun 2020 18:44:23 +0000\n\
-             Hop: From: proxy02.posteo.name; By: dovecot03.posteo.local; Date: Tue, 9 Jun 2020 18:44:24 +0000";
-        check_parse_receive_headers(raw, expected);
-    }
-
-    fn check_parse_receive_headers(raw: &[u8], expected: &str) {
-        let mail = mailparse::parse_mail(raw).unwrap();
-        let hop_info = parse_receive_headers(&mail.get_headers());
-        assert_eq!(hop_info, expected)
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_parse_receive_headers_integration() {
-        let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let expected = r"State: Fresh
-
-hi
-
-Message-ID: 2dfdbde7@example.org
-
-Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
-Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000
-
-DKIM Results: Passed=true";
-        check_parse_receive_headers_integration(raw, expected).await;
-
-        let raw = include_bytes!("../test-data/message/encrypted_with_received_headers.eml");
-        let expected = "State: Fresh, Encrypted
-
-Re: Message from alice@example.org
-
-hi back\r\n\
-\r\n\
--- \r\n\
-Sent with my Delta Chat Messenger: https://delta.chat
-
-Message-ID: Mr.adQpEwndXLH.LPDdlFVJ7wG@example.net
-
-Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0000
-Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
-Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
-
-DKIM Results: Passed=true";
-        check_parse_receive_headers_integration(raw, expected).await;
-    }
-
-    async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
-        let t = TestContext::new_alice().await;
-        receive_imf(&t, raw, false).await.unwrap();
-        let msg = t.get_last_msg().await;
-        let msg_info = msg.id.get_info(&t).await.unwrap();
-
-        // Ignore the first rows of the msg_info because they contain a
-        // received time that depends on the test time which makes it impossible to
-        // compare with a static string
-        let capped_result = &msg_info[msg_info.find("State").unwrap()..];
-        assert_eq!(expected, capped_result);
-    }
-
-    #[test]
-    fn test_rust_ftoa() {
-        assert_eq!("1.22", format!("{}", 1.22));
-    }
-
-    #[test]
-    fn test_truncate_1() {
-        let s = "this is a little test string";
-        assert_eq!(truncate(s, 16), "this is a [...]");
-    }
-
-    #[test]
-    fn test_truncate_2() {
-        assert_eq!(truncate("1234", 2), "1234");
-    }
-
-    #[test]
-    fn test_truncate_3() {
-        assert_eq!(truncate("1234567", 1), "1[...]");
-    }
-
-    #[test]
-    fn test_truncate_4() {
-        assert_eq!(truncate("123456", 4), "123456");
-    }
-
-    #[test]
-    fn test_truncate_edge() {
-        assert_eq!(truncate("", 4), "");
-
-        assert_eq!(truncate("\n  hello \n world", 4), "\n  [...]");
-
-        assert_eq!(truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
-        assert_eq!(truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0), "[...]");
-
-        // 9 characters, so no truncation
-        assert_eq!(truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
-
-        // 12 characters, truncation
-        assert_eq!(
-            truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6),
-            "𑒀ὐ￠🜀\u{1e01b}A[...]",
-        );
-    }
-
-    mod truncate_by_lines {
-        use super::*;
-
-        #[test]
-        fn test_just_text() {
-            let s = "this is a little test string".to_string();
-            assert_eq!(
-                truncate_by_lines(s, 4, 6),
-                ("this is a little test [...]".to_string(), true)
-            );
-        }
-
-        #[test]
-        fn test_with_linebreaks() {
-            let s = "this\n is\n a little test string".to_string();
-            assert_eq!(
-                truncate_by_lines(s, 4, 6),
-                ("this\n is\n a little [...]".to_string(), true)
-            );
-        }
-
-        #[test]
-        fn test_only_linebreaks() {
-            let s = "\n\n\n\n\n\n\n".to_string();
-            assert_eq!(
-                truncate_by_lines(s, 4, 5),
-                ("\n\n\n[...]".to_string(), true)
-            );
-        }
-
-        #[test]
-        fn limit_hits_end() {
-            let s = "hello\n world !".to_string();
-            assert_eq!(
-                truncate_by_lines(s, 2, 8),
-                ("hello\n world !".to_string(), false)
-            );
-        }
-
-        #[test]
-        fn test_edge() {
-            assert_eq!(
-                truncate_by_lines("".to_string(), 2, 4),
-                ("".to_string(), false)
-            );
-
-            assert_eq!(
-                truncate_by_lines("\n  hello \n world".to_string(), 2, 4),
-                ("\n  [...]".to_string(), true)
-            );
-            assert_eq!(
-                truncate_by_lines("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ".to_string(), 1, 2),
-                ("𐠈0[...]".to_string(), true)
-            );
-            assert_eq!(
-                truncate_by_lines("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ".to_string(), 1, 0),
-                ("[...]".to_string(), true)
-            );
-
-            // 9 characters, so no truncation
-            assert_eq!(
-                truncate_by_lines("𑒀ὐ￠🜀\u{1e01b}A a🟠".to_string(), 1, 12),
-                ("𑒀ὐ￠🜀\u{1e01b}A a🟠".to_string(), false),
-            );
-
-            // 12 characters, truncation
-            assert_eq!(
-                truncate_by_lines("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd".to_string(), 1, 7),
-                ("𑒀ὐ￠🜀\u{1e01b}A [...]".to_string(), true),
-            );
-        }
-    }
-
-    #[test]
-    fn test_create_id() {
-        let buf = create_id();
-        assert_eq!(buf.len(), 24);
-    }
-
-    #[test]
-    fn test_validate_id() {
-        for _ in 0..10 {
-            assert!(validate_id(&create_id()));
-        }
-
-        assert_eq!(validate_id("aaaaaaaaaaaa"), true);
-        assert_eq!(validate_id("aa-aa_aaaXaa"), true);
-
-        // ID cannot contain whitespace.
-        assert_eq!(validate_id("aaaaa aaaaaa"), false);
-        assert_eq!(validate_id("aaaaa\naaaaaa"), false);
-
-        // ID cannot contain "/", "+".
-        assert_eq!(validate_id("aaaaa/aaaaaa"), false);
-        assert_eq!(validate_id("aaaaaaaa+aaa"), false);
-
-        // Too long ID.
-        assert_eq!(validate_id("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), false);
-    }
-
-    #[test]
-    fn test_create_id_invalid_chars() {
-        for _ in 1..1000 {
-            let buf = create_id();
-            assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
-            assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
-        }
-    }
-
-    #[test]
-    fn test_create_outgoing_rfc724_mid() {
-        let mid = create_outgoing_rfc724_mid();
-        assert_eq!(mid.len(), 34);
-        assert!(mid.ends_with("@localhost"));
-    }
-
-    proptest! {
-        #[test]
-        fn test_truncate(
-            buf: String,
-            approx_chars in 0..100usize
-        ) {
-            let res = truncate(&buf, approx_chars);
-            let el_len = 5;
-            let l = res.chars().count();
-            assert!(
-                l <= approx_chars + el_len,
-                "buf: '{}' - res: '{}' - len {}, approx {}",
-                &buf, &res, res.len(), approx_chars
-            );
-
-            if buf.chars().count() > approx_chars + el_len {
-                let l = res.len();
-                assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
-            }
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_file_handling() {
-        let t = TestContext::new().await;
-        let context = &t;
-        macro_rules! file_exist {
-            ($ctx:expr, $fname:expr) => {
-                $ctx.get_blobdir()
-                    .join(Path::new($fname).file_name().unwrap())
-                    .exists()
-            };
-        }
-
-        assert!(delete_file(context, "$BLOBDIR/lkqwjelqkwlje")
-            .await
-            .is_err());
-        assert!(write_file(context, "$BLOBDIR/foobar", b"content")
-            .await
-            .is_ok());
-        assert!(file_exist!(context, "$BLOBDIR/foobar"));
-        assert!(!file_exist!(context, "$BLOBDIR/foobarx"));
-        assert_eq!(get_filebytes(context, "$BLOBDIR/foobar").await.unwrap(), 7);
-
-        let abs_path = context
-            .get_blobdir()
-            .join("foobar")
-            .to_string_lossy()
-            .to_string();
-
-        assert!(file_exist!(context, &abs_path));
-
-        assert!(delete_file(context, "$BLOBDIR/foobar").await.is_ok());
-        assert!(create_folder(context, "$BLOBDIR/foobar-folder")
-            .await
-            .is_ok());
-        assert!(file_exist!(context, "$BLOBDIR/foobar-folder"));
-        assert!(delete_file(context, "$BLOBDIR/foobar-folder")
-            .await
-            .is_err());
-
-        let fn0 = "$BLOBDIR/data.data";
-        assert!(write_file(context, &fn0, b"content").await.is_ok());
-
-        assert!(delete_file(context, &fn0).await.is_ok());
-        assert!(!file_exist!(context, &fn0));
-    }
-
-    #[test]
-    fn test_duration_to_str() {
-        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
-        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
-        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 59)),
-            "0h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 60)),
-            "1h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
-            "2h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
-            "3h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
-            "3h 0m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
-            "3h 1m 0s"
-        );
-    }
-
-    #[test]
-    fn test_get_filemeta() {
-        let (w, h) = get_filemeta(test_utils::AVATAR_900x900_BYTES).unwrap();
-        assert_eq!(w, 900);
-        assert_eq!(h, 900);
-
-        let data = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        let (w, h) = get_filemeta(data).unwrap();
-        assert_eq!(w, 1000);
-        assert_eq!(h, 1000);
-
-        let data = include_bytes!("../test-data/image/image100x50.gif");
-        let (w, h) = get_filemeta(data).unwrap();
-        assert_eq!(w, 100);
-        assert_eq!(h, 50);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_maybe_warn_on_bad_time() {
-        let t = TestContext::new().await;
-        let timestamp_now = time();
-        let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
-        let timestamp_past = NaiveDateTime::new(
-            NaiveDate::from_ymd_opt(2020, 9, 1).unwrap(),
-            NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
-        )
-        .and_utc()
-        .timestamp_millis()
-            / 1_000;
-
-        // a correct time must not add a device message
-        maybe_warn_on_bad_time(&t, timestamp_now, get_release_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-
-        // we cannot find out if a date in the future is wrong - a device message is not added
-        maybe_warn_on_bad_time(&t, timestamp_future, get_release_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-
-        // a date in the past must add a device message
-        maybe_warn_on_bad_time(&t, timestamp_past, get_release_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        assert_eq!(msgs.len(), 1);
-
-        // the message should be added only once a day - test that an hour later and nearly a day later
-        maybe_warn_on_bad_time(&t, timestamp_past + 60 * 60, get_release_timestamp()).await;
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        assert_eq!(msgs.len(), 1);
-
-        maybe_warn_on_bad_time(
-            &t,
-            timestamp_past + 60 * 60 * 24 - 1,
-            get_release_timestamp(),
-        )
-        .await;
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        assert_eq!(msgs.len(), 1);
-
-        // next day, there should be another device message
-        maybe_warn_on_bad_time(&t, timestamp_past + 60 * 60 * 24, get_release_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        assert_eq!(device_chat_id, chats.get_chat_id(0).unwrap());
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        assert_eq!(msgs.len(), 2);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_maybe_warn_on_outdated() {
-        let t = TestContext::new().await;
-        let timestamp_now: i64 = time();
-
-        // in about 6 months, the app should not be outdated
-        // (if this fails, provider-db is not updated since 6 months)
-        maybe_warn_on_outdated(
-            &t,
-            timestamp_now + 180 * 24 * 60 * 60,
-            get_release_timestamp(),
-        )
-        .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 0);
-
-        // in 1 year, the app should be considered as outdated
-        maybe_warn_on_outdated(
-            &t,
-            timestamp_now + 365 * 24 * 60 * 60,
-            get_release_timestamp(),
-        )
-        .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        assert_eq!(msgs.len(), 1);
-
-        // do not repeat the warning every day ...
-        // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
-        maybe_warn_on_outdated(
-            &t,
-            timestamp_now + (365 + 1) * 24 * 60 * 60,
-            get_release_timestamp(),
-        )
-        .await;
-        maybe_warn_on_outdated(
-            &t,
-            timestamp_now + (365 + 2) * 24 * 60 * 60,
-            get_release_timestamp(),
-        )
-        .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        let test_len = msgs.len();
-        assert!(test_len == 1 || test_len == 2);
-
-        // ... but every month
-        // (forward generous 33 days to avoid being in the same month as in the previous check)
-        maybe_warn_on_outdated(
-            &t,
-            timestamp_now + (365 + 33) * 24 * 60 * 60,
-            get_release_timestamp(),
-        )
-        .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
-        let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
-        assert_eq!(msgs.len(), test_len + 1);
-    }
-
-    #[test]
-    fn test_get_release_timestamp() {
-        let timestamp_past = NaiveDateTime::new(
-            NaiveDate::from_ymd_opt(2020, 9, 9).unwrap(),
-            NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
-        )
-        .and_utc()
-        .timestamp_millis()
-            / 1_000;
-        assert!(get_release_timestamp() <= time());
-        assert!(get_release_timestamp() > timestamp_past);
-    }
-
-    #[test]
-    fn test_remove_subject_prefix() {
-        assert_eq!(remove_subject_prefix("Subject"), "Subject");
-        assert_eq!(
-            remove_subject_prefix("Chat: Re: Subject"),
-            "Chat: Re: Subject"
-        );
-        assert_eq!(remove_subject_prefix("Re: Subject"), "Subject");
-        assert_eq!(remove_subject_prefix("Fwd: Subject"), "Subject");
-        assert_eq!(remove_subject_prefix("Fw: Subject"), "Subject");
-    }
-
-    #[test]
-    fn test_parse_mailto() {
-        let mailto_url = "mailto:someone@example.com";
-        let reps = parse_mailto(mailto_url);
-        assert_eq!(
-            Some(MailTo {
-                to: vec![EmailAddress {
-                    local: "someone".to_string(),
-                    domain: "example.com".to_string()
-                }],
-                subject: None,
-                body: None
-            }),
-            reps
-        );
-
-        let mailto_url = "mailto:someone@example.com?subject=Hello%20World";
-        let reps = parse_mailto(mailto_url);
-        assert_eq!(
-            Some(MailTo {
-                to: vec![EmailAddress {
-                    local: "someone".to_string(),
-                    domain: "example.com".to_string()
-                }],
-                subject: Some("Hello World".to_string()),
-                body: None
-            }),
-            reps
-        );
-
-        let mailto_url = "mailto:someone@example.com,someoneelse@example.com?subject=Hello%20World&body=This%20is%20a%20test";
-        let reps = parse_mailto(mailto_url);
-        assert_eq!(
-            Some(MailTo {
-                to: vec![
-                    EmailAddress {
-                        local: "someone".to_string(),
-                        domain: "example.com".to_string()
-                    },
-                    EmailAddress {
-                        local: "someoneelse".to_string(),
-                        domain: "example.com".to_string()
-                    }
-                ],
-                subject: Some("Hello World".to_string()),
-                body: Some("This is a test".to_string())
-            }),
-            reps
-        );
-    }
-}
+mod tools_tests;

src/tools/tools_tests.rs

@@ -0,0 +1,608 @@
+use chrono::NaiveDate;
+use proptest::prelude::*;
+
+use super::*;
+use crate::chatlist::Chatlist;
+use crate::{chat, test_utils};
+use crate::{receive_imf::receive_imf, test_utils::TestContext};
+
+#[test]
+fn test_parse_receive_headers() {
+    // Test `parse_receive_headers()` with some more-or-less random emails from the test-data
+    let raw = include_bytes!("../../test-data/message/mail_with_cc.txt");
+    let expected =
+        "Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000\n\
+             Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000";
+    check_parse_receive_headers(raw, expected);
+
+    let raw = include_bytes!("../../test-data/message/wrong-html.eml");
+    let expected =
+        "Hop: From: oxbsltgw18.schlund.de; By: mrelayeu.kundenserver.de; Date: Thu, 6 Aug 2020 16:40:31 +0000\n\
+             Hop: From: mout.kundenserver.de; By: dd37930.kasserver.com; Date: Thu, 6 Aug 2020 16:40:32 +0000";
+    check_parse_receive_headers(raw, expected);
+
+    let raw = include_bytes!("../../test-data/message/posteo_ndn.eml");
+    let expected =
+        "Hop: By: mout01.posteo.de; Date: Tue, 9 Jun 2020 18:44:22 +0000\n\
+             Hop: From: mout01.posteo.de; By: mx04.posteo.de; Date: Tue, 9 Jun 2020 18:44:22 +0000\n\
+             Hop: From: mx04.posteo.de; By: mailin06.posteo.de; Date: Tue, 9 Jun 2020 18:44:23 +0000\n\
+             Hop: From: mailin06.posteo.de; By: proxy02.posteo.de; Date: Tue, 9 Jun 2020 18:44:23 +0000\n\
+             Hop: From: proxy02.posteo.de; By: proxy02.posteo.name; Date: Tue, 9 Jun 2020 18:44:23 +0000\n\
+             Hop: From: proxy02.posteo.name; By: dovecot03.posteo.local; Date: Tue, 9 Jun 2020 18:44:24 +0000";
+    check_parse_receive_headers(raw, expected);
+}
+
+fn check_parse_receive_headers(raw: &[u8], expected: &str) {
+    let mail = mailparse::parse_mail(raw).unwrap();
+    let hop_info = parse_receive_headers(&mail.get_headers());
+    assert_eq!(hop_info, expected)
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_parse_receive_headers_integration() {
+    let raw = include_bytes!("../../test-data/message/mail_with_cc.txt");
+    let expected = r"State: Fresh
+
+hi
+
+Message-ID: 2dfdbde7@example.org
+
+Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
+Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000
+
+DKIM Results: Passed=true";
+    check_parse_receive_headers_integration(raw, expected).await;
+
+    let raw = include_bytes!("../../test-data/message/encrypted_with_received_headers.eml");
+    let expected = "State: Fresh, Encrypted
+
+Re: Message from alice@example.org
+
+hi back\r\n\
+\r\n\
+-- \r\n\
+Sent with my Delta Chat Messenger: https://delta.chat
+
+Message-ID: Mr.adQpEwndXLH.LPDdlFVJ7wG@example.net
+
+Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0000
+Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
+Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
+
+DKIM Results: Passed=true";
+    check_parse_receive_headers_integration(raw, expected).await;
+}
+
+async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
+    let t = TestContext::new_alice().await;
+    receive_imf(&t, raw, false).await.unwrap();
+    let msg = t.get_last_msg().await;
+    let msg_info = msg.id.get_info(&t).await.unwrap();
+
+    // Ignore the first rows of the msg_info because they contain a
+    // received time that depends on the test time which makes it impossible to
+    // compare with a static string
+    let capped_result = &msg_info[msg_info.find("State").unwrap()..];
+    assert_eq!(expected, capped_result);
+}
+
+#[test]
+fn test_rust_ftoa() {
+    assert_eq!("1.22", format!("{}", 1.22));
+}
+
+#[test]
+fn test_truncate_1() {
+    let s = "this is a little test string";
+    assert_eq!(truncate(s, 16), "this is a [...]");
+}
+
+#[test]
+fn test_truncate_2() {
+    assert_eq!(truncate("1234", 2), "1234");
+}
+
+#[test]
+fn test_truncate_3() {
+    assert_eq!(truncate("1234567", 1), "1[...]");
+}
+
+#[test]
+fn test_truncate_4() {
+    assert_eq!(truncate("123456", 4), "123456");
+}
+
+#[test]
+fn test_truncate_edge() {
+    assert_eq!(truncate("", 4), "");
+
+    assert_eq!(truncate("\n  hello \n world", 4), "\n  [...]");
+
+    assert_eq!(truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
+    assert_eq!(truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0), "[...]");
+
+    // 9 characters, so no truncation
+    assert_eq!(truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
+
+    // 12 characters, truncation
+    assert_eq!(
+        truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6),
+        "𑒀ὐ￠🜀\u{1e01b}A[...]",
+    );
+}
+
+mod truncate_by_lines {
+    use super::*;
+
+    #[test]
+    fn test_just_text() {
+        let s = "this is a little test string".to_string();
+        assert_eq!(
+            truncate_by_lines(s, 4, 6),
+            ("this is a little test [...]".to_string(), true)
+        );
+    }
+
+    #[test]
+    fn test_with_linebreaks() {
+        let s = "this\n is\n a little test string".to_string();
+        assert_eq!(
+            truncate_by_lines(s, 4, 6),
+            ("this\n is\n a little [...]".to_string(), true)
+        );
+    }
+
+    #[test]
+    fn test_only_linebreaks() {
+        let s = "\n\n\n\n\n\n\n".to_string();
+        assert_eq!(
+            truncate_by_lines(s, 4, 5),
+            ("\n\n\n[...]".to_string(), true)
+        );
+    }
+
+    #[test]
+    fn limit_hits_end() {
+        let s = "hello\n world !".to_string();
+        assert_eq!(
+            truncate_by_lines(s, 2, 8),
+            ("hello\n world !".to_string(), false)
+        );
+    }
+
+    #[test]
+    fn test_edge() {
+        assert_eq!(
+            truncate_by_lines("".to_string(), 2, 4),
+            ("".to_string(), false)
+        );
+
+        assert_eq!(
+            truncate_by_lines("\n  hello \n world".to_string(), 2, 4),
+            ("\n  [...]".to_string(), true)
+        );
+        assert_eq!(
+            truncate_by_lines("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ".to_string(), 1, 2),
+            ("𐠈0[...]".to_string(), true)
+        );
+        assert_eq!(
+            truncate_by_lines("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ".to_string(), 1, 0),
+            ("[...]".to_string(), true)
+        );
+
+        // 9 characters, so no truncation
+        assert_eq!(
+            truncate_by_lines("𑒀ὐ￠🜀\u{1e01b}A a🟠".to_string(), 1, 12),
+            ("𑒀ὐ￠🜀\u{1e01b}A a🟠".to_string(), false),
+        );
+
+        // 12 characters, truncation
+        assert_eq!(
+            truncate_by_lines("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd".to_string(), 1, 7),
+            ("𑒀ὐ￠🜀\u{1e01b}A [...]".to_string(), true),
+        );
+    }
+}
+
+#[test]
+fn test_create_id() {
+    let buf = create_id();
+    assert_eq!(buf.len(), 24);
+}
+
+#[test]
+fn test_validate_id() {
+    for _ in 0..10 {
+        assert!(validate_id(&create_id()));
+    }
+
+    assert_eq!(validate_id("aaaaaaaaaaaa"), true);
+    assert_eq!(validate_id("aa-aa_aaaXaa"), true);
+
+    // ID cannot contain whitespace.
+    assert_eq!(validate_id("aaaaa aaaaaa"), false);
+    assert_eq!(validate_id("aaaaa\naaaaaa"), false);
+
+    // ID cannot contain "/", "+".
+    assert_eq!(validate_id("aaaaa/aaaaaa"), false);
+    assert_eq!(validate_id("aaaaaaaa+aaa"), false);
+
+    // Too long ID.
+    assert_eq!(validate_id("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), false);
+}
+
+#[test]
+fn test_create_id_invalid_chars() {
+    for _ in 1..1000 {
+        let buf = create_id();
+        assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
+        assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
+    }
+}
+
+#[test]
+fn test_create_outgoing_rfc724_mid() {
+    let mid = create_outgoing_rfc724_mid();
+    assert_eq!(mid.len(), 46);
+    assert!(mid.contains("-")); // It has an UUID inside.
+    assert!(mid.ends_with("@localhost"));
+}
+
+proptest! {
+    #[test]
+    fn test_truncate(
+        buf: String,
+        approx_chars in 0..100usize
+    ) {
+        let res = truncate(&buf, approx_chars);
+        let el_len = 5;
+        let l = res.chars().count();
+        assert!(
+            l <= approx_chars + el_len,
+            "buf: '{}' - res: '{}' - len {}, approx {}",
+            &buf, &res, res.len(), approx_chars
+        );
+
+        if buf.chars().count() > approx_chars + el_len {
+            let l = res.len();
+            assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+        }
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_file_handling() {
+    let t = TestContext::new().await;
+    let context = &t;
+    macro_rules! file_exist {
+        ($ctx:expr, $fname:expr) => {
+            $ctx.get_blobdir()
+                .join(Path::new($fname).file_name().unwrap())
+                .exists()
+        };
+    }
+
+    assert!(delete_file(context, Path::new("$BLOBDIR/lkqwjelqkwlje"))
+        .await
+        .is_err());
+    assert!(
+        write_file(context, Path::new("$BLOBDIR/foobar"), b"content")
+            .await
+            .is_ok()
+    );
+    assert!(file_exist!(context, "$BLOBDIR/foobar"));
+    assert!(!file_exist!(context, "$BLOBDIR/foobarx"));
+    assert_eq!(
+        get_filebytes(context, Path::new("$BLOBDIR/foobar"))
+            .await
+            .unwrap(),
+        7
+    );
+
+    let abs_path = context
+        .get_blobdir()
+        .join("foobar")
+        .to_string_lossy()
+        .to_string();
+
+    assert!(file_exist!(context, &abs_path));
+
+    assert!(delete_file(context, Path::new("$BLOBDIR/foobar"))
+        .await
+        .is_ok());
+    assert!(create_folder(context, Path::new("$BLOBDIR/foobar-folder"))
+        .await
+        .is_ok());
+    assert!(file_exist!(context, "$BLOBDIR/foobar-folder"));
+    assert!(delete_file(context, Path::new("$BLOBDIR/foobar-folder"))
+        .await
+        .is_err());
+
+    let fn0 = "$BLOBDIR/data.data";
+    assert!(write_file(context, Path::new(fn0), b"content")
+        .await
+        .is_ok());
+
+    assert!(delete_file(context, Path::new(fn0)).await.is_ok());
+    assert!(!file_exist!(context, &fn0));
+}
+
+#[test]
+fn test_duration_to_str() {
+    assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
+    assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
+    assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
+    assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
+    assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
+    assert_eq!(
+        duration_to_str(Duration::from_secs(59 * 60 + 59)),
+        "0h 59m 59s"
+    );
+    assert_eq!(
+        duration_to_str(Duration::from_secs(59 * 60 + 60)),
+        "1h 0m 0s"
+    );
+    assert_eq!(
+        duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
+        "2h 59m 59s"
+    );
+    assert_eq!(
+        duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
+        "3h 0m 0s"
+    );
+    assert_eq!(
+        duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
+        "3h 0m 59s"
+    );
+    assert_eq!(
+        duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
+        "3h 1m 0s"
+    );
+}
+
+#[test]
+fn test_get_filemeta() {
+    let (w, h) = get_filemeta(test_utils::AVATAR_900x900_BYTES).unwrap();
+    assert_eq!(w, 900);
+    assert_eq!(h, 900);
+
+    let data = include_bytes!("../../test-data/image/avatar1000x1000.jpg");
+    let (w, h) = get_filemeta(data).unwrap();
+    assert_eq!(w, 1000);
+    assert_eq!(h, 1000);
+
+    let data = include_bytes!("../../test-data/image/image100x50.gif");
+    let (w, h) = get_filemeta(data).unwrap();
+    assert_eq!(w, 100);
+    assert_eq!(h, 50);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_maybe_warn_on_bad_time() {
+    let t = TestContext::new().await;
+    let timestamp_now = time();
+    let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
+    let timestamp_past = NaiveDateTime::new(
+        NaiveDate::from_ymd_opt(2020, 9, 1).unwrap(),
+        NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
+    )
+    .and_utc()
+    .timestamp_millis()
+        / 1_000;
+
+    // a correct time must not add a device message
+    maybe_warn_on_bad_time(&t, timestamp_now, get_release_timestamp()).await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 0);
+
+    // we cannot find out if a date in the future is wrong - a device message is not added
+    maybe_warn_on_bad_time(&t, timestamp_future, get_release_timestamp()).await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 0);
+
+    // a date in the past must add a device message
+    maybe_warn_on_bad_time(&t, timestamp_past, get_release_timestamp()).await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 1);
+    let device_chat_id = chats.get_chat_id(0).unwrap();
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    assert_eq!(msgs.len(), 1);
+
+    // the message should be added only once a day - test that an hour later and nearly a day later
+    maybe_warn_on_bad_time(&t, timestamp_past + 60 * 60, get_release_timestamp()).await;
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    assert_eq!(msgs.len(), 1);
+
+    maybe_warn_on_bad_time(
+        &t,
+        timestamp_past + 60 * 60 * 24 - 1,
+        get_release_timestamp(),
+    )
+    .await;
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    assert_eq!(msgs.len(), 1);
+
+    // next day, there should be another device message
+    maybe_warn_on_bad_time(&t, timestamp_past + 60 * 60 * 24, get_release_timestamp()).await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 1);
+    assert_eq!(device_chat_id, chats.get_chat_id(0).unwrap());
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    assert_eq!(msgs.len(), 2);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_maybe_warn_on_outdated() {
+    let t = TestContext::new().await;
+    let timestamp_now: i64 = time();
+
+    // in about 6 months, the app should not be outdated
+    // (if this fails, provider-db is not updated since 6 months)
+    maybe_warn_on_outdated(
+        &t,
+        timestamp_now + 180 * 24 * 60 * 60,
+        get_release_timestamp(),
+    )
+    .await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 0);
+
+    // in 1 year, the app should be considered as outdated
+    maybe_warn_on_outdated(
+        &t,
+        timestamp_now + 365 * 24 * 60 * 60,
+        get_release_timestamp(),
+    )
+    .await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 1);
+    let device_chat_id = chats.get_chat_id(0).unwrap();
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    assert_eq!(msgs.len(), 1);
+
+    // do not repeat the warning every day ...
+    // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
+    maybe_warn_on_outdated(
+        &t,
+        timestamp_now + (365 + 1) * 24 * 60 * 60,
+        get_release_timestamp(),
+    )
+    .await;
+    maybe_warn_on_outdated(
+        &t,
+        timestamp_now + (365 + 2) * 24 * 60 * 60,
+        get_release_timestamp(),
+    )
+    .await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 1);
+    let device_chat_id = chats.get_chat_id(0).unwrap();
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    let test_len = msgs.len();
+    assert!(test_len == 1 || test_len == 2);
+
+    // ... but every month
+    // (forward generous 33 days to avoid being in the same month as in the previous check)
+    maybe_warn_on_outdated(
+        &t,
+        timestamp_now + (365 + 33) * 24 * 60 * 60,
+        get_release_timestamp(),
+    )
+    .await;
+    let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+    assert_eq!(chats.len(), 1);
+    let device_chat_id = chats.get_chat_id(0).unwrap();
+    let msgs = chat::get_chat_msgs(&t, device_chat_id).await.unwrap();
+    assert_eq!(msgs.len(), test_len + 1);
+}
+
+#[test]
+fn test_get_release_timestamp() {
+    let timestamp_past = NaiveDateTime::new(
+        NaiveDate::from_ymd_opt(2020, 9, 9).unwrap(),
+        NaiveTime::from_hms_opt(0, 0, 0).unwrap(),
+    )
+    .and_utc()
+    .timestamp_millis()
+        / 1_000;
+    assert!(get_release_timestamp() <= time());
+    assert!(get_release_timestamp() > timestamp_past);
+}
+
+#[test]
+fn test_remove_subject_prefix() {
+    assert_eq!(remove_subject_prefix("Subject"), "Subject");
+    assert_eq!(
+        remove_subject_prefix("Chat: Re: Subject"),
+        "Chat: Re: Subject"
+    );
+    assert_eq!(remove_subject_prefix("Re: Subject"), "Subject");
+    assert_eq!(remove_subject_prefix("Fwd: Subject"), "Subject");
+    assert_eq!(remove_subject_prefix("Fw: Subject"), "Subject");
+}
+
+#[test]
+fn test_parse_mailto() {
+    let mailto_url = "mailto:someone@example.com";
+    let reps = parse_mailto(mailto_url);
+    assert_eq!(
+        Some(MailTo {
+            to: vec![EmailAddress {
+                local: "someone".to_string(),
+                domain: "example.com".to_string()
+            }],
+            subject: None,
+            body: None
+        }),
+        reps
+    );
+
+    let mailto_url = "mailto:someone@example.com?subject=Hello%20World";
+    let reps = parse_mailto(mailto_url);
+    assert_eq!(
+        Some(MailTo {
+            to: vec![EmailAddress {
+                local: "someone".to_string(),
+                domain: "example.com".to_string()
+            }],
+            subject: Some("Hello World".to_string()),
+            body: None
+        }),
+        reps
+    );
+
+    let mailto_url = "mailto:someone@example.com,someoneelse@example.com?subject=Hello%20World&body=This%20is%20a%20test";
+    let reps = parse_mailto(mailto_url);
+    assert_eq!(
+        Some(MailTo {
+            to: vec![
+                EmailAddress {
+                    local: "someone".to_string(),
+                    domain: "example.com".to_string()
+                },
+                EmailAddress {
+                    local: "someoneelse".to_string(),
+                    domain: "example.com".to_string()
+                }
+            ],
+            subject: Some("Hello World".to_string()),
+            body: Some("This is a test".to_string())
+        }),
+        reps
+    );
+}
+
+#[test]
+fn test_sanitize_filename() {
+    let name = sanitize_filename("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
+    assert!(!name.is_empty());
+
+    let name = sanitize_filename("wot.tar.gz");
+    assert_eq!(name, "wot.tar.gz");
+
+    let name = sanitize_filename(".foo.bar");
+    assert_eq!(name, "file.foo.bar");
+
+    let name = sanitize_filename("foo?.bar");
+    assert_eq!(name, "foo.bar");
+    assert!(!name.contains('?'));
+
+    let name = sanitize_filename("no-extension");
+    assert_eq!(name, "no-extension");
+
+    let name = sanitize_filename("path/ignored\\this: is* forbidden?.c");
+    assert_eq!(name, "this is forbidden.c");
+
+    let name =
+        sanitize_filename("file.with_lots_of_characters_behind_point_and_double_ending.tar.gz");
+    assert_eq!(
+        name,
+        "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz"
+    );
+
+    let name = sanitize_filename("a. tar.tar.gz");
+    assert_eq!(name, "a. tar.tar.gz");
+
+    let name = sanitize_filename("Guia_uso_GNB (v0.8).pdf");
+    assert_eq!(name, "Guia_uso_GNB (v0.8).pdf");
+}

src/update_helper.rs

@@ -213,6 +213,44 @@ mod tests {
         // Assert that the \n was correctly removed from the group name also in the system message
         assert_eq!(msg.text.contains('\n'), false);
 
+        // This doesn't update the name because Date is the same and name is greater.
+        receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.org\n\
+                 Message-ID: <msg4@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde123456\n\
+                 Chat-Group-Name: another name update 4\n\
+                 Chat-Group-Name-Changed: another name update\n\
+                 Date: Sun, 22 Mar 2021 03:00:00 +0000\n\
+                 \n\
+                 4th message\n",
+            false,
+        )
+        .await?;
+        let chat = Chat::load_from_db(&t, chat.id).await?;
+        assert_eq!(chat.name, "another name update");
+
+        // This updates the name because Date is the same and name is lower.
+        receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.org\n\
+                 Message-ID: <msg5@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: abcde123456\n\
+                 Chat-Group-Name: another name updat\n\
+                 Chat-Group-Name-Changed: another name update\n\
+                 Date: Sun, 22 Mar 2021 03:00:00 +0000\n\
+                 \n\
+                 5th message\n",
+            false,
+        )
+        .await?;
+        let chat = Chat::load_from_db(&t, chat.id).await?;
+        assert_eq!(chat.name, "another name updat");
+
         Ok(())
     }
 }

src/webxdc.rs

@@ -27,7 +27,7 @@ use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result};
 use async_zip::tokio::read::seek::ZipFileReader as SeekZipFileReader;
 use deltachat_contact_tools::sanitize_bidi_characters;
 use deltachat_derive::FromSql;
-use lettre_email::PartBuilder;
+use mail_builder::mime::MimePart;
 use rusqlite::OptionalExtension;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -41,7 +41,6 @@ use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{load_self_public_key, DcKey};
 use crate::message::{Message, MessageState, MsgId, Viewtype};
-use crate::mimefactory::wrapped_base64_encode;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
@@ -651,17 +650,8 @@ impl Context {
         }
     }
 
-    pub(crate) fn build_status_update_part(&self, json: &str) -> PartBuilder {
-        let encoded_body = wrapped_base64_encode(json.as_bytes());
-
-        PartBuilder::new()
-            .content_type(&"application/json".parse::<mime::Mime>().unwrap())
-            .header((
-                "Content-Disposition",
-                "attachment; filename=\"status-update.json\"",
-            ))
-            .header(("Content-Transfer-Encoding", "base64"))
-            .body(encoded_body)
+    pub(crate) fn build_status_update_part(&self, json: &str) -> MimePart<'static> {
+        MimePart::new("application/json", json.as_bytes().to_vec()).attachment("status-update.json")
     }
 
     /// Receives status updates from receive_imf to the database

test-data/message/mailparse-0.16.0-panic.eml

@@ -0,0 +1,3 @@
+Content-Type: multipart/mixed; boundary="foobar"
+
+--foobar--