imap: skip sync flags update if highest modseq has not increased

This operation takes roughly 0.3 s on a moderate size database.
Calling it once before scanning all folders and scanning
the watched folder instead of each time after downloading
a message from a folder speeds up IMAP loop.

.github/mergeable.yml

@@ -23,4 +23,4 @@ mergeable:
         status: 'action_required' 
         payload:
           title: Changlog might need an update
-          summary: "Check if CHANGELOG.md needs an update or add #skip-update to the PR description."
+          summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## Unreleased
+
+### Changes
+- don't watch Sent folder by default #3025
+- use webxdc app name in chatlist/quotes/replies etc. #3027
+- refactorings #3023
+- remove direct dependency on `byteorder` crate #3031
+- make it possible to cancel message sending by removing the message #3034,
+  this was previosuly removed in 1.71.0 #2939
+- always skip Seen flag synchronization when there are no updates #3039
+
+### Fixes
+- fix splitting off text from webxdc messages #3032
+- call slow `delete_expired_imap_messages()` less often #3037
+- make synchronization of Seen status more robust in case unsolicited FETCH
+  result without UID is returned #3022
+
+
+## 1.72.0
+
+### Fixes
+- run migrations on backup import #3006
+
+
 ## 1.71.0
 
 ### API Changes

Cargo.lock

@@ -420,9 +420,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "backtrace"
-version = "0.3.63"
+version = "0.3.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "321629d8ba6513061f26707241fa9bc89524ff1cd7a915a97ef0c62c666ce1b6"
+checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f"
 dependencies = [
  "addr2line",
  "cc",
@@ -1063,7 +1063,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.71.0"
+version = "1.72.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1077,7 +1077,6 @@ dependencies = [
  "backtrace",
  "base64 0.13.0",
  "bitflags",
- "byteorder",
  "chrono",
  "criterion",
  "deltachat_derive",
@@ -1091,6 +1090,7 @@ dependencies = [
  "hex",
  "humansize",
  "image",
+ "imap-proto",
  "kamadak-exif",
  "lettre_email",
  "libc",
@@ -1120,7 +1120,7 @@ dependencies = [
  "sha-1 0.10.0",
  "sha2 0.10.1",
  "smallvec",
- "stop-token 0.6.1",
+ "stop-token 0.7.0",
  "strum",
  "strum_macros",
  "surf",
@@ -1144,7 +1144,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.71.0"
+version = "1.72.0"
 dependencies = [
  "anyhow",
  "async-std",
@@ -2122,9 +2122,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.113"
+version = "0.2.114"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9"
+checksum = "b0005d08a8f7b65fb8073cb697aa0b12b631ed251ce73d862ce50eeb52ce3b50"
 
 [[package]]
 name = "libm"
@@ -2185,12 +2185,12 @@ dependencies = [
 
 [[package]]
 name = "mailparse"
-version = "0.13.7"
+version = "0.13.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d70ae0840b192a2f7d1dc46e75f38720a7e3c52dfdc968ba3202fa270668dc67"
+checksum = "8cae768a50835557749599277fc59f7c728118724eb34185e8feb633ef266a32"
 dependencies = [
- "base64 0.13.0",
  "charset",
+ "data-encoding",
  "quoted_printable",
 ]
 
@@ -3301,9 +3301,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "serde"
-version = "1.0.135"
+version = "1.0.136"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cf9235533494ea2ddcdb794665461814781c53f19d87b76e571a1c35acbad2b"
+checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
 dependencies = [
  "serde_derive",
 ]
@@ -3320,9 +3320,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.135"
+version = "1.0.136"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8dcde03d87d4c973c04be249e7d8f0b35db1c848c487bd43032808e59dd8328d"
+checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3610,9 +3610,9 @@ dependencies = [
 
 [[package]]
 name = "stop-token"
-version = "0.6.1"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3829f0fc642e19ef8a1072ba8e33139a385f609ba56761df501c5272a9704918"
+checksum = "af91f480ee899ab2d9f8435bfdfc14d08a5754bd9d3fef1f1a1c23336aad6c8b"
 dependencies = [
  "async-channel",
  "cfg-if 1.0.0",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.71.0"
+version = "1.72.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -30,7 +30,6 @@ async-trait = "0.1"
 backtrace = "0.3"
 base64 = "0.13"
 bitflags = "1.3"
-byteorder = "1.3"
 chrono = "0.4"
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
@@ -39,6 +38,7 @@ escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+imap-proto = "0.14.3"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -66,7 +66,7 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-stop-token = "0.6"
+stop-token = "0.7"
 strum = "0.23"
 strum_macros = "0.23"
 surf = { version = "2.3", default-features = false, features = ["h1-client"] }

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.71.0"
+version = "1.72.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -335,8 +335,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    1=send a copy of outgoing messages to self.
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
- * - `sentbox_watch`= 1=watch `Sent`-folder for changes (default),
- *                    0=do not watch the `Sent`-folder,
+ * - `sentbox_watch`= 1=watch `Sent`-folder for changes,
+ *                    0=do not watch the `Sent`-folder (default),
  *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `mvbox_move`   = 1=detect chat messages,
  *                    move them to the `DeltaChat` folder,

draft/webxdc-dev-reference.md

@@ -68,38 +68,46 @@ The callback is called for updates sent by you or other peers.
 
 ### getAllUpdates()
 
-```
-payloads = window.webxdc.getAllUpdates()
+```js
+updates = await window.webxdc.getAllUpdates();
 ```
 
 In case your Webxdc was just started,
 you may want to reconstruct the state from the last run -
 and also incorporate updates that may have arrived while the app was not running.
 
-- `payloads`: the function returns all previous updates in an array, 
+- `updates`: All previous updates in an array, 
   eg. `[{payload: "foo"},{payload: "bar"}]`
-  if `webxdc.sendUpdate("foo"); webxdc.sendUpdate("bar");` was called on the last run.
+  if `webxdc.sendUpdate({payload: "foo"}); webxdc.sendUpdate({payload: "bar"};` was called on the last run.
 
-
-### selfAddr()
+The updates are wrapped into a Promise that you can `await` for.
+If you are not in an async function and cannot use `await` therefore,
+you can get the updates with `then()`:
 
 ```js
-addr = window.webxdc.selfAddr()
+window.webxdc.getAllUpdates().then(updates => {});
 ```
 
-Returns the peer's own address.
+
+### selfAddr
+
+```js
+window.webxdc.selfAddr
+```
+
+Property with the peer's own address.
 This is esp. useful if you want to differ between different peers -
 just send the address along with the payload,
 and, if needed, compare the payload addresses against selfAddr() later on.
 
 
-### selfName()
+### selfName
 
 ```js
-addr = window.webxdc.selfName()
+window.webxdc.selfName
 ```
 
-Returns the peer's own name.
+Property with the peer's own name.
 This is name chosen by the user in their settings,
 if there is nothing set, that defaults to the peer's address.
 
@@ -128,7 +136,7 @@ round corners etc. will be added by the implementations as needed.
 If no icon is set, a default icon will be used.
 
 
-## Webxdc Example
+## Webxdc Examples
 
 The following example shows an input field and  every input is show on all peers.
 
@@ -155,14 +163,26 @@ The following example shows an input field and  every input is show on all peers
       }
     
       window.webxdc.setUpdateListener(receiveUpdate);
-      window.webxdc.getAllUpdates().forEach(receiveUpdate);
+      window.webxdc.getAllUpdates().then(updates => updates.forEach(receiveUpdate));
 
     </script>
   </body>
 </html>
 ```
 
-For a more advanved example, see https://github.com/r10s/webxdc-poll/ .
+[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
+offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
+You can also use that repository as a template for your own app -
+just clone and start adapting things to your need.
+
+
+### Advanced Examples
+
+- [2048](https://github.com/adbenitez/2048.xdc)
+- [Draw](https://github.com/adbenitez/draw.xdc)
+- [Poll](https://github.com/r10s/webxdc-poll/)
+- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
+- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc)
 
 
 ## Closing Remarks

src/config.rs

@@ -65,7 +65,7 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MdnsEnabled,
 
-    #[strum(props(default = "1"))]
+    #[strum(props(default = "0"))]
     SentboxWatch,
 
     #[strum(props(default = "1"))]
@@ -309,7 +309,7 @@ impl Context {
     }
 
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { None })
+        self.set_config(key, if value { Some("1") } else { Some("0") })
             .await?;
         Ok(())
     }
@@ -401,4 +401,17 @@ mod tests {
 
         Ok(())
     }
+
+    /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
+    #[async_std::test]
+    async fn test_set_config_bool() -> Result<()> {
+        let t = TestContext::new().await;
+
+        // We need some config that defaults to true
+        let c = Config::E2eeEnabled;
+        assert_eq!(t.get_config_bool(c).await?, true);
+        t.set_config_bool(c, false).await?;
+        assert_eq!(t.get_config_bool(c).await?, false);
+        Ok(())
+    }
 }

src/dc_tools.rs

@@ -195,44 +195,26 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time
 }
 
 /* Message-ID tools */
+
+/// Generate an ID. The generated ID should be as short and as unique as possible:
+/// - short, because it may also used as part of Message-ID headers or in QR codes
+/// - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
+/// IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
+///
+/// Additional information when used as a message-id or group-id:
+/// - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
+/// - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
+/// - the group-id should be a string with the characters [a-zA-Z0-9\-_]
 pub(crate) fn dc_create_id() -> String {
-    /* generate an id. the generated ID should be as short and as unique as possible:
-    - short, because it may also used as part of Message-ID headers or in QR codes
-    - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
-    IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
-    If possible, RNG of OpenSSL is used.
-
-    Additional information when used as a message-id or group-id:
-    - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
-    - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
-    - the group-id should be a string with the characters [a-zA-Z0-9\-_] */
-
+    // ThreadRng implements CryptoRng trait and is supposed to be cryptographically secure.
     let mut rng = thread_rng();
-    let buf: [u32; 3] = [rng.gen(), rng.gen(), rng.gen()];
 
-    encode_66bits_as_base64(buf[0usize], buf[1usize], buf[2usize])
-}
+    // Generate 72 random bits.
+    let mut arr = [0u8; 9];
+    rng.fill(&mut arr[..]);
 
-/// Encode 66 bits as a base64 string.
-/// This is useful for ID generating with short strings as we save 5 character
-/// in each id compared to 64 bit hex encoding. For a typical group ID, these
-/// are 10 characters (grpid+msgid):
-///    hex:    64 bit, 4 bits/character, length = 64/4 = 16 characters
-///    base64: 64 bit, 6 bits/character, length = 64/6 = 11 characters (plus 2 additional bits)
-/// Only the lower 2 bits of `fill` are used.
-fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
-    use byteorder::{BigEndian, WriteBytesExt};
-
-    let mut wrapped_writer = Vec::new();
-    {
-        let mut enc = base64::write::EncoderWriter::new(&mut wrapped_writer, base64::URL_SAFE);
-        enc.write_u32::<BigEndian>(v1).unwrap();
-        enc.write_u32::<BigEndian>(v2).unwrap();
-        enc.write_u8(((fill & 0x3) as u8) << 6).unwrap();
-        enc.finish().unwrap();
-    }
-    assert_eq!(wrapped_writer.pop(), Some(b'A')); // Remove last "A"
-    String::from_utf8(wrapped_writer).unwrap()
+    // Take 11 base64 characters containing 66 random bits.
+    base64::encode(&arr).chars().take(11).collect()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -780,26 +762,6 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         assert_eq!(buf.len(), 11);
     }
 
-    #[test]
-    fn test_encode_66bits_as_base64() {
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 0),
-            "ASNFZ4mrze8"
-        );
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 1),
-            "ASNFZ4mrze9"
-        );
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 2),
-            "ASNFZ4mrze-"
-        );
-        assert_eq!(
-            encode_66bits_as_base64(0x01234567, 0x89abcdef, 3),
-            "ASNFZ4mrze_"
-        );
-    }
-
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid

src/imap.rs

@@ -28,7 +28,6 @@ use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf_inner, from_field_to_contact_id, get_prefetch_parent_message, ReceivedMsg,
 };
-use crate::ephemeral::delete_expired_imap_messages;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
@@ -459,9 +458,6 @@ impl Imap {
             .await
             .context("fetch_new_messages")?;
 
-        // Mark expired messages for deletion.
-        delete_expired_imap_messages(context).await?;
-
         self.move_messages(context, watch_folder)
             .await
             .context("move_messages")?;
@@ -1013,7 +1009,9 @@ impl Imap {
             return Ok(());
         }
 
-        self.select_folder(context, Some(folder)).await?;
+        self.select_folder(context, Some(folder))
+            .await
+            .context("failed to select folder")?;
         let session = self
             .session
             .as_mut()
@@ -1025,77 +1023,74 @@ impl Imap {
             .as_ref()
             .with_context(|| format!("No mailbox selected, folder: {}", folder))?;
 
-        // Check if the mailbox supports MODSEQ.
-        // We are not interested in actual value of HIGHESTMODSEQ.
-        if mailbox.highest_modseq.is_none() {
+        let remote_highest_modseq = if let Some(remote_highest_modseq) = mailbox.highest_modseq {
+            remote_highest_modseq
+        } else {
             info!(
                 context,
                 "Mailbox {} does not support mod-sequences, skipping flag synchronization.", folder
             );
             return Ok(());
+        };
+
+        let mut highest_modseq = get_modseq(context, folder)
+            .await
+            .with_context(|| format!("failed to get MODSEQ for folder {}", folder))?;
+        if highest_modseq >= remote_highest_modseq {
+            info!(
+                context,
+                "MODSEQ {} is already new, HIGHESTMODSEQ={}, skipping seen flag update",
+                highest_modseq,
+                remote_highest_modseq
+            );
+            return Ok(());
         }
 
         let mut updated_chat_ids = BTreeSet::new();
-        let uid_validity = get_uidvalidity(context, folder).await?;
-        let mut highest_modseq = get_modseq(context, folder).await?;
+        let uid_validity = get_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to get UID validity for folder {}", folder))?;
         let mut list = session
             .uid_fetch("1:*", format!("(FLAGS) (CHANGEDSINCE {})", highest_modseq))
             .await
             .context("failed to fetch flags")?;
 
         while let Some(fetch) = list.next().await {
-            let msg = fetch?;
-
-            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
+            let fetch = fetch.context("failed to get FETCH result")?;
+            let uid = if let Some(uid) = fetch.uid {
+                uid
+            } else {
+                info!(context, "FETCH result contains no UID, skipping");
+                continue;
+            };
+            let is_seen = fetch.flags().any(|flag| flag == Flag::Seen);
             if is_seen {
-                if let Some((msg_id, chat_id)) = context
-                    .sql
-                    .query_row_optional(
-                        "SELECT id, chat_id FROM msgs
-                         WHERE rfc724_mid IN (
-                           SELECT rfc724_mid FROM imap
-                           WHERE folder=?1
-                           AND uidvalidity=?2
-                           AND uid=?3
-                           LIMIT 1
-                         )",
-                        paramsv![&folder, uid_validity, msg.uid],
-                        |row| {
-                            let msg_id: MsgId = row.get(0)?;
-                            let chat_id: ChatId = row.get(1)?;
-                            Ok((msg_id, chat_id))
-                        },
-                    )
-                    .await?
+                if let Some(chat_id) = mark_seen_by_uid(context, folder, uid_validity, uid)
+                    .await
+                    .with_context(|| {
+                        format!("failed to update seen status for msg {}/{}", folder, uid)
+                    })?
                 {
-                    let updated = context
-                        .sql
-                        .execute(
-                            "UPDATE msgs SET state=?1
-                             WHERE (state=?2 OR state=?3)
-                             AND id=?4",
-                            paramsv![
-                                MessageState::InSeen,
-                                MessageState::InFresh,
-                                MessageState::InNoticed,
-                                msg_id
-                            ],
-                        )
-                        .await?
-                        > 0;
-
-                    if updated {
-                        updated_chat_ids.insert(chat_id);
-                        let modseq = msg.modseq.unwrap_or_default();
-                        if modseq > highest_modseq {
-                            highest_modseq = modseq;
-                        }
-                    }
+                    updated_chat_ids.insert(chat_id);
                 }
             }
+
+            if let Some(modseq) = fetch.modseq {
+                if modseq > highest_modseq {
+                    highest_modseq = modseq;
+                }
+            } else {
+                warn!(context, "FETCH result contains no MODSEQ");
+            }
         }
 
-        set_modseq(context, folder, highest_modseq).await?;
+        if remote_highest_modseq > highest_modseq {
+            // We haven't seen the message with the highest MODSEQ, maybe it was deleted already.
+            highest_modseq = remote_highest_modseq;
+        }
+        set_modseq(context, folder, highest_modseq)
+            .await
+            .with_context(|| format!("failed to set MODSEQ for folder {}", folder))?;
         for updated_chat_id in updated_chat_ids {
             context.emit_event(EventType::MsgsNoticed(updated_chat_id));
         }
@@ -1589,6 +1584,23 @@ impl Imap {
         Ok(())
     }
 
+    /// Update HIGHESTMODSEQ on selected mailbox.
+    ///
+    /// Should be called when MODSEQ is seen on the response, such as IDLE response.
+    pub(crate) fn update_modseq(&mut self, modseq: u64) {
+        self.config.selected_mailbox =
+            self.config
+                .selected_mailbox
+                .as_ref()
+                .map(|mailbox| Mailbox {
+                    highest_modseq: Some(std::cmp::max(
+                        mailbox.highest_modseq.unwrap_or_default(),
+                        modseq,
+                    )),
+                    ..mailbox.clone()
+                });
+    }
+
     /// Return whether the server sent an unsolicited EXISTS response.
     /// Drains all responses from `session.unsolicited_responses` in the process.
     /// If this returns `true`, this means that new emails arrived and you should
@@ -1980,6 +1992,70 @@ fn get_fallback_folder(delimiter: &str) -> String {
     format!("INBOX{}DeltaChat", delimiter)
 }
 
+/// Marks messages in `msgs` table as seen, searching for them by UID.
+///
+/// Returns updated chat ID if any message was marked as seen.
+async fn mark_seen_by_uid(
+    context: &Context,
+    folder: &str,
+    uid_validity: u32,
+    uid: u32,
+) -> Result<Option<ChatId>> {
+    if let Some((msg_id, chat_id)) = context
+        .sql
+        .query_row_optional(
+            "SELECT id, chat_id FROM msgs
+                 WHERE rfc724_mid IN (
+                   SELECT rfc724_mid FROM imap
+                   WHERE folder=?1
+                   AND uidvalidity=?2
+                   AND uid=?3
+                   LIMIT 1
+                 )",
+            paramsv![&folder, uid_validity, uid],
+            |row| {
+                let msg_id: MsgId = row.get(0)?;
+                let chat_id: ChatId = row.get(1)?;
+                Ok((msg_id, chat_id))
+            },
+        )
+        .await
+        .with_context(|| {
+            format!(
+                "failed to get msg and chat ID for IMAP message {}/{}",
+                folder, uid
+            )
+        })?
+    {
+        let updated = context
+            .sql
+            .execute(
+                "UPDATE msgs SET state=?1
+                     WHERE (state=?2 OR state=?3)
+                     AND id=?4",
+                paramsv![
+                    MessageState::InSeen,
+                    MessageState::InFresh,
+                    MessageState::InNoticed,
+                    msg_id
+                ],
+            )
+            .await
+            .with_context(|| format!("failed to update msg {} state", msg_id))?
+            > 0;
+
+        if updated {
+            Ok(Some(chat_id))
+        } else {
+            // Message state has not chnaged.
+            Ok(None)
+        }
+    } else {
+        // There is no message is `msgs` table matchng the given UID.
+        Ok(None)
+    }
+}
+
 /// uid_next is the next unique identifier value from the last time we fetched a folder
 /// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
 /// This function is used to update our uid_next after fetching messages.

src/imap/idle.rs

@@ -3,6 +3,7 @@ use super::Imap;
 use anyhow::{bail, Context as _, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
+use imap_proto::types::{AttributeValue, Response};
 use std::time::{Duration, SystemTime};
 
 use crate::{context::Context, scheduler::InterruptInfo};
@@ -71,6 +72,13 @@ impl Imap {
             match fut.await {
                 Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
                     info!(context, "Idle has NewData {:?}", x);
+                    if let Response::Fetch(_message, attrs) = x.parsed() {
+                        for attr in attrs {
+                            if let AttributeValue::ModSeq(modseq) = attr {
+                                self.update_modseq(*modseq);
+                            }
+                        }
+                    }
                 }
                 Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
                     info!(context, "Idle-wait timeout or interruption");

src/imex.rs

@@ -423,7 +423,8 @@ async fn imex_inner(
             export_backup(context, path, passphrase.unwrap_or_default()).await
         }
         ImexMode::ImportBackup => {
-            import_backup(context, path, passphrase.unwrap_or_default()).await
+            import_backup(context, path, passphrase.unwrap_or_default()).await?;
+            context.sql.run_migrations(context).await
         }
     }
 }

src/message.rs

@@ -180,7 +180,7 @@ impl rusqlite::types::ToSql for MsgId {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         if self.0 <= DC_MSG_ID_LAST_SPECIAL {
             return Err(rusqlite::Error::ToSqlConversionFailure(
-                format_err!("Invalid MsgId").into(),
+                format_err!("Invalid MsgId {}", self.0).into(),
             ));
         }
         let val = rusqlite::types::Value::Integer(self.0 as i64);

src/mimeparser.rs

@@ -403,16 +403,18 @@ impl MimeMessage {
     #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
-            let need_drop = {
-                textpart.typ == Viewtype::Text
-                    && (filepart.typ == Viewtype::Image
-                        || filepart.typ == Viewtype::Gif
-                        || filepart.typ == Viewtype::Sticker
-                        || filepart.typ == Viewtype::Audio
-                        || filepart.typ == Viewtype::Voice
-                        || filepart.typ == Viewtype::Video
-                        || filepart.typ == Viewtype::File)
-            };
+            let need_drop = textpart.typ == Viewtype::Text
+                && match filepart.typ {
+                    Viewtype::Image
+                    | Viewtype::Gif
+                    | Viewtype::Sticker
+                    | Viewtype::Audio
+                    | Viewtype::Voice
+                    | Viewtype::Video
+                    | Viewtype::File
+                    | Viewtype::Webxdc => true,
+                    Viewtype::Unknown | Viewtype::Text | Viewtype::VideochatInvitation => false,
+                };
 
             if need_drop {
                 let mut filepart = self.parts.swap_remove(1);

src/scheduler.rs

@@ -1,4 +1,4 @@
-use anyhow::{bail, Result};
+use anyhow::{bail, Context as _, Result};
 use async_std::prelude::*;
 use async_std::{
     channel::{self, Receiver, Sender},
@@ -8,6 +8,7 @@ use async_std::{
 use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
+use crate::ephemeral::delete_expired_imap_messages;
 use crate::imap::Imap;
 use crate::job::{self, Thread};
 use crate::smtp::{send_smtp_messages, Smtp};
@@ -160,6 +161,14 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
+            // Mark expired messages for deletion.
+            if let Err(err) = delete_expired_imap_messages(ctx)
+                .await
+                .context("delete_expired_imap_messages failed")
+            {
+                warn!(ctx, "{:#}", err);
+            }
+
             // Scan other folders before fetching from watched folder. This may result in the
             // messages being moved into the watched folder, for example from the Spam folder to
             // the Inbox folder.

src/smtp.rs

@@ -4,10 +4,10 @@ pub mod send;
 
 use std::time::{Duration, SystemTime};
 
-use anyhow::{format_err, Context as _};
+use anyhow::{bail, format_err, Context as _, Result};
 use async_smtp::smtp::client::net::ClientTlsParameters;
 use async_smtp::smtp::response::{Category, Code, Detail};
-use async_smtp::{error, smtp, EmailAddress, ServerAddress};
+use async_smtp::{smtp, EmailAddress, ServerAddress};
 
 use crate::constants::DC_LP_AUTH_OAUTH2;
 use crate::events::EventType;
@@ -23,28 +23,6 @@ use crate::{context::Context, scheduler::connectivity::ConnectivityStore};
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
 
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("Bad parameters")]
-    BadParameters,
-    #[error("Invalid login address {address}: {error}")]
-    InvalidLoginAddress {
-        address: String,
-        #[source]
-        error: error::Error,
-    },
-    #[error("SMTP failed to connect: {0}")]
-    ConnectionFailure(#[source] smtp::error::Error),
-    #[error("SMTP oauth2 error {address}")]
-    Oauth2 { address: String },
-    #[error("TLS error {0}")]
-    Tls(#[from] async_native_tls::Error),
-    #[error("{0}")]
-    Other(#[from] anyhow::Error),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
 #[derive(Default)]
 pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
@@ -135,14 +113,11 @@ impl Smtp {
         }
 
         if lp.server.is_empty() || lp.port == 0 {
-            return Err(Error::BadParameters);
+            bail!("bad connection parameters");
         }
 
-        let from =
-            EmailAddress::new(addr.to_string()).map_err(|err| Error::InvalidLoginAddress {
-                address: addr.to_string(),
-                error: err,
-            })?;
+        let from = EmailAddress::new(addr.to_string())
+            .with_context(|| format!("invalid login address {}", addr))?;
 
         self.from = Some(from);
 
@@ -163,9 +138,7 @@ impl Smtp {
             let send_pw = &lp.password;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false).await?;
             if access_token.is_none() {
-                return Err(Error::Oauth2 {
-                    address: addr.to_string(),
-                });
+                bail!("SMTP OAuth 2 error {}", addr);
             }
             let user = &lp.user;
             (
@@ -209,9 +182,7 @@ impl Smtp {
         }
 
         let mut trans = client.into_transport();
-        if let Err(err) = trans.connect().await {
-            return Err(Error::ConnectionFailure(err));
-        }
+        trans.connect().await.context("SMTP failed to connect")?;
 
         self.transport = Some(trans);
         self.last_success = Some(SystemTime::now());
@@ -398,6 +369,19 @@ pub(crate) async fn send_msg_to_smtp(
         )
         .collect::<Vec<_>>();
 
+    // If there is a msg-id and it does not exist in the db, cancel sending. this happens if
+    // dc_delete_msgs() was called before the generated mime was sent out.
+    if !message::exists(context, msg_id)
+        .await
+        .with_context(|| format!("failed to check message {} existence", msg_id))?
+    {
+        info!(
+            context,
+            "Sending of message {} was cancelled by the user.", msg_id
+        );
+        return Ok(());
+    }
+
     let status = match smtp_send(
         context,
         &recipients_list,

src/sql.rs

@@ -215,6 +215,12 @@ impl Sql {
             conn.pragma_update(None, "synchronous", &"NORMAL".to_string())?;
         }
 
+        self.run_migrations(context).await?;
+
+        Ok(())
+    }
+
+    pub async fn run_migrations(&self, context: &Context) -> Result<()> {
         // (1) update low-level database structure.
         // this should be done before updates that use high-level objects that
         // rely themselves on the low-level structure.

src/summary.rs

@@ -139,7 +139,10 @@ impl Message {
             }
             Viewtype::Webxdc => {
                 append_text = true;
-                "Webxdc".to_string()
+                self.get_webxdc_info(context)
+                    .await
+                    .map(|info| info.name)
+                    .unwrap_or_else(|_| "ErrWebxdcName".to_string())
             }
             Viewtype::Text | Viewtype::Unknown => {
                 if self.param.get_cmd() != SystemMessage::LocationOnly {

src/webxdc.rs

@@ -488,6 +488,7 @@ mod tests {
         add_contact_to_chat, create_group_chat, forward_msgs, send_msg, send_text_msg, ChatId,
         ProtectionStatus,
     };
+    use crate::chatlist::Chatlist;
     use crate::contact::Contact;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
@@ -1514,4 +1515,63 @@ sth_for_the = "future""#
 
         Ok(())
     }
+
+    #[async_std::test]
+    async fn test_webxdc_chatlist_summary() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "chat").await?;
+        let mut instance = create_webxdc_instance(
+            &t,
+            "with-minimal-manifest.xdc",
+            include_bytes!("../test-data/webxdc/with-minimal-manifest.xdc"),
+        )
+        .await?;
+        send_msg(&t, chat_id, &mut instance).await?;
+
+        let chatlist = Chatlist::try_load(&t, 0, None, None).await?;
+        assert_eq!(chatlist.len(), 1);
+        let summary = chatlist.get_summary(&t, 0, None).await?;
+        assert_eq!(summary.text, "nice app!".to_string());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_webxdc_and_text() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Alice sends instance and adds some text
+        let alice_chat = alice.create_chat(&bob).await;
+        let mut alice_instance = create_webxdc_instance(
+            &alice,
+            "minimal.xdc",
+            include_bytes!("../test-data/webxdc/minimal.xdc"),
+        )
+        .await?;
+        alice_instance.set_text(Some("user added text".to_string()));
+        send_msg(&alice, alice_chat.id, &mut alice_instance).await?;
+        let alice_instance = alice.get_last_msg().await;
+        assert_eq!(
+            alice_instance.get_text(),
+            Some("user added text".to_string())
+        );
+
+        // Bob receives that instance
+        let sent1 = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent1).await;
+        let bob_instance = bob.get_last_msg().await;
+        assert_eq!(bob_instance.get_text(), Some("user added text".to_string()));
+
+        // Alice's second device receives the instance as well
+        let alice2 = TestContext::new_alice().await;
+        alice2.recv_msg(&sent1).await;
+        let alice2_instance = alice2.get_last_msg().await;
+        assert_eq!(
+            alice2_instance.get_text(),
+            Some("user added text".to_string())
+        );
+
+        Ok(())
+    }
 }