Release 1.102.0 (#3773)

Update webxdc-dev-reference.md

CHANGELOG.md

@@ -2,11 +2,42 @@
 
 ## Unreleased
 
+### Changes
+
 ### API-Changes
 
+### Fixes
+
+
+## 1.102.0
+
 ### Changes
 
+- If an email has multiple From addresses, handle this as if there was
+  no From address, to prevent from forgery attacks. Also, improve
+  handling of emails with invalid From addresses in general #3667
+
+### API-Changes
+
 ### Fixes
+- fix detection of "All mail", "Trash", "Junk" etc folders. #3760
+- fetch messages sequentially to fix reactions on partially downloaded messages #3688
+- Fix a bug where one malformed message blocked receiving any further messages #3769
+
+
+## 1.101.0
+
+### Changes
+- add `configured_inbox_folder` to account info #3748
+- `dc_delete_contact()` hides contacts if referenced #3751
+- add IMAP UIDs to message info #3755
+
+### Fixes
+- improve IMAP logging, in particular fix incorrect "IMAP IDLE protocol
+  timed out" message on network error during IDLE #3749
+- pop Recently Seen Loop event out of the queue when it is in the past
+  to avoid busy looping #3753
+- fix build failures by going back to standard `async_zip` #3747
 
 
 ## 1.100.0

Cargo.lock

@@ -109,9 +109,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.3.14"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "345fd392ab01f746c717b1357165b76f0b67a60192007b234058c9045fdcf695"
+checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a"
 dependencies = [
  "flate2",
  "futures-core",
@@ -197,16 +197,18 @@ dependencies = [
 
 [[package]]
 name = "async_io_utilities"
-version = "0.1.3"
-source = "git+https://github.com/Majored/rs-async-io-utilities#6661a0fb914ccc1f97a6acd446bc03e9547d64fc"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b20cffc5590f4bf33f05f97a3ea587feba9c50d20325b401daa096b92ff7da0"
 dependencies = [
  "tokio",
 ]
 
 [[package]]
 name = "async_zip"
-version = "0.0.8"
-source = "git+https://github.com/dignifiedquire/rs-async-zip?branch=main#5556c5ac5e0bbc89e2d440291a9a6b77c74070aa"
+version = "0.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a36d43bdefc7215b2b3a97edd03b1553b7969ad76551025eedd3b913c645f6e"
 dependencies = [
  "async-compression",
  "async_io_utilities",
@@ -824,7 +826,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.100.0"
+version = "1.102.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -896,7 +898,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.100.0"
+version = "1.102.0"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -918,7 +920,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.100.0"
+version = "1.102.0"
 dependencies = [
  "anyhow",
  "deltachat-jsonrpc",
@@ -941,7 +943,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.100.0"
+version = "1.102.0"
 dependencies = [
  "anyhow",
  "deltachat",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.100.0"
+version = "1.102.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -82,7 +82,7 @@ async-channel = "1.6.1"
 futures-lite = "1.12.0"
 tokio-stream = { version = "0.1.11", features = ["fs"] }
 reqwest = { version = "0.11.12", features = ["json"] }
-async_zip = { git = "https://github.com/dignifiedquire/rs-async-zip", branch = "main", default-features = false, features = ["deflate"] }
+async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.100.0"
+version = "1.102.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -2056,8 +2056,9 @@ char*           dc_get_contact_encrinfo      (dc_context_t* context, uint32_t co
 
 
 /**
- * Delete a contact. The contact is deleted from the local device. It may happen that this is not
- * possible as the contact is in use. In this case, the contact can be blocked.
+ * Delete a contact so that it disappears from the corresponding lists.
+ * Depending on whether there are ongoing chats, deletion is done by physical deletion or hiding.
+ * The contact is deleted from the local device.
  *
  * May result in a #DC_EVENT_CONTACTS_CHANGED event.
  *

deltachat-ffi/src/lib.rs

@@ -2144,7 +2144,10 @@ pub unsafe extern "C" fn dc_delete_contact(
     block_on(async move {
         match Contact::delete(ctx, contact_id).await {
             Ok(_) => 1,
-            Err(_) => 0,
+            Err(err) => {
+                error!(ctx, "cannot delete contact: {}", err);
+                0
+            }
         }
     })
 }

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.100.0"
+version = "1.102.0"
 description = "DeltaChat JSON-RPC API"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"

deltachat-jsonrpc/src/api/mod.rs

@@ -1146,12 +1146,12 @@ impl CommandApi {
         Ok(contacts)
     }
 
-    async fn delete_contact(&self, account_id: u32, contact_id: u32) -> Result<bool> {
+    async fn delete_contact(&self, account_id: u32, contact_id: u32) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
         let contact_id = ContactId::new(contact_id);
 
         Contact::delete(&ctx, contact_id).await?;
-        Ok(true)
+        Ok(())
     }
 
     async fn change_contact_name(

deltachat-jsonrpc/typescript/package.json

@@ -48,5 +48,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.100.0"
+  "version": "1.102.0"
 }

deltachat-jsonrpc/typescript/src/client.ts

@@ -43,12 +43,14 @@ export class BaseDeltaChat<
       const method = request.method;
       if (method === "event") {
         const event = request.params! as DCWireEvent<Event>;
+	//@ts-ignore
         this.emit(event.event.type, event.contextId, event.event as any);
         this.emit("ALL", event.contextId, event.event as any);
 
         if (this.contextEmitters[event.contextId]) {
           this.contextEmitters[event.contextId].emit(
             event.event.type,
+            //@ts-ignore
             event.event as any
           );
           this.contextEmitters[event.contextId].emit("ALL", event.event);

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.100.0"
+version = "1.102.0"
 description = "DeltaChat JSON-RPC server"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"

draft/webxdc-dev-reference.md

@@ -4,7 +4,7 @@ This document gives a quick overview about the Webxdc specification,
 It is meant for both, developing Webxdc apps
 and developing Webxdc implementations.
 
-The [Webxdc guidebook](https://deltachat.github.io/webxdc_docs/) shows more detailed information
+The [Webxdc guidebook](https://docs.webxdc.org/) shows more detailed information
 when developing Webxdc apps.
 
 

node/test/test.js

@@ -616,7 +616,7 @@ describe('Offline Tests with unconfigured account', function () {
     const id = context.createContact('someuser', 'someuser@site.com')
     const contact = context.getContact(id)
     strictEqual(contact.getId(), id, 'contact id matches')
-    strictEqual(context.deleteContact(id), true, 'delete call succesful')
+    context.deleteContact(id)
     strictEqual(context.getContact(id), null, 'contact is gone')
   })
 

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.100.0"
+  "version": "1.102.0"
 }

python/tests/test_1_online.py

@@ -901,6 +901,34 @@ def test_dont_show_emails(acfactory, lp):
             ac1.get_config("configured_addr")
         ),
     )
+    ac1.direct_imap.append(
+        "Spam",
+        """
+        From: unknown.address@junk.org, unkwnown.add@junk.org
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message2@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Unknown & malformed message in Spam
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Spam",
+        """
+        From: alice@example.org
+        Subject: subj
+        To: {}
+        Message-ID: <spam.message3@junk.org>
+        Content-Type: text/plain; charset=utf-8
+
+        Actually interesting message in Spam
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
     ac1.direct_imap.append(
         "Junk",
         """
@@ -926,7 +954,9 @@ def test_dont_show_emails(acfactory, lp):
     ac1._evtracker.wait_idle_inbox_ready()
 
     assert msg.text == "subj – message in Sent"
-    assert len(msg.chat.get_messages()) == 1
+    chat_msgs = msg.chat.get_messages()
+    assert len(chat_msgs) == 2
+    assert any(msg.text == "subj – Actually interesting message in Spam" for msg in chat_msgs)
 
     assert not any("unknown.address" in c.get_name() for c in ac1.get_chats())
     ac1.direct_imap.select_folder("Spam")
@@ -942,7 +972,7 @@ def test_dont_show_emails(acfactory, lp):
     msg2 = ac1._evtracker.wait_next_messages_changed()
 
     assert msg2.text == "subj – message in Drafts that is moved to Sent later"
-    assert len(msg.chat.get_messages()) == 2
+    assert len(msg.chat.get_messages()) == 3
 
 
 def test_no_old_msg_is_fresh(acfactory, lp):

python/tests/test_3_offline.py

@@ -200,11 +200,11 @@ class TestOfflineContact:
         assert ac1.delete_contact(contact1)
         assert contact1 not in ac1.get_contacts()
 
-    def test_get_contacts_and_delete_fails(self, acfactory):
+    def test_delete_referenced_contact_hides_contact(self, acfactory):
         ac1 = acfactory.get_pseudo_configured_account()
         contact1 = ac1.create_contact("some1@example.com", name="some1")
         msg = contact1.create_chat().send_text("one message")
-        assert not ac1.delete_contact(contact1)
+        assert ac1.delete_contact(contact1)
         assert not msg.filemime
 
     def test_create_chat_flexibility(self, acfactory):

src/authres.rs

@@ -13,6 +13,8 @@ use once_cell::sync::Lazy;
 use crate::config::Config;
 use crate::context::Context;
 use crate::headerdef::HeaderDef;
+use crate::mimeparser;
+use crate::mimeparser::ParserErrorExt;
 use crate::tools::time;
 use crate::tools::EmailAddress;
 
@@ -30,20 +32,23 @@ pub(crate) async fn handle_authres(
     mail: &ParsedMail<'_>,
     from: &str,
     message_time: i64,
-) -> Result<DkimResults> {
+) -> mimeparser::ParserResult<DkimResults> {
     let from_domain = match EmailAddress::new(from) {
         Ok(email) => email.domain,
         Err(e) => {
-            warn!(context, "invalid email {:#}", e);
             // This email is invalid, but don't return an error, we still want to
             // add a stub to the database so that it's not downloaded again
-            return Ok(DkimResults::default());
+            return Err(anyhow::format_err!("invalid email {}: {:#}", from, e)).map_err_malformed();
         }
     };
 
     let authres = parse_authres_headers(&mail.get_headers(), &from_domain);
-    update_authservid_candidates(context, &authres).await?;
-    compute_dkim_results(context, authres, &from_domain, message_time).await
+    update_authservid_candidates(context, &authres)
+        .await
+        .map_err_sql()?;
+    compute_dkim_results(context, authres, &from_domain, message_time)
+        .await
+        .map_err_sql()
 }
 
 #[derive(Default, Debug)]
@@ -574,7 +579,7 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 file.read_to_end(&mut bytes).await.unwrap();
 
                 let mail = mailparse::parse_mail(&bytes)?;
-                let from = &mimeparser::get_from(&mail.headers)[0].addr;
+                let from = &mimeparser::get_from(&mail.headers).unwrap().addr;
 
                 let res = handle_authres(&t, &mail, from, time()).await?;
                 assert!(res.allow_keychange);
@@ -586,7 +591,7 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 file.read_to_end(&mut bytes).await.unwrap();
 
                 let mail = mailparse::parse_mail(&bytes)?;
-                let from = &mimeparser::get_from(&mail.headers)[0].addr;
+                let from = &mimeparser::get_from(&mail.headers).unwrap().addr;
 
                 let res = handle_authres(&t, &mail, from, time()).await?;
                 if !res.allow_keychange {
@@ -637,9 +642,10 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
         // Even if the format is wrong and parsing fails, handle_authres() shouldn't
         // return an Err because this would prevent the message from being added
         // to the database and downloaded again and again
-        let bytes = b"Authentication-Results: dkim=";
+        let bytes = b"From: invalid@from.com
+Authentication-Results: dkim=";
         let mail = mailparse::parse_mail(bytes).unwrap();
-        handle_authres(&t, &mail, "invalidfrom.com", time())
+        handle_authres(&t, &mail, "invalid@rom.com", time())
             .await
             .unwrap();
     }
@@ -681,7 +687,7 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
             .await;
 
         sent.payload
-            .insert_str(0, "Authentication-Results: example.org; dkim=fail");
+            .insert_str(0, "Authentication-Results: example.org; dkim=fail\n");
 
         let received = alice.recv_msg(&sent).await;
 
@@ -717,7 +723,7 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
         loop {
             if let Some(mut sent) = bob2.pop_sent_msg_opt(Duration::ZERO).await {
                 sent.payload
-                    .insert_str(0, "Authentication-Results: example.org; dkim=fail");
+                    .insert_str(0, "Authentication-Results: example.org; dkim=fail\n");
                 alice.recv_msg(&sent).await;
             } else if let Some(sent) = alice.pop_sent_msg_opt(Duration::ZERO).await {
                 bob2.recv_msg(&sent).await;

src/contact.rs

@@ -229,7 +229,7 @@ pub enum Origin {
     /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling contact_is_verified() !
     SecurejoinJoined = 0x0200_0000,
 
-    /// contact added mannually by create_contact(), this should be the largest origin as otherwise the user cannot modify the names
+    /// contact added manually by create_contact(), this should be the largest origin as otherwise the user cannot modify the names
     ManuallyCreated = 0x0400_0000,
 }
 
@@ -455,7 +455,7 @@ impl Contact {
     /// - "row_authname": name as authorized from a contact, set only through a From-header
     /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
     ///
-    /// Returns the contact_id and a `Modifier` value indicating if a modification occured.
+    /// Returns the contact_id and a `Modifier` value indicating if a modification occurred.
     pub(crate) async fn add_or_lookup(
         context: &Context,
         name: &str,
@@ -944,43 +944,36 @@ impl Contact {
         Ok(ret)
     }
 
-    /// Delete a contact. The contact is deleted from the local device. It may happen that this is not
-    /// possible as the contact is in use. In this case, the contact can be blocked.
+    /// Delete a contact so that it disappears from the corresponding lists.
+    /// Depending on whether there are ongoing chats, deletion is done by physical deletion or hiding.
+    /// The contact is deleted from the local device.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
     pub async fn delete(context: &Context, contact_id: ContactId) -> Result<()> {
         ensure!(!contact_id.is_special(), "Can not delete special contact");
 
-        let count_chats = context
+        context
             .sql
-            .count(
-                "SELECT COUNT(*) FROM chats_contacts WHERE contact_id=?;",
-                paramsv![contact_id],
-            )
+            .transaction(move |transaction| {
+                // make sure, the transaction starts with a write command and becomes EXCLUSIVE by that -
+                // upgrading later may be impossible by races.
+                let deleted_contacts = transaction.execute(
+                    "DELETE FROM contacts WHERE id=?
+                     AND (SELECT COUNT(*) FROM chats_contacts WHERE contact_id=?)=0;",
+                    paramsv![contact_id, contact_id],
+                )?;
+                if deleted_contacts == 0 {
+                    transaction.execute(
+                        "UPDATE contacts SET origin=? WHERE id=?;",
+                        paramsv![Origin::Hidden, contact_id],
+                    )?;
+                }
+                Ok(())
+            })
             .await?;
 
-        if count_chats == 0 {
-            match context
-                .sql
-                .execute("DELETE FROM contacts WHERE id=?;", paramsv![contact_id])
-                .await
-            {
-                Ok(_) => {
-                    context.emit_event(EventType::ContactsChanged(None));
-                    return Ok(());
-                }
-                Err(err) => {
-                    error!(context, "delete_contact {} failed ({})", contact_id, err);
-                    return Err(err);
-                }
-            }
-        }
-
-        info!(
-            context,
-            "could not delete contact {}, there are {} chats with it", contact_id, count_chats
-        );
-        bail!("Could not delete contact with ongoing chats");
+        context.emit_event(EventType::ContactsChanged(None));
+        Ok(())
     }
 
     /// Get a single contact object.  For a list, see eg. get_contacts().
@@ -1524,7 +1517,7 @@ impl RecentlySeenLoop {
             if let Ok(duration) = until.duration_since(now) {
                 info!(
                     context,
-                    "Recently seen loop waiting for {} or interupt",
+                    "Recently seen loop waiting for {} or interrupt",
                     duration_to_str(duration)
                 );
 
@@ -1550,6 +1543,17 @@ impl RecentlySeenLoop {
                         unseen_queue.push((Reverse(timestamp + SEEN_RECENTLY_SECONDS), contact_id));
                     }
                 }
+            } else {
+                info!(
+                    context,
+                    "Recently seen loop is not waiting, event is already due."
+                );
+
+                // Event is already in the past.
+                if let Some(contact_id) = contact_id {
+                    context.emit_event(EventType::ContactsChanged(Some(*contact_id)));
+                }
+                unseen_queue.pop();
             }
         }
     }
@@ -1727,7 +1731,7 @@ mod tests {
         );
         assert_eq!(Contact::add_address_book(&t, book).await.unwrap(), 4);
 
-        // check first added contact, this modifies authname beacuse it is empty
+        // check first added contact, this modifies authname because it is empty
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
                 .await
@@ -1935,21 +1939,70 @@ mod tests {
         // Create Bob contact
         let (contact_id, _) =
             Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated)
-                .await
-                .unwrap();
-
+                .await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.net")
             .await;
+        assert_eq!(
+            Contact::get_all(&alice, 0, Some("bob@example.net"))
+                .await?
+                .len(),
+            1
+        );
 
-        // Can't delete a contact with ongoing chats.
-        assert!(Contact::delete(&alice, contact_id).await.is_err());
+        // If a contact has ongoing chats, contact is only hidden on deletion
+        Contact::delete(&alice, contact_id).await?;
+        let contact = Contact::load_from_db(&alice, contact_id).await?;
+        assert_eq!(contact.origin, Origin::Hidden);
+        assert_eq!(
+            Contact::get_all(&alice, 0, Some("bob@example.net"))
+                .await?
+                .len(),
+            0
+        );
 
         // Delete chat.
         chat.get_id().delete(&alice).await?;
 
-        // Can delete contact now.
+        // Can delete contact physically now
         Contact::delete(&alice, contact_id).await?;
+        assert!(Contact::load_from_db(&alice, contact_id).await.is_err());
+        assert_eq!(
+            Contact::get_all(&alice, 0, Some("bob@example.net"))
+                .await?
+                .len(),
+            0
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_delete_and_recreate_contact() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        // test recreation after physical deletion
+        let contact_id1 = Contact::create(&t, "Foo", "foo@bar.de").await?;
+        assert_eq!(Contact::get_all(&t, 0, Some("foo@bar.de")).await?.len(), 1);
+        Contact::delete(&t, contact_id1).await?;
+        assert!(Contact::load_from_db(&t, contact_id1).await.is_err());
+        assert_eq!(Contact::get_all(&t, 0, Some("foo@bar.de")).await?.len(), 0);
+        let contact_id2 = Contact::create(&t, "Foo", "foo@bar.de").await?;
+        assert_ne!(contact_id2, contact_id1);
+        assert_eq!(Contact::get_all(&t, 0, Some("foo@bar.de")).await?.len(), 1);
+
+        // test recreation after hiding
+        t.create_chat_with_contact("Foo", "foo@bar.de").await;
+        Contact::delete(&t, contact_id2).await?;
+        let contact = Contact::load_from_db(&t, contact_id2).await?;
+        assert_eq!(contact.origin, Origin::Hidden);
+        assert_eq!(Contact::get_all(&t, 0, Some("foo@bar.de")).await?.len(), 0);
+
+        let contact_id3 = Contact::create(&t, "Foo", "foo@bar.de").await?;
+        let contact = Contact::load_from_db(&t, contact_id3).await?;
+        assert_eq!(contact.origin, Origin::ManuallyCreated);
+        assert_eq!(contact_id3, contact_id2);
+        assert_eq!(Contact::get_all(&t, 0, Some("foo@bar.de")).await?.len(), 1);
 
         Ok(())
     }

src/context.rs

@@ -570,6 +570,10 @@ impl Context {
             .await?
             .unwrap_or_default();
 
+        let configured_inbox_folder = self
+            .get_config(Config::ConfiguredInboxFolder)
+            .await?
+            .unwrap_or_else(|| "<unset>".to_string());
         let configured_sentbox_folder = self
             .get_config(Config::ConfiguredSentboxFolder)
             .await?
@@ -641,6 +645,7 @@ impl Context {
         res.insert("mvbox_move", mvbox_move.to_string());
         res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
         res.insert("folders_configured", folders_configured.to_string());
+        res.insert("configured_inbox_folder", configured_inbox_folder);
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());

src/decrypt.rs

@@ -4,7 +4,6 @@ use std::collections::HashSet;
 
 use anyhow::{Context as _, Result};
 use mailparse::ParsedMail;
-use mailparse::SingleInfo;
 
 use crate::aheader::Aheader;
 use crate::authres;
@@ -14,6 +13,7 @@ use crate::context::Context;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::Keyring;
 use crate::log::LogExt;
+use crate::mimeparser::{self, ParserErrorExt};
 use crate::peerstate::Peerstate;
 use crate::pgp;
 
@@ -56,18 +56,12 @@ pub async fn try_decrypt(
     .await
 }
 
-pub async fn prepare_decryption(
+pub(crate) async fn prepare_decryption(
     context: &Context,
     mail: &ParsedMail<'_>,
-    from: &[SingleInfo],
+    from: &str,
     message_time: i64,
-) -> Result<DecryptionInfo> {
-    let from = if let Some(f) = from.first() {
-        &f.addr
-    } else {
-        return Ok(DecryptionInfo::default());
-    };
-
+) -> mimeparser::ParserResult<DecryptionInfo> {
     let autocrypt_header = Aheader::from_headers(from, &mail.headers)
         .ok_or_log_msg(context, "Failed to parse Autocrypt header")
         .flatten();
@@ -82,7 +76,8 @@ pub async fn prepare_decryption(
         // Disallowing keychanges is disabled for now:
         true, // dkim_results.allow_keychange,
     )
-    .await?;
+    .await
+    .map_err_sql()?;
 
     Ok(DecryptionInfo {
         from: from.to_string(),

src/imap.rs

@@ -56,6 +56,8 @@ use session::Session;
 
 use self::select_folder::NewlySelected;
 
+pub(crate) const GENERATED_PREFIX: &str = "GEN_";
+
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapActionResult {
     Failed,
@@ -494,6 +496,8 @@ impl Imap {
     }
 
     async fn disconnect(&mut self, context: &Context) {
+        info!(context, "disconnecting");
+
         // Close folder if messages should be expunged
         if let Err(err) = self.close_folder(context).await {
             warn!(context, "failed to close folder: {:?}", err);
@@ -778,8 +782,7 @@ impl Imap {
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
             .unwrap_or_default();
         let download_limit = context.download_limit().await?;
-        let mut uids_fetch_fully = Vec::with_capacity(msgs.len());
-        let mut uids_fetch_partially = Vec::with_capacity(msgs.len());
+        let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1);
         let mut uid_message_ids = BTreeMap::new();
         let mut largest_uid_skipped = None;
 
@@ -794,7 +797,7 @@ impl Imap {
             };
 
             // Get the Message-ID or generate a fake one to identify the message in the database.
-            let message_id = prefetch_get_message_id(&headers).unwrap_or_else(create_id);
+            let message_id = prefetch_get_or_create_message_id(&headers);
 
             let target = match target_folder(context, folder, is_spam_folder, &headers).await? {
                 Some(config) => match context.get_config(config).await? {
@@ -838,14 +841,11 @@ impl Imap {
                 .await?
             {
                 match download_limit {
-                    Some(download_limit) => {
-                        if fetch_response.size.unwrap_or_default() > download_limit {
-                            uids_fetch_partially.push(uid);
-                        } else {
-                            uids_fetch_fully.push(uid)
-                        }
-                    }
-                    None => uids_fetch_fully.push(uid),
+                    Some(download_limit) => uids_fetch.push((
+                        uid,
+                        fetch_response.size.unwrap_or_default() > download_limit,
+                    )),
+                    None => uids_fetch.push((uid, false)),
                 }
                 uid_message_ids.insert(uid, message_id);
             } else {
@@ -853,33 +853,37 @@ impl Imap {
             }
         }
 
-        if !uids_fetch_fully.is_empty() || !uids_fetch_partially.is_empty() {
+        if !uids_fetch.is_empty() {
             self.connectivity.set_working(context).await;
         }
 
         // Actually download messages.
-        let (largest_uid_fully_fetched, mut received_msgs) = self
-            .fetch_many_msgs(
-                context,
-                folder,
-                uids_fetch_fully,
-                &uid_message_ids,
-                false,
-                fetch_existing_msgs,
-            )
-            .await?;
-
-        let (largest_uid_partially_fetched, received_msgs_2) = self
-            .fetch_many_msgs(
-                context,
-                folder,
-                uids_fetch_partially,
-                &uid_message_ids,
-                true,
-                fetch_existing_msgs,
-            )
-            .await?;
-        received_msgs.extend(received_msgs_2);
+        let mut largest_uid_fetched: u32 = 0;
+        let mut received_msgs = Vec::with_capacity(uids_fetch.len());
+        let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1));
+        let mut fetch_partially = false;
+        uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1));
+        for (uid, fp) in uids_fetch {
+            if fp != fetch_partially {
+                let (largest_uid_fetched_in_batch, received_msgs_in_batch) = self
+                    .fetch_many_msgs(
+                        context,
+                        folder,
+                        uids_fetch_in_batch.split_off(0),
+                        &uid_message_ids,
+                        fetch_partially,
+                        fetch_existing_msgs,
+                    )
+                    .await?;
+                received_msgs.extend(received_msgs_in_batch);
+                largest_uid_fetched = max(
+                    largest_uid_fetched,
+                    largest_uid_fetched_in_batch.unwrap_or(0),
+                );
+                fetch_partially = fp;
+            }
+            uids_fetch_in_batch.push(uid);
+        }
 
         // determine which uid_next to use to update to
         // receive_imf() returns an `Err` value only on recoverable errors, otherwise it just logs an error.
@@ -887,13 +891,7 @@ impl Imap {
 
         // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was
         // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times.
-        let largest_uid_without_errors = max(
-            max(
-                largest_uid_fully_fetched.unwrap_or(0),
-                largest_uid_partially_fetched.unwrap_or(0),
-            ),
-            largest_uid_skipped.unwrap_or(0),
-        );
+        let largest_uid_without_errors = max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0));
         let new_uid_next = largest_uid_without_errors + 1;
 
         if new_uid_next > old_uid_next {
@@ -1282,7 +1280,7 @@ impl Imap {
                 let msg = fetch?;
                 match get_fetch_headers(&msg) {
                     Ok(headers) => {
-                        if let Some(from) = mimeparser::get_from(&headers).first() {
+                        if let Some(from) = mimeparser::get_from(&headers) {
                             if context.is_self_addr(&from.addr).await? {
                                 result.extend(mimeparser::get_recipients(&headers));
                             }
@@ -1452,7 +1450,7 @@ impl Imap {
                     .context("we checked that message has body right above, but it has vanished")?;
                 let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
-                let rfc724_mid = if let Some(rfc724_mid) = &uid_message_ids.get(&server_uid) {
+                let rfc724_mid = if let Some(rfc724_mid) = uid_message_ids.get(&server_uid) {
                     rfc724_mid
                 } else {
                     warn!(
@@ -1460,7 +1458,7 @@ impl Imap {
                         "No Message-ID corresponding to UID {} passed in uid_messsage_ids",
                         server_uid
                     );
-                    ""
+                    continue;
                 };
                 match receive_imf_inner(
                     &context,
@@ -1756,9 +1754,13 @@ async fn should_move_out_of_spam(
             return Ok(false);
         }
     } else {
+        let from = match mimeparser::get_from(headers) {
+            Some(f) => f,
+            None => return Ok(false),
+        };
         // No chat found.
         let (from_id, blocked_contact, _origin) =
-            from_field_to_contact_id(context, &mimeparser::get_from(headers), true).await?;
+            from_field_to_contact_id(context, &from, true).await?;
         if blocked_contact {
             // Contact is blocked, leave the message in spam.
             return Ok(false);
@@ -1947,15 +1949,20 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
 
 fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
     for attr in folder_name.attributes() {
-        if let NameAttribute::Extension(ref label) = attr {
-            match label.as_ref() {
-                "\\Trash" => return FolderMeaning::Other,
-                "\\Sent" => return FolderMeaning::Sent,
-                "\\Spam" | "\\Junk" => return FolderMeaning::Spam,
-                "\\Drafts" => return FolderMeaning::Drafts,
-                "\\All" | "\\Important" | "\\Flagged" => return FolderMeaning::Virtual,
-                _ => {}
-            };
+        match attr {
+            NameAttribute::Trash => return FolderMeaning::Other,
+            NameAttribute::Sent => return FolderMeaning::Sent,
+            NameAttribute::Junk => return FolderMeaning::Spam,
+            NameAttribute::Drafts => return FolderMeaning::Drafts,
+            NameAttribute::All | NameAttribute::Flagged => return FolderMeaning::Virtual,
+            NameAttribute::Extension(ref label) => {
+                match label.as_ref() {
+                    "\\Spam" => return FolderMeaning::Spam,
+                    "\\Important" => return FolderMeaning::Virtual,
+                    _ => {}
+                };
+            }
+            _ => {}
         }
     }
     FolderMeaning::Unknown
@@ -1973,13 +1980,15 @@ fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>>
 }
 
 fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Option<String> {
-    if let Some(message_id) = headers.get_header_value(HeaderDef::XMicrosoftOriginalMessageId) {
-        crate::mimeparser::parse_message_id(&message_id).ok()
-    } else if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId) {
-        crate::mimeparser::parse_message_id(&message_id).ok()
-    } else {
-        None
-    }
+    headers
+        .get_header_value(HeaderDef::XMicrosoftOriginalMessageId)
+        .or_else(|| headers.get_header_value(HeaderDef::MessageId))
+        .and_then(|msgid| mimeparser::parse_message_id(&msgid).ok())
+}
+
+pub(crate) fn prefetch_get_or_create_message_id(headers: &[mailparse::MailHeader]) -> String {
+    prefetch_get_message_id(headers)
+        .unwrap_or_else(|| format!("{}{}", GENERATED_PREFIX, create_id()))
 }
 
 /// Returns chat by prefetched headers.
@@ -2036,8 +2045,12 @@ pub(crate) async fn prefetch_should_download(
         .get_header_value(HeaderDef::AutocryptSetupMessage)
         .is_some();
 
+    let from = match mimeparser::get_from(headers) {
+        Some(f) => f,
+        None => return Ok(false),
+    };
     let (_from_id, blocked_contact, origin) =
-        from_field_to_contact_id(context, &mimeparser::get_from(headers), true).await?;
+        from_field_to_contact_id(context, &from, true).await?;
     // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact.
     // (prevent_rename is the last argument of from_field_to_contact_id())
 

src/imap/idle.rs

@@ -54,10 +54,10 @@ impl Imap {
                 Interrupt(InterruptInfo),
             }
 
+            let folder_name = watch_folder.as_deref().unwrap_or("None");
             info!(
                 context,
-                "{}: Idle entering wait-on-remote state",
-                watch_folder.as_deref().unwrap_or("None")
+                "{}: Idle entering wait-on-remote state", folder_name
             );
             let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
                 let info = self.idle_interrupt.recv().await;
@@ -70,26 +70,36 @@ impl Imap {
 
             match fut.await {
                 Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
-                    info!(context, "Idle has NewData {:?}", x);
+                    info!(context, "{}: Idle has NewData {:?}", folder_name, x);
                 }
                 Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                    info!(context, "Idle-wait timeout or interruption");
+                    info!(
+                        context,
+                        "{}: Idle-wait timeout or interruption", folder_name
+                    );
                 }
                 Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                    info!(context, "Idle wait was interrupted");
+                    info!(
+                        context,
+                        "{}: Idle wait was interrupted manually", folder_name
+                    );
                 }
                 Ok(Event::Interrupt(i)) => {
+                    info!(
+                        context,
+                        "{}: Idle wait was interrupted: {:?}", folder_name, &i
+                    );
                     info = i;
-                    info!(context, "Idle wait was interrupted");
                 }
                 Err(err) => {
-                    warn!(context, "Idle wait errored: {:?}", err);
+                    warn!(context, "{}: Idle wait errored: {:?}", folder_name, err);
                 }
             }
 
             let session = tokio::time::timeout(Duration::from_secs(15), handle.done())
-                .await?
-                .context("IMAP IDLE protocol timed out")?;
+                .await
+                .with_context(|| format!("{}: IMAP IDLE protocol timed out", folder_name))?
+                .with_context(|| format!("{}: IMAP IDLE failed", folder_name))?;
             self.session = Some(Session { inner: session });
         } else {
             warn!(context, "Attempted to idle without a session");
@@ -173,6 +183,7 @@ impl Imap {
                 }
                 Event::Interrupt(info) => {
                     // Interrupt
+                    info!(context, "Fake IDLE interrupted");
                     break info;
                 }
             }

src/imap/select_folder.rs

@@ -19,35 +19,31 @@ pub enum Error {
     #[error("Got a NO response when trying to select {0}, usually this means that it doesn't exist: {1}")]
     NoFolder(String, String),
 
-    #[error("IMAP close/expunge failed")]
-    CloseExpungeFailed(#[from] async_imap::error::Error),
-
     #[error("IMAP other error: {0}")]
     Other(String),
 }
 
+impl From<anyhow::Error> for Error {
+    fn from(err: anyhow::Error) -> Error {
+        Error::Other(format!("{:#}", err))
+    }
+}
+
 impl Imap {
     /// Issues a CLOSE command to expunge selected folder.
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// <https://tools.ietf.org/html/rfc3501#section-6.4.2>
-    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> anyhow::Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 
-            if let Some(ref mut session) = self.session {
-                match session.close().await {
-                    Ok(_) => {
-                        info!(context, "close/expunge succeeded");
-                    }
-                    Err(err) => {
-                        self.trigger_reconnect(context).await;
-                        return Err(Error::CloseExpungeFailed(err));
-                    }
-                }
-            } else {
-                return Err(Error::NoSession);
+            let session = self.session.as_mut().context("no session")?;
+            if let Err(err) = session.close().await.context("IMAP close/expunge failed") {
+                self.trigger_reconnect(context).await;
+                return Err(err);
             }
+            info!(context, "close/expunge succeeded");
         }
         self.config.selected_folder = None;
         self.config.selected_folder_needs_expunge = false;
@@ -56,7 +52,7 @@ impl Imap {
     }
 
     /// Issues a CLOSE command if selected folder needs expunge.
-    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> anyhow::Result<()> {
         if self.config.selected_folder_needs_expunge {
             self.close_folder(context).await?;
         }

src/message.rs

@@ -1122,6 +1122,28 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     }
     if !msg.rfc724_mid.is_empty() {
         ret += &format!("\nMessage-ID: {}", msg.rfc724_mid);
+
+        let server_uids = context
+            .sql
+            .query_map(
+                "SELECT folder, uid FROM imap WHERE rfc724_mid=?",
+                paramsv![msg.rfc724_mid],
+                |row| {
+                    let folder: String = row.get("folder")?;
+                    let uid: u32 = row.get("uid")?;
+                    Ok((folder, uid))
+                },
+                |rows| {
+                    rows.collect::<std::result::Result<Vec<_>, _>>()
+                        .map_err(Into::into)
+                },
+            )
+            .await?;
+
+        for (folder, uid) in server_uids {
+            // Format as RFC 5092 relative IMAP URL.
+            ret += &format!("\n</{}/;UID={}>", folder, uid);
+        }
     }
     let hop_info: Option<String> = context
         .sql

src/mimeparser.rs

@@ -21,7 +21,6 @@ use crate::events::EventType;
 use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
-use crate::location;
 use crate::message::{self, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
@@ -29,6 +28,7 @@ use crate::simplify::{simplify, SimplifiedText};
 use crate::stock_str;
 use crate::sync::SyncItems;
 use crate::tools::{get_filemeta, parse_receive_headers, truncate_by_lines};
+use crate::{location, tools};
 
 /// A parsed MIME message.
 ///
@@ -46,7 +46,7 @@ pub struct MimeMessage {
 
     /// Addresses are normalized and lowercased:
     pub recipients: Vec<SingleInfo>,
-    pub from: Vec<SingleInfo>,
+    pub from: SingleInfo,
     /// Whether the From address was repeated in the signed part
     /// (and we know that the signer intended to send from this address)
     pub from_is_signed: bool,
@@ -157,21 +157,50 @@ impl Default for SystemMessage {
 
 const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum ParserError {
+    #[error("{}", _0)]
+    Malformed(anyhow::Error),
+
+    #[error("{:#}", _0)]
+    Sql(anyhow::Error),
+}
+
+pub(crate) type ParserResult<T> = std::result::Result<T, ParserError>;
+
+pub(crate) trait ParserErrorExt<T, E>
+where
+    Self: std::marker::Sized,
+{
+    fn map_err_malformed(self) -> ParserResult<T>;
+    fn map_err_sql(self) -> ParserResult<T>;
+}
+
+impl<T, E: Into<anyhow::Error>> ParserErrorExt<T, E> for Result<T, E> {
+    fn map_err_malformed(self) -> ParserResult<T> {
+        self.map_err(|e| ParserError::Malformed(e.into()))
+    }
+
+    fn map_err_sql(self) -> ParserResult<T> {
+        self.map_err(|e| ParserError::Sql(e.into()))
+    }
+}
+
 impl MimeMessage {
     pub async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
-        MimeMessage::from_bytes_with_partial(context, body, None).await
+        Ok(MimeMessage::from_bytes_with_partial(context, body, None).await?)
     }
 
     /// Parse a mime message.
     ///
     /// If `partial` is set, it contains the full message size in bytes
     /// and `body` contains the header only.
-    pub async fn from_bytes_with_partial(
+    pub(crate) async fn from_bytes_with_partial(
         context: &Context,
         body: &[u8],
         partial: Option<u32>,
-    ) -> Result<Self> {
-        let mail = mailparse::parse_mail(body)?;
+    ) -> ParserResult<Self> {
+        let mail = mailparse::parse_mail(body).map_err_malformed()?;
 
         let message_time = mail
             .headers
@@ -198,7 +227,7 @@ impl MimeMessage {
         );
 
         // Parse hidden headers.
-        let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+        let mimetype = mail.ctype.mimetype.parse::<Mime>().map_err_malformed()?;
         if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" {
             if let Some(part) = mail.subparts.first() {
                 for field in &part.headers {
@@ -216,11 +245,14 @@ impl MimeMessage {
         headers.remove("secure-join-fingerprint");
         headers.remove("chat-verified");
 
+        let from = from.context("No from in message").map_err_malformed()?;
+        let mut decryption_info =
+            prepare_decryption(context, &mail, &from.addr, message_time).await?;
+
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
         let mut gossiped_addr = Default::default();
         let mut from_is_signed = false;
-        let mut decryption_info = prepare_decryption(context, &mail, &from, message_time).await?;
         hop_info += "\n\n";
         hop_info += &decryption_info.dkim_results.to_string();
 
@@ -233,7 +265,7 @@ impl MimeMessage {
                     // autocrypt message.
 
                     mail_raw = raw;
-                    let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
+                    let decrypted_mail = mailparse::parse_mail(&mail_raw).map_err_malformed()?;
                     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
                         info!(context, "decrypted message mime-body:");
                         println!("{}", String::from_utf8_lossy(&mail_raw));
@@ -247,7 +279,8 @@ impl MimeMessage {
                             decrypted_mail.headers.get_all_values("Autocrypt-Gossip");
                         gossiped_addr =
                             update_gossip_peerstates(context, message_time, &mail, gossip_headers)
-                                .await?;
+                                .await
+                                .map_err_sql()?;
                     }
 
                     // let known protected headers from the decrypted
@@ -255,7 +288,7 @@ impl MimeMessage {
 
                     // Signature was checked for original From, so we
                     // do not allow overriding it.
-                    let mut signed_from = Vec::new();
+                    let mut signed_from = None;
 
                     // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
                     // See <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
@@ -270,23 +303,21 @@ impl MimeMessage {
                         &mut chat_disposition_notification_to,
                         &decrypted_mail.headers,
                     );
-                    if let Some(signed_from) = signed_from.first() {
-                        if let Some(from) = from.first() {
-                            if addr_cmp(&signed_from.addr, &from.addr) {
-                                from_is_signed = true;
-                            } else {
-                                // There is a From: header in the encrypted &
-                                // signed part, but it doesn't match the outer one.
-                                // This _might_ be because the sender's mail server
-                                // replaced the sending address, e.g. in a mailing list.
-                                // Or it's because someone is doing some replay attack
-                                // - OTOH, I can't come up with an attack scenario
-                                // where this would be useful.
-                                warn!(
-                                    context,
-                                    "From header in signed part does't match the outer one"
-                                );
-                            }
+                    if let Some(signed_from) = signed_from {
+                        if addr_cmp(&signed_from.addr, &from.addr) {
+                            from_is_signed = true;
+                        } else {
+                            // There is a From: header in the encrypted &
+                            // signed part, but it doesn't match the outer one.
+                            // This _might_ be because the sender's mail server
+                            // replaced the sending address, e.g. in a mailing list.
+                            // Or it's because someone is doing some replay attack
+                            // - OTOH, I can't come up with an attack scenario
+                            // where this would be useful.
+                            warn!(
+                                context,
+                                "From header in signed part does't match the outer one"
+                            );
                         }
                     }
 
@@ -302,7 +333,10 @@ impl MimeMessage {
                         // && decryption_info.dkim_results.allow_keychange
                         {
                             peerstate.degrade_encryption(message_time);
-                            peerstate.save_to_db(&context.sql, false).await?;
+                            peerstate
+                                .save_to_db(&context.sql, false)
+                                .await
+                                .map_err_sql()?;
                         }
                     }
                     (Ok(mail), HashSet::new(), false)
@@ -346,11 +380,15 @@ impl MimeMessage {
             Some(org_bytes) => {
                 parser
                     .create_stub_from_partial_download(context, org_bytes)
-                    .await?;
+                    .await
+                    .map_err_sql()?;
             }
             None => match mail {
                 Ok(mail) => {
-                    parser.parse_mime_recursive(context, &mail, false).await?;
+                    parser
+                        .parse_mime_recursive(context, &mail, false)
+                        .await
+                        .map_err_malformed()?;
                 }
                 Err(err) => {
                     let msg_body = stock_str::cant_decrypt_msg_body(context).await;
@@ -371,7 +409,7 @@ impl MimeMessage {
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
-        parser.parse_headers(context).await?;
+        parser.parse_headers(context).await.map_err_malformed()?;
 
         // Disallowing keychanges is disabled for now
         // if !decryption_info.dkim_results.allow_keychange {
@@ -389,11 +427,14 @@ impl MimeMessage {
             parser.decoded_data = mail_raw;
         }
 
-        crate::peerstate::maybe_do_aeap_transition(context, &mut decryption_info, &parser).await?;
+        crate::peerstate::maybe_do_aeap_transition(context, &mut decryption_info, &parser)
+            .await
+            .map_err_sql()?;
         if let Some(peerstate) = decryption_info.peerstate {
             peerstate
                 .handle_fingerprint_change(context, message_time)
-                .await?;
+                .await
+                .map_err_sql()?;
         }
 
         Ok(parser)
@@ -582,21 +623,18 @@ impl MimeMessage {
         // See if an MDN is requested from the other side
         if !self.decrypting_failed && !self.parts.is_empty() {
             if let Some(ref dn_to) = self.chat_disposition_notification_to {
-                if let Some(from) = self.from.get(0) {
-                    // Check that the message is not outgoing.
-                    if !context.is_self_addr(&from.addr).await? {
-                        if from.addr.to_lowercase() == dn_to.addr.to_lowercase() {
-                            if let Some(part) = self.parts.last_mut() {
-                                part.param.set_int(Param::WantsMdn, 1);
-                            }
-                        } else {
-                            warn!(
-                                context,
-                                "{} requested a read receipt to {}, ignoring",
-                                from.addr,
-                                dn_to.addr
-                            );
+                // Check that the message is not outgoing.
+                let from = &self.from.addr;
+                if !context.is_self_addr(from).await? {
+                    if from.to_lowercase() == dn_to.addr.to_lowercase() {
+                        if let Some(part) = self.parts.last_mut() {
+                            part.param.set_int(Param::WantsMdn, 1);
                         }
+                    } else {
+                        warn!(
+                            context,
+                            "{} requested a read receipt to {}, ignoring", from, dn_to.addr
+                        );
                     }
                 }
             }
@@ -1226,7 +1264,7 @@ impl MimeMessage {
         context: &Context,
         headers: &mut HashMap<String, String>,
         recipients: &mut Vec<SingleInfo>,
-        from: &mut Vec<SingleInfo>,
+        from: &mut Option<SingleInfo>,
         list_post: &mut Option<String>,
         chat_disposition_notification_to: &mut Option<SingleInfo>,
         fields: &[mailparse::MailHeader<'_>],
@@ -1255,7 +1293,7 @@ impl MimeMessage {
             *recipients = recipients_new;
         }
         let from_new = get_from(fields);
-        if !from_new.is_empty() {
+        if from_new.is_some() {
             *from = from_new;
         }
         let list_post_new = get_list_post(fields);
@@ -1800,8 +1838,9 @@ pub(crate) fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
 }
 
 /// Returned addresses are normalized and lowercased.
-pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
-    get_all_addresses_from_header(headers, |header_key| header_key == "from")
+pub(crate) fn get_from(headers: &[MailHeader]) -> Option<SingleInfo> {
+    let all = get_all_addresses_from_header(headers, |header_key| header_key == "from");
+    tools::single_value(all)
 }
 
 /// Returned addresses are normalized and lowercased.
@@ -1877,35 +1916,35 @@ mod tests {
         let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi")
             .await
             .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, None);
 
         let mimemsg = MimeMessage::from_bytes(&ctx, b"From:   g@c.de  \n\nhi")
             .await
             .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, None);
 
         let mimemsg = MimeMessage::from_bytes(&ctx, b"From: <g@c.de>\n\nhi")
             .await
             .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, None);
 
         let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C <g@c.de>\n\nhi")
             .await
             .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, Some("Goetz C".to_string()));
 
         let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" <g@c.de>\n\nhi")
             .await
             .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, Some("Goetz C".to_string()));
 
@@ -1913,7 +1952,7 @@ mod tests {
             MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C <g@c.de>\n\nhi")
                 .await
                 .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, Some("Götz C".to_string()));
 
@@ -1923,7 +1962,7 @@ mod tests {
             MimeMessage::from_bytes(&ctx, b"From: \"=?utf-8?q?G=C3=B6tz?= C\" <g@c.de>\n\nhi")
                 .await
                 .unwrap();
-        let contact = mimemsg.from.first().unwrap();
+        let contact = mimemsg.from;
         assert_eq!(contact.addr, "g@c.de");
         assert_eq!(contact.display_name, Some("Götz C".to_string()));
     }
@@ -2155,14 +2194,9 @@ mod tests {
                     test1\n\
                     ";
 
-        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
-            .await
-            .unwrap();
+        let mimeparser = MimeMessage::from_bytes_with_partial(&context.ctx, &raw[..], None).await;
 
-        let of = &mimeparser.from[0];
-        assert_eq!(of.addr, "hello@one.org");
-
-        assert!(mimeparser.chat_disposition_notification_to.is_none());
+        assert!(matches!(mimeparser, Err(ParserError::Malformed(_))));
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -2201,7 +2235,7 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mimeparser_with_context() {
         let context = TestContext::new().await;
-        let raw = b"From: hello\n\
+        let raw = b"From: hello@example.org\n\
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
                     Secure-Join-Group: no\n\

src/peerstate.rs

@@ -617,10 +617,9 @@ pub async fn maybe_do_aeap_transition(
     mime_parser: &crate::mimeparser::MimeMessage,
 ) -> Result<()> {
     if let Some(peerstate) = &mut info.peerstate {
-        if let Some(from) = mime_parser.from.first() {
-            // If the from addr is different from the peerstate address we know,
-            // we may want to do an AEAP transition.
-            if !addr_cmp(&peerstate.addr, &from.addr)
+        // If the from addr is different from the peerstate address we know,
+        // we may want to do an AEAP transition.
+        if !addr_cmp(&peerstate.addr, &mime_parser.from.addr)
                 // Check if it's a chat message; we do this to avoid
                 // some accidental transitions if someone writes from multiple
                 // addresses with an MUA.
@@ -636,31 +635,30 @@ pub async fn maybe_do_aeap_transition(
                 // to the attacker's address, allowing for easier phishing.
                 && mime_parser.from_is_signed
                 && info.message_time > peerstate.last_seen
-            {
-                // Add info messages to chats with this (verified) contact
-                //
-                peerstate
-                    .handle_setup_change(
-                        context,
-                        info.message_time,
-                        PeerstateChange::Aeap(info.from.clone()),
-                    )
-                    .await?;
+        {
+            // Add info messages to chats with this (verified) contact
+            //
+            peerstate
+                .handle_setup_change(
+                    context,
+                    info.message_time,
+                    PeerstateChange::Aeap(info.from.clone()),
+                )
+                .await?;
 
-                peerstate.addr = info.from.clone();
-                let header = info.autocrypt_header.as_ref().context(
-                    "Internal error: Tried to do an AEAP transition without an autocrypt header??",
-                )?;
-                peerstate.apply_header(header, info.message_time);
-                peerstate.to_save = Some(ToSave::All);
+            peerstate.addr = info.from.clone();
+            let header = info.autocrypt_header.as_ref().context(
+                "Internal error: Tried to do an AEAP transition without an autocrypt header??",
+            )?;
+            peerstate.apply_header(header, info.message_time);
+            peerstate.to_save = Some(ToSave::All);
 
-                // We don't know whether a peerstate with this address already existed, or a
-                // new one should be created, so just try both create=false and create=true,
-                // and if this fails, create=true, one will succeed (this is a very cold path,
-                // so performance doesn't really matter).
-                peerstate.save_to_db(&context.sql, true).await?;
-                peerstate.save_to_db(&context.sql, false).await?;
-            }
+            // We don't know whether a peerstate with this address already existed, or a
+            // new one should be created, so just try both create=false and create=true,
+            // and if this fails, create=true, one will succeed (this is a very cold path,
+            // so performance doesn't really matter).
+            peerstate.save_to_db(&context.sql, true).await?;
+            peerstate.save_to_db(&context.sql, false).await?;
         }
     }
 

src/receive_imf.rs

@@ -13,7 +13,6 @@ use regex::Regex;
 use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, ShowEmails, DC_CHAT_ID_TRASH};
-use crate::contact;
 use crate::contact::{
     may_be_valid_addr, normalize_name, Contact, ContactId, Origin, VerifiedStatus,
 };
@@ -22,14 +21,14 @@ use crate::download::DownloadState;
 use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::imap::markseen_on_imap_table;
+use crate::imap::{markseen_on_imap_table, GENERATED_PREFIX};
 use crate::location;
 use crate::log::LogExt;
 use crate::message::{
     self, rfc724_mid_exists, Message, MessageState, MessengerMessage, MsgId, Viewtype,
 };
 use crate::mimeparser::{
-    parse_message_id, parse_message_ids, AvatarAction, MailinglistType, MimeMessage, SystemMessage,
+    parse_message_ids, AvatarAction, MailinglistType, MimeMessage, ParserError, SystemMessage,
 };
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus};
@@ -37,7 +36,8 @@ use crate::reaction::{set_msg_reaction, Reaction};
 use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
 use crate::sql;
 use crate::stock_str;
-use crate::tools::{create_id, extract_grpid_from_rfc724_mid, smeared_time};
+use crate::tools::{extract_grpid_from_rfc724_mid, smeared_time};
+use crate::{contact, imap};
 
 /// This is the struct that is returned after receiving one email (aka MIME message).
 ///
@@ -66,11 +66,7 @@ pub async fn receive_imf(
     seen: bool,
 ) -> Result<Option<ReceivedMsg>> {
     let mail = parse_mail(imf_raw).context("can't parse mail")?;
-    let rfc724_mid = mail
-        .headers
-        .get_header_value(HeaderDef::MessageId)
-        .and_then(|msgid| parse_message_id(&msgid).ok())
-        .unwrap_or_else(create_id);
+    let rfc724_mid = imap::prefetch_get_or_create_message_id(&mail.headers);
     receive_imf_inner(context, &rfc724_mid, imf_raw, seen, None, false).await
 }
 
@@ -105,10 +101,33 @@ pub(crate) async fn receive_imf_inner(
 
     let mut mime_parser =
         match MimeMessage::from_bytes_with_partial(context, imf_raw, is_partial_download).await {
-            Err(err) => {
+            Err(ParserError::Malformed(err)) => {
                 warn!(context, "receive_imf: can't parse MIME: {}", err);
-                return Ok(None);
+
+                let msg_ids;
+                if !rfc724_mid.starts_with(GENERATED_PREFIX) {
+                    let row_id = context
+                        .sql
+                        .execute(
+                            "INSERT INTO msgs(rfc724_mid, chat_id) VALUES (?,?)",
+                            paramsv![rfc724_mid, DC_CHAT_ID_TRASH],
+                        )
+                        .await?;
+                    msg_ids = vec![MsgId::new(u32::try_from(row_id)?)];
+                } else {
+                    return Ok(None);
+                    // We don't have an rfc724_mid, there's no point in adding a trash entry
+                }
+
+                return Ok(Some(ReceivedMsg {
+                    chat_id: DC_CHAT_ID_TRASH,
+                    state: MessageState::Undefined,
+                    sort_timestamp: 0,
+                    msg_ids,
+                    needs_delete_job: false,
+                }));
             }
+            Err(ParserError::Sql(err)) => return Err(err),
             Ok(mime_parser) => mime_parser,
         };
 
@@ -141,7 +160,6 @@ pub(crate) async fn receive_imf_inner(
             None
         };
 
-    // the function returns the number of created messages in the database
     let prevent_rename =
         mime_parser.is_mailinglist_message() || mime_parser.get_header(HeaderDef::Sender).is_some();
 
@@ -168,7 +186,6 @@ pub(crate) async fn receive_imf_inner(
         } else {
             Origin::IncomingUnknownTo
         },
-        prevent_rename,
     )
     .await?;
 
@@ -353,28 +370,25 @@ pub(crate) async fn receive_imf_inner(
 /// * `prevent_rename`: passed through to `add_or_lookup_contacts_by_address_list()`
 pub async fn from_field_to_contact_id(
     context: &Context,
-    from_address_list: &[SingleInfo],
+    from: &SingleInfo,
     prevent_rename: bool,
 ) -> Result<(ContactId, bool, Origin)> {
-    let from_ids = add_or_lookup_contacts_by_address_list(
+    let display_name = if prevent_rename {
+        Some("")
+    } else {
+        from.display_name.as_deref()
+    };
+    let from_id = add_or_lookup_contact_by_addr(
         context,
-        from_address_list,
+        display_name,
+        &from.addr,
         Origin::IncomingUnknownFrom,
-        prevent_rename,
     )
     .await?;
 
-    if from_ids.contains(&ContactId::SELF) {
+    if from_id == ContactId::SELF {
         Ok((ContactId::SELF, false, Origin::OutgoingBcc))
-    } else if !from_ids.is_empty() {
-        if from_ids.len() > 1 {
-            warn!(
-                context,
-                "mail has more than one From address, only using first: {:?}", from_address_list
-            );
-        }
-        let from_id = from_ids.get(0).cloned().unwrap_or_default();
-
+    } else {
         let mut from_id_blocked = false;
         let mut incoming_origin = Origin::Unknown;
         if let Ok(contact) = Contact::load_from_db(context, from_id).await {
@@ -382,13 +396,6 @@ pub async fn from_field_to_contact_id(
             incoming_origin = contact.origin;
         }
         Ok((from_id, from_id_blocked, incoming_origin))
-    } else {
-        warn!(
-            context,
-            "mail has an empty From header: {:?}", from_address_list
-        );
-
-        Ok((ContactId::UNDEFINED, false, Origin::Unknown))
     }
 }
 
@@ -570,9 +577,10 @@ async fn add_parts(
                 if chat.is_protected() {
                     let s = stock_str::unknown_sender_for_chat(context).await;
                     mime_parser.repl_msg_by_error(&s);
-                } else if let Some(from) = mime_parser.from.first() {
+                } else {
                     // In non-protected chats, just mark the sender as overridden. Therefore, the UI will prepend `~`
                     // to the sender's name, indicating to the user that he/she is not part of the group.
+                    let from = &mime_parser.from;
                     let name: &str = from.display_name.as_ref().unwrap_or(&from.addr);
                     for part in mime_parser.parts.iter_mut() {
                         part.param.set(Param::OverrideSenderDisplayname, name);
@@ -637,11 +645,9 @@ async fn add_parts(
         // if contact renaming is prevented (for mailinglists and bots),
         // we use name from From:-header as override name
         if prevent_rename {
-            if let Some(from) = mime_parser.from.first() {
-                if let Some(name) = &from.display_name {
-                    for part in mime_parser.parts.iter_mut() {
-                        part.param.set(Param::OverrideSenderDisplayname, name);
-                    }
+            if let Some(name) = &mime_parser.from.display_name {
+                for part in mime_parser.parts.iter_mut() {
+                    part.param.set(Param::OverrideSenderDisplayname, name);
                 }
             }
         }
@@ -1801,10 +1807,8 @@ async fn create_or_lookup_mailinglist(
     // a usable name for these lists is in the `From` header
     // and we can detect these lists by a unique `ListId`-suffix.
     if listid.ends_with(".list-id.mcsv.net") {
-        if let Some(from) = mime_parser.from.first() {
-            if let Some(display_name) = &from.display_name {
-                name = display_name.clone();
-            }
+        if let Some(display_name) = &mime_parser.from.display_name {
+            name = display_name.clone();
         }
     }
 
@@ -1824,18 +1828,15 @@ async fn create_or_lookup_mailinglist(
     //
     // this pattern is similar to mailchimp above, however,
     // with weaker conditions and does not overwrite existing names.
-    if name.is_empty() {
-        if let Some(from) = mime_parser.from.first() {
-            if from.addr.contains("noreply")
-                || from.addr.contains("no-reply")
-                || from.addr.starts_with("notifications@")
-                || from.addr.starts_with("newsletter@")
-                || listid.ends_with(".xt.local")
-            {
-                if let Some(display_name) = &from.display_name {
-                    name = display_name.clone();
-                }
-            }
+    if name.is_empty()
+        && (mime_parser.from.addr.contains("noreply")
+            || mime_parser.from.addr.contains("no-reply")
+            || mime_parser.from.addr.starts_with("notifications@")
+            || mime_parser.from.addr.starts_with("newsletter@")
+            || listid.ends_with(".xt.local"))
+    {
+        if let Some(display_name) = &mime_parser.from.display_name {
+            name = display_name.clone();
         }
     }
 
@@ -2233,7 +2234,6 @@ async fn add_or_lookup_contacts_by_address_list(
     context: &Context,
     address_list: &[SingleInfo],
     origin: Origin,
-    prevent_rename: bool,
 ) -> Result<Vec<ContactId>> {
     let mut contact_ids = HashSet::new();
     for info in address_list.iter() {
@@ -2241,11 +2241,7 @@ async fn add_or_lookup_contacts_by_address_list(
         if !may_be_valid_addr(addr) {
             continue;
         }
-        let display_name = if prevent_rename {
-            Some("")
-        } else {
-            info.display_name.as_deref()
-        };
+        let display_name = info.display_name.as_deref();
         contact_ids
             .insert(add_or_lookup_contact_by_addr(context, display_name, addr, origin).await?);
     }
@@ -2288,7 +2284,7 @@ mod tests {
     async fn test_grpid_simple() {
         let context = TestContext::new().await;
         let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                    From: hello\n\
+                    From: hello@example.org\n\
                     Subject: outer-subject\n\
                     In-Reply-To: <lqkjwelq123@123123>\n\
                     References: <Gr.HcxyMARjyJy.9-uvzWPTLtV@nauta.cu>\n\
@@ -2303,11 +2299,25 @@ mod tests {
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_grpid_from_multiple() {
+    async fn test_bad_from() {
         let context = TestContext::new().await;
         let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
                     From: hello\n\
                     Subject: outer-subject\n\
+                    In-Reply-To: <lqkjwelq123@123123>\n\
+                    References: <Gr.HcxyMARjyJy.9-uvzWPTLtV@nauta.cu>\n\
+                    \n\
+                    hello\x00";
+        let mimeparser = MimeMessage::from_bytes_with_partial(&context.ctx, &raw[..], None).await;
+        assert!(matches!(mimeparser, Err(ParserError::Malformed(_))));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_grpid_from_multiple() {
+        let context = TestContext::new().await;
+        let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                    From: hello@example.org\n\
+                    Subject: outer-subject\n\
                     In-Reply-To: <Gr.HcxyMARjyJy.9-qweqwe@asd.net>\n\
                     References: <qweqweqwe>, <Gr.HcxyMARjyJy.9-uvzWPTLtV@nau.ca>\n\
                     \n\
@@ -2593,8 +2603,55 @@ mod tests {
         .unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        // Check that the message was added to the database:
-        assert!(chats.get_msg_id(0).is_ok());
+        // Check that the message is not shown to the user:
+        assert!(chats.is_empty());
+
+        // Check that the message was added to the db:
+        assert!(message::rfc724_mid_exists(context, "3924@example.com")
+            .await
+            .unwrap()
+            .is_some());
+    }
+
+    /// If there is no Message-Id header, we generate a random id.
+    /// But there is no point in adding a trash entry in the database
+    /// if the email is malformed (e.g. because `From` is missing)
+    /// with this random id we just generated.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_no_message_id_header() {
+        let t = TestContext::new_alice().await;
+
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert!(chats.get_msg_id(0).is_err());
+
+        let received = receive_imf(
+            &t,
+            b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+              To: bob@example.com\n\
+              Subject: foo\n\
+              Chat-Version: 1.0\n\
+              Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+              \n\
+              hello\n",
+            false,
+        )
+        .await
+        .unwrap();
+        dbg!(&received);
+        assert!(received.is_none());
+
+        assert!(!t
+            .sql
+            .exists(
+                "SELECT COUNT(*) FROM msgs WHERE chat_id=?;",
+                paramsv![DC_CHAT_ID_TRASH],
+            )
+            .await
+            .unwrap());
+
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        // Check that the message is not shown to the user:
+        assert!(chats.is_empty());
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

src/scheduler.rs

@@ -166,112 +166,117 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         .await;
 }
 
-async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
-    match ctx.get_config(folder).await {
-        Ok(Some(watch_folder)) => {
-            // connect and fake idle if unable to connect
-            if let Err(err) = connection.prepare(ctx).await {
-                warn!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(ctx, Some(watch_folder)).await;
-            }
-
-            if folder == Config::ConfiguredInboxFolder {
-                if let Err(err) = connection
-                    .store_seen_flags_on_imap(ctx)
-                    .await
-                    .context("store_seen_flags_on_imap failed")
-                {
-                    warn!(ctx, "{:#}", err);
-                }
-            }
-
-            // Fetch the watched folder.
-            if let Err(err) = connection
-                .fetch_move_delete(ctx, &watch_folder, false)
-                .await
-            {
-                connection.trigger_reconnect(ctx).await;
-                warn!(ctx, "{:#}", err);
-                return InterruptInfo::new(false);
-            }
-
-            // Mark expired messages for deletion. Marked messages will be deleted from the server
-            // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
-            // called right before `fetch_move_delete` because it is not well optimized and would
-            // otherwise slow down message fetching.
-            if let Err(err) = delete_expired_imap_messages(ctx)
-                .await
-                .context("delete_expired_imap_messages failed")
-            {
-                warn!(ctx, "{:#}", err);
-            }
-
-            // Scan additional folders only after finishing fetching the watched folder.
-            //
-            // On iOS the application has strictly limited time to work in background, so we may not
-            // be able to scan all folders before time is up if there are many of them.
-            if folder == Config::ConfiguredInboxFolder {
-                // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
-                match connection.scan_folders(ctx).await {
-                    Err(err) => {
-                        // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
-                        // but maybe just one folder can't be selected or something
-                        warn!(ctx, "{}", err);
-                    }
-                    Ok(true) => {
-                        // Fetch the watched folder again in case scanning other folder moved messages
-                        // there.
-                        //
-                        // In most cases this will select the watched folder and return because there are
-                        // no new messages. We want to select the watched folder anyway before going IDLE
-                        // there, so this does not take additional protocol round-trip.
-                        if let Err(err) = connection
-                            .fetch_move_delete(ctx, &watch_folder, false)
-                            .await
-                        {
-                            connection.trigger_reconnect(ctx).await;
-                            warn!(ctx, "{:#}", err);
-                            return InterruptInfo::new(false);
-                        }
-                    }
-                    Ok(false) => {}
-                }
-            }
-
-            // Synchronize Seen flags.
-            connection
-                .sync_seen_flags(ctx, &watch_folder)
-                .await
-                .context("sync_seen_flags")
-                .ok_or_log(ctx);
-
-            connection.connectivity.set_connected(ctx).await;
-
-            // idle
-            if connection.can_idle() {
-                match connection.idle(ctx, Some(watch_folder)).await {
-                    Ok(v) => v,
-                    Err(err) => {
-                        connection.trigger_reconnect(ctx).await;
-                        warn!(ctx, "{}", err);
-                        InterruptInfo::new(false)
-                    }
-                }
-            } else {
-                connection.fake_idle(ctx, Some(watch_folder)).await
-            }
-        }
-        Ok(None) => {
-            connection.connectivity.set_not_configured(ctx).await;
-            info!(ctx, "Can not watch {} folder, not set", folder);
-            connection.fake_idle(ctx, None).await
-        }
+async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) -> InterruptInfo {
+    let folder = match ctx.get_config(folder_config).await {
+        Ok(folder) => folder,
         Err(err) => {
             warn!(
                 ctx,
-                "Can not watch {} folder, failed to retrieve config: {:?}", folder, err
+                "Can not watch {} folder, failed to retrieve config: {:#}", folder_config, err
             );
-            connection.fake_idle(ctx, None).await
+            return connection.fake_idle(ctx, None).await;
+        }
+    };
+
+    let watch_folder = if let Some(watch_folder) = folder {
+        watch_folder
+    } else {
+        connection.connectivity.set_not_configured(ctx).await;
+        info!(ctx, "Can not watch {} folder, not set", folder_config);
+        return connection.fake_idle(ctx, None).await;
+    };
+
+    // connect and fake idle if unable to connect
+    if let Err(err) = connection.prepare(ctx).await {
+        warn!(ctx, "imap connection failed: {}", err);
+        return connection.fake_idle(ctx, Some(watch_folder)).await;
+    }
+
+    if folder_config == Config::ConfiguredInboxFolder {
+        if let Err(err) = connection
+            .store_seen_flags_on_imap(ctx)
+            .await
+            .context("store_seen_flags_on_imap failed")
+        {
+            warn!(ctx, "{:#}", err);
+        }
+    }
+
+    // Fetch the watched folder.
+    if let Err(err) = connection
+        .fetch_move_delete(ctx, &watch_folder, false)
+        .await
+        .context("fetch_move_delete")
+    {
+        connection.trigger_reconnect(ctx).await;
+        warn!(ctx, "{:#}", err);
+        return InterruptInfo::new(false);
+    }
+
+    // Mark expired messages for deletion. Marked messages will be deleted from the server
+    // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
+    // called right before `fetch_move_delete` because it is not well optimized and would
+    // otherwise slow down message fetching.
+    if let Err(err) = delete_expired_imap_messages(ctx)
+        .await
+        .context("delete_expired_imap_messages")
+    {
+        warn!(ctx, "{:#}", err);
+    }
+
+    // Scan additional folders only after finishing fetching the watched folder.
+    //
+    // On iOS the application has strictly limited time to work in background, so we may not
+    // be able to scan all folders before time is up if there are many of them.
+    if folder_config == Config::ConfiguredInboxFolder {
+        // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
+        match connection.scan_folders(ctx).await.context("scan_folders") {
+            Err(err) => {
+                // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
+                // but maybe just one folder can't be selected or something
+                warn!(ctx, "{:#}", err);
+            }
+            Ok(true) => {
+                // Fetch the watched folder again in case scanning other folder moved messages
+                // there.
+                //
+                // In most cases this will select the watched folder and return because there are
+                // no new messages. We want to select the watched folder anyway before going IDLE
+                // there, so this does not take additional protocol round-trip.
+                if let Err(err) = connection
+                    .fetch_move_delete(ctx, &watch_folder, false)
+                    .await
+                    .context("fetch_move_delete after scan_folders")
+                {
+                    connection.trigger_reconnect(ctx).await;
+                    warn!(ctx, "{:#}", err);
+                    return InterruptInfo::new(false);
+                }
+            }
+            Ok(false) => {}
+        }
+    }
+
+    // Synchronize Seen flags.
+    connection
+        .sync_seen_flags(ctx, &watch_folder)
+        .await
+        .context("sync_seen_flags")
+        .ok_or_log(ctx);
+
+    connection.connectivity.set_connected(ctx).await;
+
+    // idle
+    if !connection.can_idle() {
+        return connection.fake_idle(ctx, Some(watch_folder)).await;
+    }
+
+    match connection.idle(ctx, Some(watch_folder)).await {
+        Ok(v) => v,
+        Err(err) => {
+            connection.trigger_reconnect(ctx).await;
+            warn!(ctx, "{:#}", err);
+            InterruptInfo::new(false)
         }
     }
 }
@@ -280,11 +285,11 @@ async fn simple_imap_loop(
     ctx: Context,
     started: Sender<()>,
     inbox_handlers: ImapConnectionHandlers,
-    folder: Config,
+    folder_config: Config,
 ) {
     use futures::future::FutureExt;
 
-    info!(ctx, "starting simple loop for {}", folder.as_ref());
+    info!(ctx, "starting simple loop for {}", folder_config);
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
@@ -300,7 +305,7 @@ async fn simple_imap_loop(
         }
 
         loop {
-            fetch_idle(&ctx, &mut connection, folder).await;
+            fetch_idle(&ctx, &mut connection, folder_config).await;
         }
     };
 

src/tools.rs

@@ -670,6 +670,18 @@ pub(crate) fn parse_receive_headers(headers: &Headers) -> String {
         .join("\n")
 }
 
+/// If `collection` contains exactly one element, return this element.
+/// Otherwise, return None.
+pub(crate) fn single_value<T>(collection: impl IntoIterator<Item = T>) -> Option<T> {
+    let mut iter = collection.into_iter();
+    if let Some(value) = iter.next() {
+        if iter.next().is_none() {
+            return Some(value);
+        }
+    }
+    None
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]