try to benchmark get_fresh_msg_cnt()

add an index to significantly speed up get_fresh_msg_cnt()

.circleci/config.yml

@@ -138,6 +138,12 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+
   remote_tests_python:
     machine: true
     steps:
@@ -172,6 +178,11 @@ workflows:
     jobs:
       # - cargo_fetch
 
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
       - remote_tests_python:
           filters:
             tags:

ci_scripts/remote_python_packaging.sh

@@ -30,6 +30,7 @@ set +x
 
 ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     set +x -e
+    shopt -s huponexit
     cd $BUILDDIR
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_python.sh

@@ -24,6 +24,11 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     # let's share the target dir with our last run on this branch/job-type
     # cargo will make sure to block/unblock us properly 

ci_scripts/remote_tests_rust.sh

@@ -20,6 +20,9 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     # let's share the target dir with our last run on this branch/job-type
     # cargo will make sure to block/unblock us properly 

deltachat-ffi/deltachat.h

@@ -274,10 +274,12 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `mail_user`    = IMAP-username, guessed if left out
  * - `mail_pw`      = IMAP-password (always needed)
  * - `mail_port`    = IMAP-port, guessed if left out
+ * - `mail_security`= IMAP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `send_server`  = SMTP-server, guessed if left out
  * - `send_user`    = SMTP-user, guessed if left out
  * - `send_pw`      = SMTP-password, guessed if left out
  * - `send_port`    = SMTP-port, guessed if left out
+ * - `send_security`= SMTP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
@@ -4167,6 +4169,44 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_MSG_VIDEOCHAT_INVITATION 70
 
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_SOCKET DC_SOCKET
+ *
+ * These constants configure socket security.
+ * To set socket security, use dc_set_config() with the keys "mail_security" and/or "send_security".
+ * If no socket-configuration is explicitly specified, #DC_SOCKET_AUTO is used.
+ *
+ * @addtogroup DC_SOCKET
+ * @{
+ */
+
+/**
+ * Choose socket security automatically.
+ */
+#define DC_SOCKET_AUTO 0
+
+
+/**
+ * Connect via SSL/TLS.
+ */
+#define DC_SOCKET_SSL 1
+
+
+/**
+ * Connect via STARTTLS.
+ */
+#define DC_SOCKET_STARTTLS 2
+
+
+/**
+ * Connect unencrypted, this should not be used.
+ */
+#define DC_SOCKET_PLAIN 3
 
 /**
  * @}
@@ -4201,54 +4241,11 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_LP_AUTH_NORMAL                0x4
 
 
-/**
- * Connect to IMAP via STARTTLS.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_IMAP_SOCKET_STARTTLS     0x100
-
-
-/**
- * Connect to IMAP via SSL.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_IMAP_SOCKET_SSL          0x200
-
-
-/**
- * Connect to IMAP unencrypted, this should not be used.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_IMAP_SOCKET_PLAIN        0x400
-
-
-/**
- * Connect to SMTP via STARTTLS.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_SMTP_SOCKET_STARTTLS   0x10000
-
-
-/**
- * Connect to SMTP via SSL.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_SMTP_SOCKET_SSL        0x20000
-
-
-/**
- * Connect to SMTP unencrypted, this should not be used.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_SMTP_SOCKET_PLAIN      0x40000 ///<
-
 /**
  * @}
  */
 
 #define DC_LP_AUTH_FLAGS        (DC_LP_AUTH_OAUTH2|DC_LP_AUTH_NORMAL) // if none of these flags are set, the default is chosen
-#define DC_LP_IMAP_SOCKET_FLAGS (DC_LP_IMAP_SOCKET_STARTTLS|DC_LP_IMAP_SOCKET_SSL|DC_LP_IMAP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
-#define DC_LP_SMTP_SOCKET_FLAGS (DC_LP_SMTP_SOCKET_STARTTLS|DC_LP_SMTP_SOCKET_SSL|DC_LP_SMTP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 
 /**
  * @defgroup DC_CERTCK DC_CERTCK

python/src/deltachat/account.py

@@ -216,7 +216,7 @@ class Account(object):
     def create_contact(self, obj, name=None):
         """ create a (new) Contact or return an existing one.
 
-        Calling this method will always resulut in the same
+        Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
         with that e-mail address, it is unblocked and its display
         `name` is updated if specified.

python/src/deltachat/contact.py

@@ -51,6 +51,10 @@ class Contact(object):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
+    def set_blocked(self, block=True):
+        """ Block or unblock a contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, block)
+
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)

python/tests/test_account.py

@@ -986,7 +986,10 @@ class TestOnlineAccount:
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
         lp.sec("sending multi-line non-unicode message from ac1 to ac2")
-        text1 = "hello\nworld"
+        text1 = (
+            "hello\nworld\nthis is a very long message that should be"
+            + " wrapped using format=flowed and unwrapped on the receiver"
+        )
         msg_out = chat.send_text(text1)
         assert not msg_out.is_encrypted()
 
@@ -1402,6 +1405,46 @@ class TestOnlineAccount:
         assert ev.action == "removed"
         assert ev.message.get_sender_contact().addr == ac1_addr
 
+    def test_system_group_msg_from_blocked_user(self, acfactory, lp):
+        """
+        Tests that a blocked user removes you from a group.
+        The message has to be fetched even though the user is blocked
+        to avoid inconsistent group state.
+        Also tests blocking in general.
+        """
+        lp.sec("Create a group chat with ac1 and ac2")
+        (ac1, ac2) = acfactory.get_two_online_accounts()
+        acfactory.introduce_each_other((ac1, ac2))
+        chat_on_ac1 = ac1.create_group_chat("title", contacts=[ac2])
+        chat_on_ac1.send_text("First group message")
+        chat_on_ac2 = ac2._evtracker.wait_next_incoming_message().chat
+
+        lp.sec("ac1 blocks ac2")
+        contact = ac1.create_contact(ac2)
+        contact.set_blocked()
+        assert contact.is_blocked()
+
+        lp.sec("ac2 sends a message to ac1 that does not arrive because it is blocked")
+        ac2.create_chat(ac1).send_text("This will not arrive!")
+
+        lp.sec("ac2 sends a group message to ac1 that arrives")
+        # Groups would be hardly usable otherwise: If you have blocked some
+        # users, they write messages and you only see replies to them without context
+        chat_on_ac2.send_text("This will arrive")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "This will arrive"
+        message_texts = [m.text for m in chat_on_ac1.get_messages()]
+        assert len(message_texts) == 2
+        assert "First group message" in message_texts
+        assert "This will arrive" in message_texts
+
+        lp.sec("ac2 removes ac1 from their group")
+        assert ac1.get_self_contact() in chat_on_ac1.get_contacts()
+        assert contact.is_blocked()
+        chat_on_ac2.remove_contact(ac1)
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert not ac1.get_self_contact() in chat_on_ac1.get_contacts()
+
     def test_set_get_group_image(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1690,6 +1733,46 @@ class TestOnlineAccount:
 
         assert len(imap2.get_all_messages()) == 1
 
+    def test_name_changes(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("displayname", "Account 1")
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        contact = None
+
+        def update_name():
+            """Send a message from ac1 to ac2 to update the name"""
+            nonlocal contact
+            chat12.send_text("Hello")
+            msg = ac2._evtracker.wait_next_incoming_message()
+            contact = msg.get_sender_contact()
+            return contact.name
+
+        assert update_name() == "Account 1"
+
+        ac1.set_config("displayname", "Account 1 revision 2")
+        assert update_name() == "Account 1 revision 2"
+
+        # Explicitly rename contact on ac2 to "Renamed"
+        ac2.create_contact(contact, name="Renamed")
+        assert contact.name == "Renamed"
+
+        # ac1 also renames itself into "Renamed"
+        assert update_name() == "Renamed"
+        ac1.set_config("displayname", "Renamed")
+        assert update_name() == "Renamed"
+
+        # Contact name was set to "Renamed" explicitly before,
+        # so it should not be changed.
+        ac1.set_config("displayname", "Renamed again")
+        updated_name = update_name()
+        if updated_name == "Renamed again":
+            # Known bug, mark as XFAIL
+            pytest.xfail("Contact was renamed after explicit rename")
+        else:
+            # No renames should happen after explicit rename
+            assert updated_name == "Renamed"
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):

src/chat.rs

@@ -375,6 +375,16 @@ impl ChatId {
     }
 
     pub async fn get_fresh_msg_cnt(self, context: &Context) -> usize {
+        // this function is typically used to show a badge counter beside _each_ chatlist item.
+        // to make this as fast as possible, esp. on older devices, we added an combined index over the rows used for querying.
+        // so if you alter the query here, you may want to alter the index over `(state, hidden, chat_id)` in `sql.rs`.
+        //
+        // the impact of the index is significant once the database grows:
+        // - on an older android4 with 18k messages, query-time decreased from 110ms to 2ms
+        // - on an mid-class moto-g or iphone7 with 50k messages, query-time decreased from 26ms or 6ms to 0-1ms
+        // the times are average, no matter if there are fresh messages or not -
+        // and have to be multiplied by the number of items shown at once on the chatlist,
+        // so savings up to 2 seconds are possible on older devices - newer ones will feel "snappier" :)
         context
             .sql
             .query_get_value::<i32>(
@@ -1408,7 +1418,9 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
                 message::guess_msgtype_from_suffix(&blob.to_abs_path())
             {
                 msg.viewtype = better_type;
-                msg.param.set(Param::MimeType, better_mime);
+                if !msg.param.exists(Param::MimeType) {
+                    msg.param.set(Param::MimeType, better_mime);
+                }
             }
         } else if !msg.param.exists(Param::MimeType) {
             if let Some((_, mime)) = message::guess_msgtype_from_suffix(&blob.to_abs_path()) {
@@ -1952,7 +1964,8 @@ pub async fn create_group_chat(
     verified: VerifiedStatus,
     chat_name: impl AsRef<str>,
 ) -> Result<ChatId, Error> {
-    ensure!(!chat_name.as_ref().is_empty(), "Invalid chat name");
+    let chat_name = improve_single_line_input(chat_name);
+    ensure!(!chat_name.is_empty(), "Invalid chat name");
 
     let draft_txt = context
         .stock_string_repl_str(StockMessage::NewGroupDraft, &chat_name)
@@ -1967,7 +1980,7 @@ pub async fn create_group_chat(
             } else {
                 Chattype::Group
             },
-            chat_name.as_ref().to_string(),
+            chat_name,
             grpid,
             time(),
         ],
@@ -2432,17 +2445,18 @@ pub async fn set_chat_name(
     chat_id: ChatId,
     new_name: impl AsRef<str>,
 ) -> Result<(), Error> {
+    let new_name = improve_single_line_input(new_name);
     /* the function only sets the names of group chats; normal chats get their names from the contacts */
     let mut success = false;
 
-    ensure!(!new_name.as_ref().is_empty(), "Invalid name");
+    ensure!(!new_name.is_empty(), "Invalid name");
     ensure!(!chat_id.is_special(), "Invalid chat ID");
 
     let chat = Chat::load_from_db(context, chat_id).await?;
     let mut msg = Message::default();
 
     if real_group_exists(context, chat_id).await {
-        if chat.name == new_name.as_ref() {
+        if chat.name == new_name {
             success = true;
         } else if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
             emit_event!(
@@ -2455,7 +2469,7 @@ pub async fn set_chat_name(
                 .sql
                 .execute(
                     "UPDATE chats SET name=? WHERE id=?;",
-                    paramsv![new_name.as_ref().to_string(), chat_id],
+                    paramsv![new_name.to_string(), chat_id],
                 )
                 .await
                 .is_ok()
@@ -2467,7 +2481,7 @@ pub async fn set_chat_name(
                             .stock_system_msg(
                                 StockMessage::MsgGrpName,
                                 &chat.name,
-                                new_name.as_ref(),
+                                &new_name,
                                 DC_CONTACT_ID_SELF,
                             )
                             .await,
@@ -2872,6 +2886,7 @@ pub(crate) async fn add_info_msg(context: &Context, chat_id: ChatId, text: impl
 mod tests {
     use super::*;
 
+    use crate::chat;
     use crate::contact::Contact;
     use crate::test_utils::*;
 
@@ -3534,4 +3549,30 @@ mod tests {
         chat_id.set_draft(&t.ctx, Some(&mut msg)).await;
         assert!(!chat_id.parent_is_encrypted(&t.ctx).await.unwrap());
     }
+
+    #[async_std::test]
+    async fn test_get_fresh_msg_cnt() {
+        let t = TestContext::new().await;
+        // create 50 chats with 1k messages each
+        let mut chat_ids = Vec::new();
+        for _i in 1..50 {
+            let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo")
+                .await
+                .unwrap();
+            for _i in 1..1000 {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = Some("message text".to_string());
+                chat::send_msg(&t.ctx, chat_id, &mut msg)
+                    .await
+                    .unwrap_or_default();
+            }
+            chat_ids.push(chat_id);
+        }
+        // load all chats 20k times
+        for _i in 1..20_000 {
+            for chat_id in chat_ids.iter() {
+                chat_id.get_fresh_msg_cnt(&t.ctx).await;
+            }
+        }
+    }
 }

src/config.rs

@@ -9,6 +9,7 @@ use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::events::EventType;
+use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
@@ -24,11 +25,13 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    MailSecurity,
     ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SendSecurity,
     SmtpCertificateChecks,
     ServerFlags,
 
@@ -232,6 +235,15 @@ impl Context {
                 });
                 ret
             }
+            Config::Displayname => {
+                let value = value.map(improve_single_line_input);
+                self.sql.set_raw_config(self, key, value.as_deref()).await
+            }
+            Config::DeleteServerAfter => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                job::schedule_resync(self).await;
+                ret
+            }
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }

src/configure/auto_mozilla.rs

@@ -1,32 +1,31 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
+use quick_xml::events::{BytesStart, Event};
+
+use std::io::BufRead;
+use std::str::FromStr;
 
-use crate::constants::*;
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
 #[derive(Debug)]
-struct MozAutoconfigure<'a> {
-    pub in_emailaddr: &'a str,
-    pub in_emaildomain: &'a str,
-    pub in_emaillocalpart: &'a str,
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub tag_server: MozServer,
-    pub tag_config: MozConfigTag,
+struct Server {
+    pub typ: String,
+    pub hostname: String,
+    pub port: u16,
+    pub sockettype: Socket,
+    pub username: String,
 }
 
-#[derive(Debug, PartialEq)]
-enum MozServer {
-    Undefined,
-    Imap,
-    Smtp,
+#[derive(Debug)]
+struct MozAutoconfigure {
+    pub incoming_servers: Vec<Server>,
+    pub outgoing_servers: Vec<Server>,
 }
 
 #[derive(Debug)]
@@ -38,10 +37,147 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
-    let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+impl Default for MozConfigTag {
+    fn default() -> Self {
+        Self::Undefined
+    }
+}
 
+impl FromStr for MozConfigTag {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.trim().to_lowercase().as_ref() {
+            "hostname" => Ok(MozConfigTag::Hostname),
+            "port" => Ok(MozConfigTag::Port),
+            "sockettype" => Ok(MozConfigTag::Sockettype),
+            "username" => Ok(MozConfigTag::Username),
+            _ => Err(()),
+        }
+    }
+}
+
+/// Parses a single IncomingServer or OutgoingServer section.
+fn parse_server<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+    server_event: &BytesStart,
+) -> Result<Option<Server>, quick_xml::Error> {
+    let end_tag = String::from_utf8_lossy(server_event.name())
+        .trim()
+        .to_lowercase();
+
+    let typ = server_event
+        .attributes()
+        .find(|attr| {
+            attr.as_ref()
+                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
+                .unwrap_or_default()
+        })
+        .map(|typ| {
+            typ.unwrap()
+                .unescape_and_decode_value(reader)
+                .unwrap_or_default()
+                .to_lowercase()
+        })
+        .unwrap_or_default();
+
+    let mut hostname = None;
+    let mut port = None;
+    let mut sockettype = Socket::Automatic;
+    let mut username = None;
+
+    let mut tag_config = MozConfigTag::Undefined;
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                tag_config = String::from_utf8_lossy(event.name())
+                    .parse()
+                    .unwrap_or_default();
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == end_tag {
+                    break;
+                }
+            }
+            Event::Text(ref event) => {
+                let val = event
+                    .unescape_and_decode(reader)
+                    .unwrap_or_default()
+                    .trim()
+                    .to_owned();
+
+                match tag_config {
+                    MozConfigTag::Hostname => hostname = Some(val),
+                    MozConfigTag::Port => port = Some(val.parse().unwrap_or_default()),
+                    MozConfigTag::Username => username = Some(val),
+                    MozConfigTag::Sockettype => {
+                        sockettype = match val.to_lowercase().as_ref() {
+                            "ssl" => Socket::SSL,
+                            "starttls" => Socket::STARTTLS,
+                            "plain" => Socket::Plain,
+                            _ => Socket::Automatic,
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+    }
+
+    if let (Some(hostname), Some(port), Some(username)) = (hostname, port, username) {
+        Ok(Some(Server {
+            typ,
+            hostname,
+            port,
+            sockettype,
+            username,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<MozAutoconfigure, quick_xml::Error> {
+    let mut incoming_servers = Vec::new();
+    let mut outgoing_servers = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == "incomingserver" {
+                    if let Some(incoming_server) = parse_server(reader, event)? {
+                        incoming_servers.push(incoming_server);
+                    }
+                } else if tag == "outgoingserver" {
+                    if let Some(outgoing_server) = parse_server(reader, event)? {
+                        outgoing_servers.push(outgoing_server);
+                    }
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+        buf.clear();
+    }
+
+    Ok(MozAutoconfigure {
+        incoming_servers,
+        outgoing_servers,
+    })
+}
+
+/// Parses XML and fills in address and domain placeholders.
+fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoconfigure, Error> {
     // Split address into local part and domain part.
     let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
     let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
@@ -49,59 +185,78 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
         _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
     };
 
-    let mut moz_ac = MozAutoconfigure {
-        in_emailaddr,
-        in_emaildomain,
-        in_emaillocalpart,
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        tag_server: MozServer::Undefined,
-        tag_config: MozConfigTag::Undefined,
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
+    reader.trim_text(true);
+
+    let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })?;
+
+    let fill_placeholders = |val: &str| -> String {
+        val.replace("%EMAILADDRESS%", in_emailaddr)
+            .replace("%EMAILLOCALPART%", in_emaillocalpart)
+            .replace("%EMAILDOMAIN%", in_emaildomain)
     };
 
-    let mut buf = Vec::new();
-    loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                moz_autoconfigure_starttag_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::End(ref e) => moz_autoconfigure_endtag_cb(e, &mut moz_ac),
-            quick_xml::events::Event::Text(ref e) => {
-                moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::Eof => break,
-            _ => (),
+    let fill_server_placeholders = |server: Server| -> Server {
+        Server {
+            typ: server.typ,
+            hostname: fill_placeholders(&server.hostname),
+            port: server.port,
+            sockettype: server.sockettype,
+            username: fill_placeholders(&server.username),
         }
-        buf.clear();
-    }
+    };
 
-    if moz_ac.out.mail_server.is_empty()
-        || moz_ac.out.mail_port == 0
-        || moz_ac.out.send_server.is_empty()
-        || moz_ac.out.send_port == 0
-    {
-        Err(Error::IncompleteAutoconfig(moz_ac.out))
-    } else {
-        Ok(moz_ac.out)
-    }
+    Ok(MozAutoconfigure {
+        incoming_servers: moz_ac
+            .incoming_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+        outgoing_servers: moz_ac
+            .outgoing_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+    })
 }
 
-pub async fn moz_autoconfigure(
+/// Parses XML into `ServerParams` vector.
+fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerParams>, Error> {
+    let moz_ac = parse_xml_with_address(in_emailaddr, xml_raw)?;
+
+    let res = moz_ac
+        .incoming_servers
+        .into_iter()
+        .chain(moz_ac.outgoing_servers.into_iter())
+        .filter_map(|server| {
+            let protocol = match server.typ.as_ref() {
+                "imap" => Some(Protocol::IMAP),
+                "smtp" => Some(Protocol::SMTP),
+                _ => None,
+            };
+            Some(ServerParams {
+                protocol: protocol?,
+                socket: server.sockettype,
+                hostname: server.hostname,
+                port: server.port,
+                username: server.username,
+            })
+        })
+        .collect();
+    Ok(res)
+}
+
+pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_xml(&param_in.addr, &xml_raw);
+    let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,
@@ -111,212 +266,60 @@ pub async fn moz_autoconfigure(
     res
 }
 
-fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
-    event: &BytesText,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let val = event.unescape_and_decode(reader).unwrap_or_default();
-
-    let addr = moz_ac.in_emailaddr;
-    let email_local = moz_ac.in_emaillocalpart;
-    let email_domain = moz_ac.in_emaildomain;
-
-    let val = val
-        .trim()
-        .replace("%EMAILADDRESS%", addr)
-        .replace("%EMAILLOCALPART%", email_local)
-        .replace("%EMAILDOMAIN%", email_domain);
-
-    match moz_ac.tag_server {
-        MozServer::Imap => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
-            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.mail_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Smtp => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.send_server = val,
-            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.send_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Undefined => {}
-    }
-}
-
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        if moz_ac.tag_server == MozServer::Imap {
-            moz_ac.out_imap_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        if moz_ac.tag_server == MozServer::Smtp {
-            moz_ac.out_smtp_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else {
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    }
-}
-
-fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
-    event: &BytesStart,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        moz_ac.tag_server = if let Some(typ) = event.attributes().find(|attr| {
-            attr.as_ref()
-                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
-                .unwrap_or_default()
-        }) {
-            let typ = typ
-                .unwrap()
-                .unescape_and_decode_value(reader)
-                .unwrap_or_default()
-                .to_lowercase();
-
-            if typ == "imap" && !moz_ac.out_imap_set {
-                MozServer::Imap
-            } else {
-                MozServer::Undefined
-            }
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if !moz_ac.out_smtp_set {
-            MozServer::Smtp
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "hostname" {
-        moz_ac.tag_config = MozConfigTag::Hostname;
-    } else if tag == "port" {
-        moz_ac.tag_config = MozConfigTag::Port;
-    } else if tag == "sockettype" {
-        moz_ac.tag_config = MozConfigTag::Sockettype;
-    } else if tag == "username" {
-        moz_ac.tag_config = MozConfigTag::Username;
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
     fn test_parse_outlook_autoconfig() {
-        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
-        let xml_raw =
-"<clientConfig version=\"1.1\">
-  <emailProvider id=\"outlook.com\">
-    <domain>hotmail.com</domain>
-    <domain>hotmail.co.uk</domain>
-    <domain>hotmail.co.jp</domain>
-    <domain>hotmail.com.br</domain>
-    <domain>hotmail.de</domain>
-    <domain>hotmail.fr</domain>
-    <domain>hotmail.it</domain>
-    <domain>hotmail.es</domain>
-    <domain>live.com</domain>
-    <domain>live.co.uk</domain>
-    <domain>live.co.jp</domain>
-    <domain>live.de</domain>
-    <domain>live.fr</domain>
-    <domain>live.it</domain>
-    <domain>live.jp</domain>
-    <domain>msn.com</domain>
-    <domain>outlook.com</domain>
-    <displayName>Outlook.com (Microsoft)</displayName>
-    <displayShortName>Outlook</displayShortName>
-    <incomingServer type=\"exchange\">
-      <hostname>outlook.office365.com</hostname>
-      <port>443</port>
-      <username>%EMAILADDRESS%</username>
-      <socketType>SSL</socketType>
-      <authentication>OAuth2</authentication>
-      <owaURL>https://outlook.office365.com/owa/</owaURL>
-      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
-      <useGlobalPreferredServer>true</useGlobalPreferredServer>
-    </incomingServer>
-    <incomingServer type=\"imap\">
-      <hostname>outlook.office365.com</hostname>
-      <port>993</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </incomingServer>
-    <incomingServer type=\"pop3\">
-      <hostname>outlook.office365.com</hostname>
-      <port>995</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-      <pop3>
-        <leaveMessagesOnServer>true</leaveMessagesOnServer>
-        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
-      </pop3>
-    </incomingServer>
-    <outgoingServer type=\"smtp\">
-      <hostname>smtp.office365.com</hostname>
-      <port>587</port>
-      <socketType>STARTTLS</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
-    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
-      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
-    </documentation>
-  </emailProvider>
-  <webMail>
-    <loginPage url=\"https://www.outlook.com/\"/>
-    <loginPageInfo url=\"https://www.outlook.com/\">
-      <username>%EMAILADDRESS%</username>
-      <usernameField id=\"i0116\" name=\"login\"/>
-      <passwordField id=\"i0118\" name=\"passwd\"/>
-      <loginButton id=\"idSIButton9\" name=\"SI\"/>
-    </loginPageInfo>
-  </webMail>
-</clientConfig>";
-        let res = parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
-        assert_eq!(res.mail_server, "outlook.office365.com");
-        assert_eq!(res.mail_port, 993);
-        assert_eq!(res.send_server, "smtp.office365.com");
-        assert_eq!(res.send_port, 587);
+        let xml_raw = include_str!("../../test-data/autoconfig/outlook.com.xml");
+        let res = parse_serverparams("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res[0].protocol, Protocol::IMAP);
+        assert_eq!(res[0].hostname, "outlook.office365.com");
+        assert_eq!(res[0].port, 993);
+        assert_eq!(res[1].protocol, Protocol::SMTP);
+        assert_eq!(res[1].hostname, "smtp.office365.com");
+        assert_eq!(res[1].port, 587);
+    }
+
+    #[test]
+    fn test_parse_lakenet_autoconfig() {
+        let xml_raw = include_str!("../../test-data/autoconfig/lakenet.ch.xml");
+        let res =
+            parse_xml_with_address("example@lakenet.ch", xml_raw).expect("XML parsing failed");
+
+        assert_eq!(res.incoming_servers.len(), 4);
+
+        assert_eq!(res.incoming_servers[0].typ, "imap");
+        assert_eq!(res.incoming_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[0].port, 993);
+        assert_eq!(res.incoming_servers[0].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[0].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[1].typ, "imap");
+        assert_eq!(res.incoming_servers[1].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[1].port, 143);
+        assert_eq!(res.incoming_servers[1].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[1].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[2].typ, "pop3");
+        assert_eq!(res.incoming_servers[2].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[2].port, 995);
+        assert_eq!(res.incoming_servers[2].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[2].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[3].typ, "pop3");
+        assert_eq!(res.incoming_servers[3].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[3].port, 110);
+        assert_eq!(res.incoming_servers[3].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[3].username, "example@lakenet.ch");
+
+        assert_eq!(res.outgoing_servers.len(), 1);
+
+        assert_eq!(res.outgoing_servers[0].typ, "smtp");
+        assert_eq!(res.outgoing_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.outgoing_servers[0].port, 587);
+        assert_eq!(res.outgoing_servers[0].sockettype, Socket::STARTTLS);
+        assert_eq!(res.outgoing_servers[0].username, "example@lakenet.ch");
     }
 }

src/configure/auto_outlook.rs

@@ -1,122 +1,194 @@
-//! Outlook's Autodiscover
+//! # Outlook's Autodiscover
+//!
+//! This module implements autoconfiguration via POX (Plain Old XML) interface to Autodiscover
+//! Service. Newer SOAP interface, introduced in Exchange 2010, is not used.
 
-use quick_xml::events::BytesEnd;
+use quick_xml::events::Event;
+
+use std::io::BufRead;
 
-use crate::constants::*;
 use crate::context::Context;
-use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
-struct OutlookAutodiscover {
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub config_type: Option<String>,
-    pub config_server: String,
-    pub config_port: i32,
-    pub config_ssl: String,
-    pub config_redirecturl: Option<String>,
+/// Result of parsing a single `Protocol` tag.
+///
+/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
+#[derive(Debug)]
+struct ProtocolTag {
+    /// Server type, such as "IMAP", "SMTP" or "POP3".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
+    pub typ: String,
+
+    /// Server identifier, hostname or IP address for IMAP and SMTP.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
+    pub server: String,
+
+    /// Network port.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
+    pub port: u16,
+
+    /// Whether connection should be secure, "on" or "off", default is "on".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
+    pub ssl: bool,
 }
 
 enum ParsingResult {
-    LoginParam(LoginParam),
+    Protocols(Vec<ProtocolTag>),
+
+    /// XML redirect via `RedirectUrl` tag.
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut outlk_ad = OutlookAutodiscover {
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        config_type: None,
-        config_server: String::new(),
-        config_port: 0,
-        config_ssl: String::new(),
-        config_redirecturl: None,
-    };
-
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
-    reader.trim_text(true);
+/// Parses a single Protocol section.
+fn parse_protocol<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<Option<ProtocolTag>, quick_xml::Error> {
+    let mut protocol_type = None;
+    let mut protocol_server = None;
+    let mut protocol_port = None;
+    let mut protocol_ssl = true;
 
     let mut buf = Vec::new();
 
     let mut current_tag: Option<String> = None;
-
     loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
-
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                current_tag = Some(String::from_utf8_lossy(event.name()).trim().to_lowercase());
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
                 if tag == "protocol" {
-                    outlk_ad.config_type = None;
-                    outlk_ad.config_server = String::new();
-                    outlk_ad.config_port = 0;
-                    outlk_ad.config_ssl = String::new();
-                    outlk_ad.config_redirecturl = None;
-
+                    break;
+                }
+                if Some(tag) == current_tag {
                     current_tag = None;
-                } else {
-                    current_tag = Some(tag);
                 }
             }
-            quick_xml::events::Event::End(ref e) => {
-                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
-                current_tag = None;
-            }
-            quick_xml::events::Event::Text(ref e) => {
+            Event::Text(ref e) => {
                 let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
-                        "type" => {
-                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        "type" => protocol_type = Some(val.trim().to_string()),
+                        "server" => protocol_server = Some(val.trim().to_string()),
+                        "port" => protocol_port = Some(val.trim().parse().unwrap_or_default()),
+                        "ssl" => {
+                            protocol_ssl = match val.trim() {
+                                "on" => true,
+                                "off" => false,
+                                _ => true,
+                            }
                         }
-                        "server" => outlk_ad.config_server = val.trim().to_string(),
-                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
-                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
-                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
                         _ => {}
                     };
                 }
             }
-            quick_xml::events::Event::Eof => break,
+            Event::Eof => break,
+            _ => {}
+        }
+    }
+
+    if let (Some(protocol_type), Some(protocol_server), Some(protocol_port)) =
+        (protocol_type, protocol_server, protocol_port)
+    {
+        Ok(Some(ProtocolTag {
+            typ: protocol_type,
+            server: protocol_server,
+            port: protocol_port,
+            ssl: protocol_ssl,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Parses `RedirectUrl` tag.
+fn parse_redirecturl<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<String, quick_xml::Error> {
+    let mut buf = Vec::new();
+    match reader.read_event(&mut buf)? {
+        Event::Text(ref e) => {
+            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            Ok(val.trim().to_string())
+        }
+        _ => Ok("".to_string()),
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<ParsingResult, quick_xml::Error> {
+    let mut protocols = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref e) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
+
+                if tag == "protocol" {
+                    if let Some(protocol) = parse_protocol(reader)? {
+                        protocols.push(protocol);
+                    }
+                } else if tag == "redirecturl" {
+                    let redirecturl = parse_redirecturl(reader)?;
+                    return Ok(ParsingResult::RedirectUrl(redirecturl));
+                }
+            }
+            Event::Eof => break,
             _ => (),
         }
         buf.clear();
     }
 
-    // XML redirect via redirecturl
-    let res = if outlk_ad.config_redirecturl.is_none()
-        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
-    {
-        if outlk_ad.out.mail_server.is_empty()
-            || outlk_ad.out.mail_port == 0
-            || outlk_ad.out.send_server.is_empty()
-            || outlk_ad.out.send_port == 0
-        {
-            return Err(Error::IncompleteAutoconfig(outlk_ad.out));
-        }
-        ParsingResult::LoginParam(outlk_ad.out)
-    } else {
-        ParsingResult::RedirectUrl(outlk_ad.config_redirecturl.unwrap())
-    };
-    Ok(res)
+    Ok(ParsingResult::Protocols(protocols))
 }
 
-pub async fn outlk_autodiscover(
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })
+}
+
+fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
+    protocols
+        .into_iter()
+        .filter_map(|protocol| {
+            Some(ServerParams {
+                protocol: match protocol.typ.to_lowercase().as_ref() {
+                    "imap" => Some(Protocol::IMAP),
+                    "smtp" => Some(Protocol::SMTP),
+                    _ => None,
+                }?,
+                socket: match protocol.ssl {
+                    true => Socket::Automatic,
+                    false => Socket::Plain,
+                },
+                hostname: protocol.server,
+                port: protocol.port,
+                username: String::new(),
+            })
+        })
+        .collect()
+}
+
+pub(crate) async fn outlk_autodiscover(
     context: &Context,
     url: &str,
-    _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
@@ -127,43 +199,12 @@ pub async fn outlk_autodiscover(
         }
         match res? {
             ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
-            ParsingResult::LoginParam(login_param) => return Ok(login_param),
-        }
-    }
-    Err(Error::RedirectionError)
-}
-
-fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodiscover) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "protocol" {
-        if let Some(type_) = &outlk_ad.config_type {
-            let port = outlk_ad.config_port;
-            let ssl_on = outlk_ad.config_ssl == "on";
-            let ssl_off = outlk_ad.config_ssl == "off";
-            if type_ == "imap" && !outlk_ad.out_imap_set {
-                outlk_ad.out.mail_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.mail_port = port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_imap_set = true
-            } else if type_ == "smtp" && !outlk_ad.out_smtp_set {
-                outlk_ad.out.send_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.send_port = outlk_ad.config_port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_smtp_set = true
+            ParsingResult::Protocols(protocols) => {
+                return Ok(protocols_to_serverparams(protocols));
             }
         }
     }
+    Err(Error::RedirectionError)
 }
 
 #[cfg(test)]
@@ -184,16 +225,13 @@ mod tests {
     </Response>
   </Autodiscover>
  ").expect("XML is not parsed successfully");
-        match res {
-            ParsingResult::LoginParam(_lp) => {
-                panic!("redirecturl is not found");
-            }
-            ParsingResult::RedirectUrl(url) => {
-                assert_eq!(
-                    url,
-                    "https://mail.example.com/autodiscover/autodiscover.xml"
-                );
-            }
+        if let ParsingResult::RedirectUrl(url) = res {
+            assert_eq!(
+                url,
+                "https://mail.example.com/autodiscover/autodiscover.xml"
+            );
+        } else {
+            panic!("redirecturl is not found");
         }
     }
 
@@ -228,11 +266,16 @@ mod tests {
         .expect("XML is not parsed successfully");
 
         match res {
-            ParsingResult::LoginParam(lp) => {
-                assert_eq!(lp.mail_server, "example.com");
-                assert_eq!(lp.mail_port, 993);
-                assert_eq!(lp.send_server, "smtp.example.com");
-                assert_eq!(lp.send_port, 25);
+            ParsingResult::Protocols(protocols) => {
+                assert_eq!(protocols[0].typ, "IMAP");
+                assert_eq!(protocols[0].server, "example.com");
+                assert_eq!(protocols[0].port, 993);
+                assert_eq!(protocols[0].ssl, true);
+
+                assert_eq!(protocols[1].typ, "SMTP");
+                assert_eq!(protocols[1].server, "smtp.example.com");
+                assert_eq!(protocols[1].port, 25);
+                assert_eq!(protocols[1].ssl, false);
             }
             ParsingResult::RedirectUrl(_) => {
                 panic!("RedirectUrl is not expected");

src/configure/mod.rs

@@ -3,6 +3,7 @@
 mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
+mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
@@ -13,14 +14,16 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::imap::Imap;
-use crate::login_param::{CertificateChecks, LoginParam};
+use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
 use crate::oauth2::*;
+use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::{chat, e2ee, provider};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
+use server_params::ServerParams;
 
 macro_rules! progress {
     ($context:tt, $progress:expr) => {
@@ -115,20 +118,37 @@ impl Context {
 }
 
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
-    let mut param_autoconfig: Option<LoginParam> = None;
-    let mut keep_flags = 0;
-
-    // Read login parameters from the database
     progress!(ctx, 1);
+
+    // Check basic settings.
     ensure!(!param.addr.is_empty(), "Please enter an email address.");
 
+    // Only check for IMAP password, SMTP password is an "advanced" setting.
+    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
+    if param.smtp.password.is_empty() {
+        param.smtp.password = param.imap.password.clone()
+    }
+
+    // Normalize authentication flags.
+    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
+        DC_LP_AUTH_OAUTH2 => true,
+        DC_LP_AUTH_NORMAL => false,
+        _ => false,
+    };
+    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
+    param.server_flags |= if oauth2 {
+        DC_LP_AUTH_OAUTH2 as i32
+    } else {
+        DC_LP_AUTH_NORMAL as i32
+    };
+
     // Step 1: Load the parameters and check email-address and password
 
-    if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
+    if oauth2 {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
-        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
+        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
             .await
             .and_then(|e| e.parse().ok())
         {
@@ -149,95 +169,106 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // Step 2: Autoconfig
     progress!(ctx, 200);
 
-    // param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then
-    // param.send_pw.is_empty()   -- the password cannot be auto-configured and is no criterion for
-    //                               autoconfig or not
-    if param.mail_server.is_empty()
-        && param.mail_port == 0
-        && param.send_server.is_empty()
-        && param.send_port == 0
-        && param.send_user.is_empty()
-        && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
+    let param_autoconfig;
+    if param.imap.server.is_empty()
+        && param.imap.port == 0
+        && param.imap.security == Socket::Automatic
+        && param.imap.user.is_empty()
+        && param.smtp.server.is_empty()
+        && param.smtp.port == 0
+        && param.smtp.security == Socket::Automatic
+        && param.smtp.user.is_empty()
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
-        keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
-        if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
-            // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
-            param_autoconfig = Some(new_param);
-        }
 
-        if param_autoconfig.is_none() {
+        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
+            param_autoconfig = Some(servers);
+        } else {
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
+    } else {
+        param_autoconfig = None;
     }
 
-    // C.  Do we have any autoconfig result?
     progress!(ctx, 500);
-    if let Some(ref cfg) = param_autoconfig {
-        info!(ctx, "Got autoconfig: {}", &cfg);
-        if !cfg.mail_user.is_empty() {
-            param.mail_user = cfg.mail_user.clone();
-        }
-        // all other values are always NULL when entering autoconfig
-        param.mail_server = cfg.mail_server.clone();
-        param.mail_port = cfg.mail_port;
-        param.send_server = cfg.send_server.clone();
-        param.send_port = cfg.send_port;
-        param.send_user = cfg.send_user.clone();
-        param.server_flags = cfg.server_flags;
-        // although param_autoconfig's data are no longer needed from,
-        // it is used to later to prevent trying variations of port/server/logins
-    }
-    param.server_flags |= keep_flags;
 
-    // Step 3: Fill missing fields with defaults
-    if param.send_user.is_empty() {
-        param.send_user = param.mail_user.clone();
-    }
-    if param.send_pw.is_empty() {
-        param.send_pw = param.mail_pw.clone()
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-        param.server_flags |= DC_LP_AUTH_NORMAL as i32
-    }
-
-    // do we have a complete configuration?
-    ensure!(
-        !param.mail_pw.is_empty() && !param.send_pw.is_empty(),
-        "Account settings incomplete."
-    );
+    let servers: Vec<ServerParams> = param_autoconfig
+        .unwrap_or_else(|| {
+            vec![
+                ServerParams {
+                    protocol: Protocol::IMAP,
+                    hostname: param.imap.server.clone(),
+                    port: param.imap.port,
+                    socket: param.imap.security,
+                    username: param.imap.user.clone(),
+                },
+                ServerParams {
+                    protocol: Protocol::SMTP,
+                    hostname: param.smtp.server.clone(),
+                    port: param.smtp.port,
+                    socket: param.smtp.security,
+                    username: param.smtp.user.clone(),
+                },
+            ]
+        })
+        .into_iter()
+        // The order of expansion is important: ports are expanded the
+        // last, so they are changed the first. Username is only
+        // changed if default value (address with domain) didn't work
+        // for all available hosts and ports.
+        .flat_map(|params| params.expand_usernames(&param.addr).into_iter())
+        .flat_map(|params| params.expand_hostnames(&param_domain).into_iter())
+        .flat_map(|params| params.expand_ports().into_iter())
+        .collect();
 
+    // Configure IMAP
     progress!(ctx, 600);
-    // try to connect to IMAP - if we did not got an autoconfig,
-    // do some further tries with different settings and username variations
     let (_s, r) = async_std::sync::channel(1);
     let mut imap = Imap::new(r);
 
-    if param_autoconfig.is_some() {
-        if try_imap_one_param(ctx, &param, &mut imap).await.is_err() {
-            bail!("IMAP autoconfig did not succeed");
-        }
-    } else {
-        *param = try_imap_hostnames(ctx, param.clone(), &mut imap).await?;
-    }
-    progress!(ctx, 750);
+    let mut imap_configured = false;
+    for imap_server in servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::IMAP)
+    {
+        param.imap.user = imap_server.username.clone();
+        param.imap.server = imap_server.hostname.clone();
+        param.imap.port = imap_server.port;
+        param.imap.security = imap_server.socket;
 
-    let mut smtp = Smtp::new();
-    if param_autoconfig.is_some() {
-        if try_smtp_one_param(ctx, &param, &mut smtp).await.is_err() {
-            bail!("SMTP autoconfig did not succeed");
+        if try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+            imap_configured = true;
+            break;
         }
-    } else {
-        *param = try_smtp_hostnames(ctx, param.clone(), &mut smtp).await?;
     }
+    if !imap_configured {
+        bail!("IMAP autoconfig did not succeed");
+    }
+
+    // Configure SMTP
+    progress!(ctx, 750);
+    let mut smtp = Smtp::new();
+
+    let mut smtp_configured = false;
+    for smtp_server in servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::SMTP)
+    {
+        param.smtp.user = smtp_server.username.clone();
+        param.smtp.server = smtp_server.hostname.clone();
+        param.smtp.port = smtp_server.port;
+        param.smtp.security = smtp_server.socket;
+
+        if try_smtp_one_param(ctx, &param.smtp, &param.addr, oauth2, &mut smtp).await {
+            smtp_configured = true;
+            break;
+        }
+    }
+    if !smtp_configured {
+        bail!("SMTP autoconfig did not succeed");
+    }
+
     progress!(ctx, 900);
 
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
@@ -306,8 +337,8 @@ impl AutoconfigSource {
             AutoconfigSource {
                 provider: AutoconfigProvider::Outlook,
                 url: format!(
-                    "https://{}{}/autodiscover/autodiscover.xml",
-                    "autodiscover.", domain
+                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
+                    domain
                 ),
             },
             // always SSL for Thunderbird's database
@@ -318,10 +349,10 @@ impl AutoconfigSource {
         ]
     }
 
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<LoginParam> {
+    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
         let params = match self.provider {
             AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url, &param).await?,
+            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
         };
 
         Ok(params)
@@ -337,7 +368,7 @@ async fn get_autoconfig(
     param: &LoginParam,
     param_domain: &str,
     param_addr_urlencoded: &str,
-) -> Option<LoginParam> {
+) -> Option<Vec<ServerParams>> {
     let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
 
     let mut progress = 300;
@@ -353,301 +384,97 @@ async fn get_autoconfig(
     None
 }
 
-fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
+fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
     info!(
         context,
         "checking internal provider-info for offline autoconfig"
     );
 
-    if let Some(provider) = provider::get_provider_info(&param.addr) {
+    if let Some(provider) = provider::get_provider_info(&addr) {
         match provider.status {
             provider::Status::OK | provider::Status::PREPARATION => {
-                let imap = provider.get_imap_server();
-                let smtp = provider.get_smtp_server();
-                // clippy complains about these is_some()/unwrap() settings,
-                // however, rewriting the code to "if let" would make things less obvious,
-                // esp. if we allow more combinations of servers (pop, jmap).
-                // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
-                if let Some(imap) = imap {
-                    if let Some(smtp) = smtp {
-                        let mut p = LoginParam::new();
-                        p.addr = param.addr.clone();
-
-                        p.mail_server = imap.hostname.to_string();
-                        p.mail_user = imap.apply_username_pattern(param.addr.clone());
-                        p.mail_port = imap.port as i32;
-                        p.imap_certificate_checks = CertificateChecks::Automatic;
-                        p.server_flags |= match imap.socket {
-                            provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
-                            provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
-                        };
-
-                        p.send_server = smtp.hostname.to_string();
-                        p.send_user = smtp.apply_username_pattern(param.addr.clone());
-                        p.send_port = smtp.port as i32;
-                        p.smtp_certificate_checks = CertificateChecks::Automatic;
-                        p.server_flags |= match smtp.socket {
-                            provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
-                            provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
-                        };
-
-                        info!(context, "offline autoconfig found: {}", p);
-                        return Some(p);
-                    }
+                if provider.server.is_empty() {
+                    info!(context, "offline autoconfig found, but no servers defined");
+                    None
+                } else {
+                    info!(context, "offline autoconfig found");
+                    let servers = provider
+                        .server
+                        .iter()
+                        .map(|s| ServerParams {
+                            protocol: s.protocol,
+                            socket: s.socket,
+                            hostname: s.hostname.to_string(),
+                            port: s.port,
+                            username: match s.username_pattern {
+                                UsernamePattern::EMAIL => addr.to_string(),
+                                UsernamePattern::EMAILLOCALPART => {
+                                    if let Some(at) = addr.find('@') {
+                                        addr.split_at(at).0.to_string()
+                                    } else {
+                                        addr.to_string()
+                                    }
+                                }
+                            },
+                        })
+                        .collect();
+                    Some(servers)
                 }
-                info!(context, "offline autoconfig found, but no servers defined");
-                return None;
             }
             provider::Status::BROKEN => {
                 info!(context, "offline autoconfig found, provider is broken");
-                return None;
+                None
             }
         }
-    }
-    info!(context, "no offline autoconfig found");
-    None
-}
-
-async fn try_imap_hostnames(
-    context: &Context,
-    mut param: LoginParam,
-    imap: &mut Imap,
-) -> Result<LoginParam> {
-    if param.mail_server.is_empty() {
-        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
-        let param_domain = parsed.domain;
-
-        param.mail_server = param_domain.clone();
-        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
-            return Ok(param);
-        }
-
-        progress!(context, 650);
-        param.mail_server = "imap.".to_string() + &param_domain;
-        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
-            return Ok(param);
-        }
-
-        progress!(context, 700);
-        param.mail_server = "mail.".to_string() + &param_domain;
-        try_imap_ports(context, param, imap).await
     } else {
-        progress!(context, 700);
-        try_imap_ports(context, param, imap).await
+        info!(context, "no offline autoconfig found");
+        None
     }
 }
 
-// Try various IMAP ports and corresponding TLS settings.
-async fn try_imap_ports(
+async fn try_imap_one_param(
     context: &Context,
-    mut param: LoginParam,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
     imap: &mut Imap,
-) -> Result<LoginParam> {
-    // Try to infer port from socket security.
-    if param.mail_port == 0 {
-        if 0 != param.server_flags & DC_LP_IMAP_SOCKET_SSL {
-            param.mail_port = 993
-        }
-        if 0 != param.server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN) {
-            param.mail_port = 143
-        }
-    }
-
-    if param.mail_port == 0 {
-        // Neither port nor security is set.
-        //
-        // Try common secure combinations.
-
-        // Try TLS over port 993
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
-        param.mail_port = 993;
-        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
-            return Ok(login_param);
-        }
-
-        // Try STARTTLS over port 143
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
-        param.mail_port = 143;
-        try_imap_usernames(context, param, imap).await
-    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
-        // Try TLS over user-provided port.
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
-        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
-            return Ok(login_param);
-        }
-
-        // Try STARTTLS over user-provided port.
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
-        try_imap_usernames(context, param, imap).await
-    } else {
-        try_imap_usernames(context, param, imap).await
-    }
-}
-
-async fn try_imap_usernames(
-    context: &Context,
-    mut param: LoginParam,
-    imap: &mut Imap,
-) -> Result<LoginParam> {
-    if param.mail_user.is_empty() {
-        param.mail_user = param.addr.clone();
-        if let Err(e) = try_imap_one_param(context, &param, imap).await {
-            if let Some(at) = param.mail_user.find('@') {
-                param.mail_user = param.mail_user.split_at(at).0.to_string();
-                try_imap_one_param(context, &param, imap).await?;
-            } else {
-                return Err(e);
-            }
-        }
-        Ok(param)
-    } else {
-        try_imap_one_param(context, &param, imap).await?;
-        Ok(param)
-    }
-}
-
-async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Imap) -> Result<()> {
+) -> bool {
     let inf = format!(
-        "imap: {}@{}:{} flags=0x{:x} certificate_checks={}",
-        param.mail_user,
-        param.mail_server,
-        param.mail_port,
-        param.server_flags,
-        param.imap_certificate_checks
+        "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if imap.connect(context, &param).await {
+    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        false
+    } else {
         info!(context, "success: {}", inf);
-        return Ok(());
+        true
     }
-
-    bail!("Could not connect: {}", inf);
 }
 
-async fn try_smtp_hostnames(
+async fn try_smtp_one_param(
     context: &Context,
-    mut param: LoginParam,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
     smtp: &mut Smtp,
-) -> Result<LoginParam> {
-    if param.send_server.is_empty() {
-        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
-        let param_domain = parsed.domain;
-
-        param.send_server = param_domain.clone();
-        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
-            return Ok(param);
-        }
-
-        progress!(context, 800);
-        param.send_server = "smtp.".to_string() + &param_domain;
-        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
-            return Ok(param);
-        }
-
-        progress!(context, 850);
-        param.send_server = "mail.".to_string() + &param_domain;
-        try_smtp_ports(context, param, smtp).await
-    } else {
-        progress!(context, 850);
-        try_smtp_ports(context, param, smtp).await
-    }
-}
-
-// Try various SMTP ports and corresponding TLS settings.
-async fn try_smtp_ports(
-    context: &Context,
-    mut param: LoginParam,
-    smtp: &mut Smtp,
-) -> Result<LoginParam> {
-    // Try to infer port from socket security.
-    if param.send_port == 0 {
-        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-            param.send_port = 587;
-        }
-        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-            param.send_port = 25;
-        }
-        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_SSL as i32 {
-            param.send_port = 465;
-        }
-    }
-
-    if param.send_port == 0 {
-        // Neither port nor security is set.
-        //
-        // Try common secure combinations.
-
-        // Try TLS over port 465.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
-        param.send_port = 465;
-        if let Ok(login_param) = try_smtp_usernames(context, param.clone(), smtp).await {
-            return Ok(login_param);
-        }
-
-        // Try STARTTLS over port 587.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-        param.send_port = 587;
-        try_smtp_usernames(context, param, smtp).await
-    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
-        // Try TLS over user-provided port.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
-        if let Ok(param) = try_smtp_usernames(context, param.clone(), smtp).await {
-            return Ok(param);
-        }
-
-        // Try STARTTLS over user-provided port.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-        try_smtp_usernames(context, param, smtp).await
-    } else {
-        try_smtp_usernames(context, param, smtp).await
-    }
-}
-
-async fn try_smtp_usernames(
-    context: &Context,
-    mut param: LoginParam,
-    smtp: &mut Smtp,
-) -> Result<LoginParam> {
-    if param.send_user.is_empty() {
-        param.send_user = param.addr.clone();
-        if let Err(e) = try_smtp_one_param(context, &param, smtp).await {
-            if let Some(at) = param.send_user.find('@') {
-                param.send_user = param.send_user.split_at(at).0.to_string();
-                try_smtp_one_param(context, &param, smtp).await?;
-            } else {
-                return Err(e);
-            }
-        }
-        Ok(param)
-    } else {
-        try_smtp_one_param(context, &param, smtp).await?;
-        Ok(param)
-    }
-}
-
-async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {
+) -> bool {
     let inf = format!(
-        "smtp: {}@{}:{} flags: 0x{:x}",
-        param.send_user, param.send_server, param.send_port, param.server_flags
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, &param).await {
-        bail!("could not connect: {}", err);
+    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        false
+    } else {
+        info!(context, "success: {}", inf);
+        smtp.disconnect().await;
+        true
     }
-
-    info!(context, "success: {}", inf);
-    smtp.disconnect().await;
-    Ok(())
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -662,9 +489,6 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
-    #[error("Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
     #[error("Failed to get URL")]
     ReadUrlError(#[from] self::read_url::Error),
 
@@ -697,23 +521,15 @@ mod tests {
     async fn test_get_offline_autoconfig() {
         let context = TestContext::new().await.ctx;
 
-        let mut params = LoginParam::new();
-        params.addr = "someone123@example.org".to_string();
-        assert!(get_offline_autoconfig(&context, &params).is_none());
+        let addr = "someone123@example.org";
+        assert!(get_offline_autoconfig(&context, addr).is_none());
 
-        let mut params = LoginParam::new();
-        params.addr = "someone123@nauta.cu".to_string();
-        let found_params = get_offline_autoconfig(&context, &params).unwrap();
-        assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
-        assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
-
-        assert_eq!(
-            found_params.imap_certificate_checks,
-            CertificateChecks::Automatic
-        );
-        assert_eq!(
-            found_params.smtp_certificate_checks,
-            CertificateChecks::Automatic
-        );
+        let addr = "someone123@nauta.cu";
+        let found_params = get_offline_autoconfig(&context, addr).unwrap();
+        assert_eq!(found_params.len(), 2);
+        assert_eq!(found_params[0].protocol, Protocol::IMAP);
+        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
+        assert_eq!(found_params[1].protocol, Protocol::SMTP);
+        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
     }
 }

src/configure/server_params.rs

@@ -0,0 +1,115 @@
+//! Variable server parameters lists
+
+use crate::provider::{Protocol, Socket};
+
+/// Set of variable parameters to try during configuration.
+///
+/// Can be loaded from offline provider database, online configuraiton
+/// or derived from user entered parameters.
+#[derive(Debug, Clone)]
+pub(crate) struct ServerParams {
+    /// Protocol, such as IMAP or SMTP.
+    pub protocol: Protocol,
+
+    /// Server hostname, empty if unknown.
+    pub hostname: String,
+
+    /// Server port, zero if unknown.
+    pub port: u16,
+
+    /// Socket security, such as TLS or STARTTLS, Socket::Automatic if unknown.
+    pub socket: Socket,
+
+    /// Username, empty if unknown.
+    pub username: String,
+}
+
+impl ServerParams {
+    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+
+        if self.username.is_empty() {
+            self.username = addr.to_string();
+            res.push(self.clone());
+
+            if let Some(at) = addr.find('@') {
+                self.username = addr.split_at(at).0.to_string();
+                res.push(self);
+            }
+        } else {
+            res.push(self)
+        }
+        res
+    }
+
+    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+        if self.hostname.is_empty() {
+            self.hostname = param_domain.to_string();
+            res.push(self.clone());
+
+            self.hostname = match self.protocol {
+                Protocol::IMAP => "imap.".to_string() + param_domain,
+                Protocol::SMTP => "smtp.".to_string() + param_domain,
+            };
+            res.push(self.clone());
+
+            self.hostname = "mail.".to_string() + param_domain;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+
+    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+        // Try to infer port from socket security.
+        if self.port == 0 {
+            self.port = match self.socket {
+                Socket::SSL => match self.protocol {
+                    Protocol::IMAP => 993,
+                    Protocol::SMTP => 465,
+                },
+                Socket::STARTTLS | Socket::Plain => match self.protocol {
+                    Protocol::IMAP => 143,
+                    Protocol::SMTP => 587,
+                },
+                Socket::Automatic => 0,
+            }
+        }
+
+        let mut res = Vec::new();
+        if self.port == 0 {
+            // Neither port nor security is set.
+            //
+            // Try common secure combinations.
+
+            // Try STARTTLS
+            self.socket = Socket::STARTTLS;
+            self.port = match self.protocol {
+                Protocol::IMAP => 143,
+                Protocol::SMTP => 587,
+            };
+            res.push(self.clone());
+
+            // Try TLS
+            self.socket = Socket::SSL;
+            self.port = match self.protocol {
+                Protocol::IMAP => 993,
+                Protocol::SMTP => 465,
+            };
+            res.push(self);
+        } else if self.socket == Socket::Automatic {
+            // Try TLS over user-provided port.
+            self.socket = Socket::SSL;
+            res.push(self.clone());
+
+            // Try STARTTLS over user-provided port.
+            self.socket = Socket::STARTTLS;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+}

src/constants.rs

@@ -199,44 +199,8 @@ pub const DC_LP_AUTH_OAUTH2: i32 = 0x2;
 /// If this flag is set, automatic configuration is skipped.
 pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 
-/// Connect to IMAP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_STARTTLS: i32 = 0x100;
-
-/// Connect to IMAP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_SSL: i32 = 0x200;
-
-/// Connect to IMAP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_PLAIN: i32 = 0x400;
-
-/// Connect to SMTP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_STARTTLS: usize = 0x10000;
-
-/// Connect to SMTP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
-
-/// Connect to SMTP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
-
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
-
-// QR code scanning (view from Bob, the joiner)
-pub const DC_VC_AUTH_REQUIRED: i32 = 2;
-pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
-pub const DC_BOB_ERROR: i32 = 0;
-pub const DC_BOB_SUCCESS: i32 = 1;
 
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;

src/contact.rs

@@ -20,6 +20,7 @@ use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
 use crate::param::*;
 use crate::peerstate::*;
+use crate::provider::Socket;
 use crate::stock::StockMessage;
 
 /// An object representing a single contact in memory.
@@ -235,6 +236,7 @@ impl Contact {
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
     ) -> Result<u32> {
+        let name = improve_single_line_input(name);
         ensure!(
             !addr.as_ref().is_empty(),
             "Cannot create contact with empty address"
@@ -728,8 +730,8 @@ impl Contact {
                     );
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
                 }
-            } else if 0 == loginparam.server_flags & DC_LP_IMAP_SOCKET_PLAIN as i32
-                && 0 == loginparam.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32
+            } else if loginparam.imap.security != Socket::Plain
+                && loginparam.smtp.security != Socket::Plain
             {
                 ret += &context.stock_str(StockMessage::EncrTransp).await;
             } else {

src/context.rs

@@ -17,9 +17,9 @@ use crate::error::*;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::lot::Lot;
 use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
+use crate::securejoin::Bob;
 use crate::sql::Sql;
 use std::time::SystemTime;
 
@@ -44,7 +44,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<BobStatus>,
+    pub(crate) bob: RwLock<Bob>,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -508,13 +508,6 @@ impl Default for RunningState {
     }
 }
 
-#[derive(Debug, Default)]
-pub(crate) struct BobStatus {
-    pub expects: i32,
-    pub status: i32,
-    pub qr_scan: Option<Lot>,
-}
-
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }

src/dc_receive_imf.rs

@@ -19,7 +19,9 @@ use crate::message::{self, MessageState, MessengerMessage, MsgId};
 use crate::mimeparser::*;
 use crate::param::*;
 use crate::peerstate::*;
-use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device};
+use crate::securejoin::{
+    self, handle_securejoin_handshake, observe_securejoin_on_other_device, BobStatus,
+};
 use crate::stock::StockMessage;
 use crate::{contact, location};
 
@@ -111,7 +113,7 @@ pub async fn dc_receive_imf(
     // or if From: is equal to SELF (in this case, it is any outgoing messages,
     // we do not check Return-Path any more as this is unreliable, see
     // https://github.com/deltachat/deltachat-core/issues/150)
-    let (from_id, from_id_blocked, incoming_origin) =
+    let (from_id, _from_id_blocked, incoming_origin) =
         from_field_to_contact_id(context, &mime_parser.from).await?;
 
     let incoming = from_id != DC_CONTACT_ID_SELF;
@@ -162,7 +164,6 @@ pub async fn dc_receive_imf(
             &rfc724_mid,
             &mut sent_timestamp,
             from_id,
-            from_id_blocked,
             &mut hidden,
             &mut chat_id,
             seen,
@@ -329,7 +330,6 @@ async fn add_parts(
     rfc724_mid: &str,
     sent_timestamp: &mut i64,
     from_id: u32,
-    from_id_blocked: bool,
     hidden: &mut bool,
     chat_id: &mut ChatId,
     seen: bool,
@@ -419,7 +419,7 @@ async fn add_parts(
                 Err(err) => {
                     *hidden = true;
 
-                    context.bob.write().await.status = 0; // secure-join failed
+                    context.bob.write().await.status = BobStatus::Error; // secure-join failed
                     context.stop_ongoing().await;
                     warn!(context, "Error in Secure-Join message handling: {}", err);
                     return Ok(());
@@ -432,6 +432,8 @@ async fn add_parts(
                 .await
                 .unwrap_or_default();
 
+        // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
+        // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() && mime_parser.failure_report.is_some() {
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
             info!(
@@ -440,11 +442,8 @@ async fn add_parts(
             );
         }
 
-        // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
-        // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() {
             // try to create a group
-            // (groups appear automatically only if the _sender_ is known, see core issue #54)
 
             let create_blocked =
                 if !test_normal_chat_id.is_unset() && test_normal_chat_id_blocked == Blocked::Not {
@@ -701,6 +700,20 @@ async fn add_parts(
     let in_fresh = state == MessageState::InFresh;
     let rcvd_timestamp = time();
     let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, in_fresh).await;
+
+    // Ensure replies to messages are sorted after the parent message.
+    //
+    // This is useful in a case where sender clocks are not
+    // synchronized and parent message has a Date: header with a
+    // timestamp higher than reply timestamp.
+    //
+    // This does not help if parent message arrives later than the
+    // reply.
+    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
+    let sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
+        std::cmp::max(sort_timestamp, parent_timestamp)
+    });
+
     *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
@@ -840,9 +853,7 @@ async fn add_parts(
     if chat_id.is_trash() || *hidden {
         *create_event_to_send = None;
     } else if incoming && state == MessageState::InFresh {
-        if from_id_blocked {
-            *create_event_to_send = None;
-        } else if Blocked::Not != chat_id_blocked {
+        if Blocked::Not != chat_id_blocked {
             *create_event_to_send = Some(CreateEvent::MsgsChanged);
         } else {
             *create_event_to_send = Some(CreateEvent::IncomingMsg);
@@ -2623,4 +2634,26 @@ mod tests {
         );
         assert_eq!(last_msg.from_id, DC_CONTACT_ID_INFO);
     }
+
+    #[async_std::test]
+    async fn test_html_only_mail() {
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+        dc_receive_imf(
+            &t.ctx,
+            include_bytes!("../test-data/message/wrong-html.eml"),
+            "INBOX",
+            0,
+            false,
+        )
+        .await
+        .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+        assert_eq!(msg.text.unwrap(), "   Guten Abend,   \n\n   Lots of text   \n\n   text with Umlaut ä...   \n\n   MfG    [...]");
+    }
 }

src/dc_tools.rs

@@ -9,7 +9,9 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
 use async_std::{fs, io};
+
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
@@ -17,10 +19,6 @@ use crate::context::Context;
 use crate::error::{bail, Error};
 use crate::events::EventType;
 
-pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
-    0 != v && 0 == v & (v - 1)
-}
-
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
 #[allow(clippy::indexing_slicing)]
@@ -296,6 +294,23 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     }
 }
 
+pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
+    match async_std::fs::read_dir(path).await {
+        Ok(mut read_dir) => {
+            while let Some(entry) = read_dir.next().await {
+                match entry {
+                    Ok(file) => {
+                        dc_delete_file(context, file.file_name()).await;
+                    }
+                    Err(e) => warn!(context, "Could not read file to delete: {}", e),
+                }
+            }
+        }
+
+        Err(e) => warn!(context, "Could not read dir to delete: {}", e),
+    }
+}
+
 pub(crate) async fn dc_copy_file(
     context: &Context,
     src_path: impl AsRef<Path>,
@@ -612,6 +627,16 @@ pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
     (listflags & bitindex) == bitindex
 }
 
+/// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
+pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+    input
+        .as_ref()
+        .replace("\n", " ")
+        .replace("\r", " ")
+        .trim()
+        .to_string()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -960,4 +985,10 @@ mod tests {
         assert_eq!(w, 100);
         assert_eq!(h, 50);
     }
+
+    #[test]
+    fn test_improve_single_line_input() {
+        assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
+        assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
+    }
 }

src/dehtml.rs

@@ -25,7 +25,19 @@ enum AddText {
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
 pub fn dehtml(buf: &str) -> String {
-    let buf = buf.trim();
+    let s = dehtml_quick_xml(buf);
+    if !s.trim().is_empty() {
+        return s;
+    }
+    let s = dehtml_manually(buf);
+    if !s.trim().is_empty() {
+        return s;
+    }
+    buf.to_string()
+}
+
+pub fn dehtml_quick_xml(buf: &str) -> String {
+    let buf = buf.trim().trim_start_matches("<!doctype html>");
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
@@ -171,9 +183,28 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     }
 }
 
+pub fn dehtml_manually(buf: &str) -> String {
+    // Just strip out everything between "<" and ">"
+    let mut strbuilder = String::new();
+    let mut show_next_chars = true;
+    for c in buf.chars() {
+        match c {
+            '<' => show_next_chars = false,
+            '>' => show_next_chars = true,
+            _ => {
+                if show_next_chars {
+                    strbuilder.push(c)
+                }
+            }
+        }
+    }
+    strbuilder
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::simplify::simplify;
 
     #[test]
     fn test_dehtml() {
@@ -182,20 +213,23 @@ mod tests {
                 "<a href='https://example.com'> Foo </a>",
                 "[ Foo ](https://example.com)",
             ),
-            ("<img href='/foo.png'>", ""),
             ("<b> bar </b>", "* bar *"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
-            // Note missing '
-            ("<a href='/foo.png>Hi</a> ", ""),
+            // Despite missing ', this should be shown:
+            ("<a href='/foo.png>Hi</a> ", "Hi "),
             (
                 "<a href='https://get.delta.chat/'/>",
                 "[](https://get.delta.chat/)",
             ),
             ("", ""),
+            ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
+            // Invalid html (at least DC should show the text if the html is invalid):
+            ("<!some invalid html code>\n<b>some text</b>", "some text"),
+            ("<This text is in brackets>", "<This text is in brackets>"),
         ];
         for (input, output) in cases {
-            assert_eq!(dehtml(input), output);
+            assert_eq!(simplify(dehtml(input), true).0, output);
         }
     }
 

src/format_flowed.rs

@@ -0,0 +1,136 @@
+///! # format=flowed support
+///!
+///! Format=flowed is defined in
+///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+///!
+///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+///! during formatting, i.e., DelSp parameter introduced in RFC 3676
+///! is assumed to be set to "no".
+///!
+///! For received messages, DelSp parameter is honoured.
+
+/// Wraps line to 72 characters using format=flowed soft breaks.
+///
+/// 72 characters is the limit recommended by RFC 3676.
+///
+/// The function breaks line only after SP and before non-whitespace
+/// characters. It also does not insert breaks before ">" to avoid the
+/// need to do space stuffing (see RFC 3676) for quotes.
+///
+/// If there are long words, line may still exceed the limits on line
+/// length. However, this should be rare and should not result in
+/// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
+/// and Spam Assassin limit is 78 characters.
+fn format_line_flowed(line: &str) -> String {
+    let mut result = String::new();
+    let mut buffer = String::new();
+    let mut after_space = false;
+
+    for c in line.chars() {
+        if c == ' ' {
+            buffer.push(c);
+            after_space = true;
+        } else {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() && c != '>' {
+                // Flush the buffer and insert soft break (SP CRLF).
+                result += &buffer;
+                result += "\r\n";
+                buffer = String::new();
+            }
+            buffer.push(c);
+            after_space = false;
+        }
+    }
+    result + &buffer
+}
+
+/// Returns text formatted according to RFC 3767 (format=flowed).
+///
+/// This function accepts text separated by LF, but returns text
+/// separated by CRLF.
+///
+/// RFC 2646 technique is used to insert soft line breaks, so DelSp
+/// SHOULD be set to "no" when sending.
+pub fn format_flowed(text: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if line.len() > 78 {
+            result += &format_line_flowed(line);
+        } else {
+            result += line;
+        }
+    }
+    result
+}
+
+/// Joins lines in format=flowed text.
+///
+/// Lines must be separated by single LF.
+///
+/// Quote processing is not supported, it is assumed that they are
+/// deleted during simplification.
+///
+/// Signature separator line is not processed here, it is assumed to
+/// be stripped beforehand.
+pub fn unformat_flowed(text: &str, delsp: bool) -> String {
+    let mut result = String::new();
+    let mut skip_newline = true;
+
+    for line in text.split('\n') {
+        // Revert space-stuffing
+        let line = line.strip_prefix(" ").unwrap_or(line);
+
+        if !skip_newline {
+            result.push('\n');
+        }
+
+        if let Some(line) = line.strip_suffix(" ") {
+            // Flowed line
+            result += line;
+            if !delsp {
+                result.push(' ');
+            }
+            skip_newline = true;
+        } else {
+            // Fixed line
+            result += line;
+            skip_newline = false;
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_flowed() {
+        let text = "Foo bar baz";
+        assert_eq!(format_flowed(text), "Foo bar baz");
+
+        let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    \n\
+                    To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "This is the Autocrypt Setup Message used to transfer your key between clients.\r\n\
+                        \r\n\
+                        To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
+    }
+
+    #[test]
+    fn test_unformat_flowed() {
+        let text = "this is a very long message that should be wrapped using format=flowed and \n\
+            unwrapped on the receiver";
+        let expected =
+            "this is a very long message that should be wrapped using format=flowed and \
+                        unwrapped on the receiver";
+        assert_eq!(unformat_flowed(text, false), expected);
+    }
+}

src/imap/idle.rs

@@ -5,31 +5,11 @@ use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
+use crate::error::{bail, format_err, Result};
 use crate::{context::Context, scheduler::InterruptInfo};
 
-use super::select_folder;
 use super::session::Session;
 
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete: {0}")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
-
-    #[error("IMAP IDLE protocol timed out: {0}")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
-
-    #[error("IMAP server does not have IDLE capability")]
-    IdleAbilityMissing,
-
-    #[error("IMAP select folder error: {0}")]
-    SelectFolderError(#[from] select_folder::Error),
-
-    #[error("Setup handle error: {0}")]
-    SetupHandleError(#[from] super::Error),
-}
-
 impl Imap {
     pub fn can_idle(&self) -> bool {
         self.config.can_idle
@@ -43,7 +23,7 @@ impl Imap {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
-            return Err(Error::IdleAbilityMissing);
+            bail!("IMAP server does not have IDLE capability");
         }
         self.setup_handle_if_needed(context).await?;
 
@@ -72,7 +52,7 @@ impl Imap {
 
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
-                return Err(Error::IdleProtocolFailed(err));
+                bail!("IMAP IDLE protocol failed to init/complete: {}", err);
             }
 
             let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
@@ -115,7 +95,7 @@ impl Imap {
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(Error::IdleTimeout)??;
+                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
             self.session = Some(Session { inner: session });
         } else {
             warn!(context, "Attempted to idle without a session");

src/imap/mod.rs

@@ -19,70 +19,30 @@ use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf, from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list,
 };
+use crate::error::{bail, format_err, Result};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
-use crate::login_param::{CertificateChecks, LoginParam};
+use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{self, update_server_uid, MessageState};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::provider::get_provider_info;
-use crate::{chat, scheduler::InterruptInfo, stock::StockMessage};
+use crate::provider::{get_provider_info, Socket};
+use crate::{
+    chat, dc_tools::dc_extract_grpid_from_rfc724_mid, scheduler::InterruptInfo, stock::StockMessage,
+};
 
 mod client;
 mod idle;
 pub mod select_folder;
 mod session;
 
+use chat::get_chat_id_by_grpid;
 use client::Client;
 use message::Message;
 use session::Session;
 
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("IMAP Connect without configured params")]
-    ConnectWithoutConfigure,
-
-    #[error("IMAP Connection Failed params: {0}")]
-    ConnectionFailed(String),
-
-    #[error("IMAP No Connection established")]
-    NoConnection,
-
-    #[error("IMAP Could not get OAUTH token")]
-    OauthError,
-
-    #[error("IMAP Could not login as {0}")]
-    LoginFailed(String),
-
-    #[error("IMAP Could not fetch")]
-    FetchFailed(#[from] async_imap::error::Error),
-
-    #[error("IMAP operation attempted while it is torn down")]
-    InTeardown,
-
-    #[error("IMAP operation attempted while it is torn down")]
-    SqlError(#[from] crate::sql::Error),
-
-    #[error("IMAP got error from elsewhere")]
-    WrappedError(#[from] crate::error::Error),
-
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
-
-    #[error("Mail parse error")]
-    MailParseError(#[from] mailparse::MailParseError),
-
-    #[error("No mailbox selected, folder: {0}")]
-    NoMailbox(String),
-
-    #[error("IMAP other error: {0}")]
-    Other(String),
-}
-
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapActionResult {
     Failed,
@@ -105,6 +65,7 @@ const PREFETCH_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
                               AUTOCRYPT-SETUP-MESSAGE\
                               )])";
 const DELETE_CHECK_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])";
+const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])";
 const JUST_UID: &str = "(UID)";
 const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
 const SELECT_ALL: &str = "1:*";
@@ -147,12 +108,9 @@ enum FolderMeaning {
 #[derive(Debug)]
 struct ImapConfig {
     pub addr: String,
-    pub imap_server: String,
-    pub imap_port: u16,
-    pub imap_user: String,
-    pub imap_pw: String,
+    pub lp: ServerLoginParam,
     pub strict_tls: bool,
-    pub server_flags: usize,
+    pub oauth2: bool,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
     pub selected_folder_needs_expunge: bool,
@@ -167,12 +125,9 @@ impl Default for ImapConfig {
     fn default() -> Self {
         ImapConfig {
             addr: "".into(),
-            imap_server: "".into(),
-            imap_port: 0,
-            imap_user: "".into(),
-            imap_pw: "".into(),
+            lp: Default::default(),
             strict_tls: false,
-            server_flags: 0,
+            oauth2: false,
             selected_folder: None,
             selected_mailbox: None,
             selected_folder_needs_expunge: false,
@@ -208,8 +163,8 @@ impl Imap {
     }
 
     async fn setup_handle_if_needed(&mut self, context: &Context) -> Result<()> {
-        if self.config.imap_server.is_empty() {
-            return Err(Error::InTeardown);
+        if self.config.lp.server.is_empty() {
+            bail!("IMAP operation attempted while it is torn down");
         }
 
         if self.should_reconnect() {
@@ -219,40 +174,40 @@ impl Imap {
             return Ok(());
         }
 
-        let server_flags = self.config.server_flags as i32;
+        let oauth2 = self.config.oauth2;
 
-        let connection_res: ImapResult<Client> =
-            if (server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN)) != 0 {
-                let config = &mut self.config;
-                let imap_server: &str = config.imap_server.as_ref();
-                let imap_port = config.imap_port;
+        let connection_res: ImapResult<Client> = if self.config.lp.security == Socket::STARTTLS
+            || self.config.lp.security == Socket::Plain
+        {
+            let config = &mut self.config;
+            let imap_server: &str = config.lp.server.as_ref();
+            let imap_port = config.lp.port;
 
-                match Client::connect_insecure((imap_server, imap_port)).await {
-                    Ok(client) => {
-                        if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.strict_tls).await
-                        } else {
-                            Ok(client)
-                        }
+            match Client::connect_insecure((imap_server, imap_port)).await {
+                Ok(client) => {
+                    if config.lp.security == Socket::STARTTLS {
+                        client.secure(imap_server, config.strict_tls).await
+                    } else {
+                        Ok(client)
                     }
-                    Err(err) => Err(err),
                 }
-            } else {
-                let config = &self.config;
-                let imap_server: &str = config.imap_server.as_ref();
-                let imap_port = config.imap_port;
+                Err(err) => Err(err),
+            }
+        } else {
+            let config = &self.config;
+            let imap_server: &str = config.lp.server.as_ref();
+            let imap_port = config.lp.port;
 
-                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
-                    .await
-            };
+            Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls).await
+        };
 
         let login_res = match connection_res {
             Ok(client) => {
                 let config = &self.config;
-                let imap_user: &str = config.imap_user.as_ref();
-                let imap_pw: &str = config.imap_pw.as_ref();
+                let imap_user: &str = config.lp.user.as_ref();
+                let imap_pw: &str = config.lp.password.as_ref();
 
-                if (server_flags & DC_LP_AUTH_OAUTH2) != 0 {
+                if oauth2 {
                     let addr: &str = config.addr.as_ref();
 
                     if let Some(token) =
@@ -264,7 +219,7 @@ impl Imap {
                         };
                         client.authenticate("XOAUTH2", auth).await
                     } else {
-                        return Err(Error::OauthError);
+                        bail!("IMAP Could not get OAUTH token");
                     }
                 } else {
                     client.login(imap_user, imap_pw).await
@@ -273,8 +228,8 @@ impl Imap {
             Err(err) => {
                 let message = {
                     let config = &self.config;
-                    let imap_server: &str = config.imap_server.as_ref();
-                    let imap_port = config.imap_port;
+                    let imap_server: &str = config.lp.server.as_ref();
+                    let imap_port = config.lp.port;
                     context
                         .stock_string_repl_str2(
                             StockMessage::ServerResponse,
@@ -285,7 +240,7 @@ impl Imap {
                 };
                 // IMAP connection failures are reported to users
                 emit_event!(context, EventType::ErrorNetwork(message));
-                return Err(Error::ConnectionFailed(err.to_string()));
+                bail!("IMAP connection failed: {}", err);
             }
         };
 
@@ -301,7 +256,7 @@ impl Imap {
             }
 
             Err((err, _)) => {
-                let imap_user = self.config.imap_user.to_owned();
+                let imap_user = self.config.lp.user.to_owned();
                 let message = context
                     .stock_string_repl_str(StockMessage::CannotLogin, &imap_user)
                     .await;
@@ -331,7 +286,7 @@ impl Imap {
                 }
 
                 self.trigger_reconnect();
-                Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
+                Err(format_err!("IMAP Could not login as {}", imap_user))
             }
         }
     }
@@ -357,10 +312,7 @@ impl Imap {
         let mut cfg = &mut self.config;
 
         cfg.addr = "".into();
-        cfg.imap_server = "".into();
-        cfg.imap_user = "".into();
-        cfg.imap_pw = "".into();
-        cfg.imap_port = 0;
+        cfg.lp = Default::default();
 
         cfg.can_idle = false;
         cfg.can_move = false;
@@ -372,41 +324,47 @@ impl Imap {
             return Ok(());
         }
         if !context.is_configured().await {
-            return Err(Error::ConnectWithoutConfigure);
+            bail!("IMAP Connect without configured params");
         }
 
         let param = LoginParam::from_database(context, "configured_").await;
         // the trailing underscore is correct
 
-        if self.connect(context, &param).await {
-            self.ensure_configured_folders(context, true).await
+        if let Err(err) = self
+            .connect(
+                context,
+                &param.imap,
+                &param.addr,
+                param.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+            )
+            .await
+        {
+            bail!("IMAP Connection Failed with params {}: {}", param, err);
         } else {
-            Err(Error::ConnectionFailed(format!("{}", param)))
+            self.ensure_configured_folders(context, true).await
         }
     }
 
     /// Tries connecting to imap account using the specific login parameters.
-    pub async fn connect(&mut self, context: &Context, lp: &LoginParam) -> bool {
-        if lp.mail_server.is_empty() || lp.mail_user.is_empty() || lp.mail_pw.is_empty() {
-            return false;
+    ///
+    /// `addr` is used to renew token if OAuth2 authentication is used.
+    pub async fn connect(
+        &mut self,
+        context: &Context,
+        lp: &ServerLoginParam,
+        addr: &str,
+        oauth2: bool,
+    ) -> Result<()> {
+        if lp.server.is_empty() || lp.user.is_empty() || lp.password.is_empty() {
+            bail!("Incomplete IMAP connection parameters");
         }
 
         {
-            let addr = &lp.addr;
-            let imap_server = &lp.mail_server;
-            let imap_port = lp.mail_port as u16;
-            let imap_user = &lp.mail_user;
-            let imap_pw = &lp.mail_pw;
-            let server_flags = lp.server_flags as usize;
-
             let mut config = &mut self.config;
             config.addr = addr.to_string();
-            config.imap_server = imap_server.to_string();
-            config.imap_port = imap_port;
-            config.imap_user = imap_user.to_string();
-            config.imap_pw = imap_pw.to_string();
-            let provider = get_provider_info(&lp.addr);
-            config.strict_tls = match lp.imap_certificate_checks {
+            config.lp = lp.clone();
+            let provider = get_provider_info(&addr);
+            config.strict_tls = match lp.certificate_checks {
                 CertificateChecks::Automatic => {
                     provider.map_or(false, |provider| provider.strict_tls)
                 }
@@ -414,20 +372,20 @@ impl Imap {
                 CertificateChecks::AcceptInvalidCertificates
                 | CertificateChecks::AcceptInvalidCertificates2 => false,
             };
-            config.server_flags = server_flags;
+            config.oauth2 = oauth2;
         }
 
         if let Err(err) = self.setup_handle_if_needed(context).await {
             warn!(context, "failed to setup imap handle: {}", err);
             self.free_connect_params().await;
-            return false;
+            return Err(err);
         }
 
         let teardown = match &mut self.session {
             Some(ref mut session) => match session.capabilities().await {
                 Ok(caps) => {
                     if !context.sql.is_open().await {
-                        warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.mail_user,);
+                        warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.user,);
                         true
                     } else {
                         let can_idle = caps.has_str("IDLE");
@@ -447,7 +405,7 @@ impl Imap {
                             context,
                             EventType::ImapConnected(format!(
                                 "IMAP-LOGIN as {}, capabilities: {}",
-                                lp.mail_user, caps_list,
+                                lp.user, caps_list,
                             ))
                         );
                         false
@@ -464,10 +422,9 @@ impl Imap {
         if teardown {
             self.disconnect(context).await;
 
-            false
-        } else {
-            true
+            bail!("IMAP disconnected immediately after connecting due to error");
         }
+        Ok(())
     }
 
     pub async fn disconnect(&mut self, context: &Context) {
@@ -478,7 +435,7 @@ impl Imap {
     pub async fn fetch(&mut self, context: &Context, watch_folder: &str) -> Result<()> {
         if !context.sql.is_open().await {
             // probably shutdown
-            return Err(Error::InTeardown);
+            bail!("IMAP operation attempted while it is torn down");
         }
         self.setup_handle_if_needed(context).await?;
 
@@ -514,6 +471,81 @@ impl Imap {
         }
     }
 
+    /// Synchronizes UIDs in the database with UIDs on the server.
+    ///
+    /// It is assumed that no operations are taking place on the same
+    /// folder at the moment. Make sure to run it in the same
+    /// thread/task as other network operations on this folder to
+    /// avoid race conditions.
+    pub(crate) async fn resync_folder_uids(
+        &mut self,
+        context: &Context,
+        folder: String,
+    ) -> Result<()> {
+        // Collect pairs of UID and Message-ID.
+        let mut msg_ids = BTreeMap::new();
+
+        self.select_folder(context, Some(&folder)).await?;
+
+        let session = if let Some(ref mut session) = &mut self.session {
+            session
+        } else {
+            bail!("IMAP No Connection established");
+        };
+
+        match session.uid_fetch("1:*", RFC724MID_UID).await {
+            Ok(mut list) => {
+                while let Some(fetch) = list.next().await {
+                    let msg = fetch?;
+
+                    // Get Message-ID
+                    let message_id = get_fetch_headers(&msg)
+                        .and_then(|headers| prefetch_get_message_id(&headers))
+                        .ok();
+
+                    if let (Some(uid), Some(rfc724_mid)) = (msg.uid, message_id) {
+                        msg_ids.insert(uid, rfc724_mid);
+                    }
+                }
+            }
+            Err(err) => {
+                bail!("Can't resync folder {}: {}", folder, err);
+            }
+        }
+
+        info!(
+            context,
+            "Resync: collected {} message IDs in folder {}",
+            msg_ids.len(),
+            &folder
+        );
+
+        // Write collected UIDs to SQLite database.
+        context
+            .sql
+            .with_conn(move |mut conn| {
+                let conn2 = &mut conn;
+                let tx = conn2.transaction()?;
+                tx.execute(
+                    "UPDATE msgs SET server_uid=0 WHERE server_folder=?",
+                    params![folder],
+                )?;
+                for (uid, rfc724_mid) in &msg_ids {
+                    // This may detect previously undetected moved
+                    // messages, so we update server_folder too.
+                    tx.execute(
+                        "UPDATE msgs \
+                         SET server_folder=?,server_uid=? WHERE rfc724_mid=?",
+                        params![folder, uid, rfc724_mid],
+                    )?;
+                }
+                tx.commit()?;
+                Ok(())
+            })
+            .await?;
+        Ok(())
+    }
+
     /// return Result with (uid_validity, last_seen_uid) tuple.
     pub(crate) async fn select_with_uidvalidity(
         &mut self,
@@ -529,13 +561,12 @@ impl Imap {
         let mailbox = config
             .selected_mailbox
             .as_ref()
-            .ok_or_else(|| Error::NoMailbox(folder.to_string()))?;
+            .ok_or_else(|| format_err!("No mailbox selected, folder: {}", folder))?;
 
         let new_uid_validity = match mailbox.uid_validity {
             Some(v) => v,
             None => {
-                let s = format!("No UIDVALIDITY for folder {:?}", folder);
-                return Err(Error::Other(s));
+                bail!("No UIDVALIDITY for folder {:?}", folder);
             }
         };
 
@@ -582,21 +613,24 @@ impl Imap {
                             if let Some(new_last_seen_uid) = new_last_seen_uid {
                                 new_last_seen_uid
                             } else {
-                                return Err(Error::Other("failed to fetch".into()));
+                                bail!("failed to fetch");
                             }
                         }
                         Err(err) => {
-                            return Err(Error::FetchFailed(err));
+                            bail!("IMAP Could not fetch: {}", err);
                         }
                     }
                 } else {
-                    return Err(Error::NoConnection);
+                    bail!("IMAP No Connection established");
                 }
             }
         };
 
         self.set_config_last_seen_uid(context, &folder, new_uid_validity, new_last_seen_uid)
             .await;
+        if uid_validity != 0 || last_seen_uid != 0 {
+            job::schedule_resync(context).await;
+        }
         info!(
             context,
             "uid/validity change: new {}/{} current {}/{}",
@@ -693,7 +727,7 @@ impl Imap {
         uid: u32,
     ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
         if self.session.is_none() {
-            return Err(Error::NoConnection);
+            bail!("IMAP No Connection established");
         }
 
         let session = self.session.as_mut().unwrap();
@@ -704,11 +738,11 @@ impl Imap {
         let mut list = session
             .uid_fetch(set, PREFETCH_FLAGS)
             .await
-            .map_err(Error::FetchFailed)?;
+            .map_err(|err| format_err!("IMAP Could not fetch: {}", err))?;
 
         let mut msgs = BTreeMap::new();
         while let Some(fetch) = list.next().await {
-            let msg = fetch.map_err(|err| Error::Other(err.to_string()))?;
+            let msg = fetch?;
             if let Some(msg_uid) = msg.uid {
                 msgs.insert(msg_uid, msg);
             }
@@ -1170,14 +1204,14 @@ impl Imap {
 
     pub async fn configure_folders(&mut self, context: &Context, create_mvbox: bool) -> Result<()> {
         if !self.is_connected() {
-            return Err(Error::NoConnection);
+            bail!("IMAP No Connection established");
         }
 
         if let Some(ref mut session) = &mut self.session {
             let mut folders = match session.list(Some(""), Some("*")).await {
                 Ok(f) => f,
                 Err(err) => {
-                    return Err(Error::Other(format!("list_folders failed {:?}", err)));
+                    bail!("list_folders failed: {}", err);
                 }
             };
 
@@ -1188,7 +1222,7 @@ impl Imap {
             let mut fallback_folder = get_fallback_folder(&delimiter);
 
             while let Some(folder) = folders.next().await {
-                let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
+                let folder = folder?;
                 info!(context, "Scanning folder: {:?}", folder);
 
                 // Update the delimiter iff there is a different one, but only once.
@@ -1473,7 +1507,7 @@ fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result<String>
     if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId) {
         Ok(crate::mimeparser::parse_message_id(&message_id)?)
     } else {
-        Err(Error::Other("prefetch: No message ID found".to_string()))
+        bail!("prefetch: No message ID found");
     }
 }
 
@@ -1501,6 +1535,18 @@ pub(crate) async fn prefetch_should_download(
     headers: &[mailparse::MailHeader<'_>],
     show_emails: ShowEmails,
 ) -> Result<bool> {
+    if let Some(rfc724_mid) = headers.get_header_value(HeaderDef::MessageId) {
+        if let Some(group_id) = dc_extract_grpid_from_rfc724_mid(&rfc724_mid) {
+            if let Ok((chat_id, _, _)) = get_chat_id_by_grpid(context, group_id).await {
+                if !chat_id.is_unset() {
+                    // This might be a group command, like removing a group member.
+                    // We really need to fetch this to avoid inconsistent group state.
+                    return Ok(true);
+                }
+            }
+        }
+    }
+
     let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
     let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await;
 

src/imex.rs

@@ -82,18 +82,46 @@ pub async fn imex(
     what: ImexMode,
     param1: Option<impl AsRef<Path>>,
 ) -> Result<()> {
-    use futures::future::FutureExt;
-
     let cancel = context.alloc_ongoing().await?;
-    let res = imex_inner(context, what, param1)
-        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
-        .await;
+
+    let res = async {
+        let success = imex_inner(context, what, param1).await;
+        match success {
+            Ok(()) => {
+                info!(context, "IMEX successfully completed");
+                context.emit_event(EventType::ImexProgress(1000));
+                Ok(())
+            }
+            Err(err) => {
+                cleanup_aborted_imex(context, what).await;
+                error!(context, "{}", err);
+                context.emit_event(EventType::ImexProgress(0));
+                bail!("IMEX FAILED to complete: {}", err);
+            }
+        }
+    }
+    .race(async {
+        cancel.recv().await.ok();
+        cleanup_aborted_imex(context, what).await;
+        Err(format_err!("canceled"))
+    })
+    .await;
 
     context.free_ongoing().await;
 
     res
 }
 
+async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
+    if what == ImexMode::ImportBackup {
+        dc_delete_file(context, context.get_dbfile()).await;
+        dc_delete_files_in_dir(context, context.get_blobdir()).await;
+    }
+    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
+        context.sql.open(context, context.get_dbfile(), false).await;
+    }
+}
+
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
@@ -425,7 +453,7 @@ async fn imex_inner(
         }
     }
 
-    let success = match what {
+    match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
@@ -434,18 +462,6 @@ async fn imex_inner(
         ImexMode::ExportBackup => export_backup_old(context, path).await,
         // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
-    };
-
-    match success {
-        Ok(()) => {
-            info!(context, "IMEX successfully completed");
-            context.emit_event(EventType::ImexProgress(1000));
-            Ok(())
-        }
-        Err(err) => {
-            context.emit_event(EventType::ImexProgress(0));
-            bail!("IMEX FAILED to complete: {}", err);
-        }
     }
 }
 
@@ -632,6 +648,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
     let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let _d = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
@@ -670,13 +687,19 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         }
         Err(e) => {
             error!(context, "backup failed: {}", e);
-            // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-            fs::remove_file(temp_path).await?;
         }
     }
 
     res
 }
+struct DeleteOnDrop(PathBuf);
+impl Drop for DeleteOnDrop {
+    fn drop(&mut self) {
+        let file = self.0.clone();
+        // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
+    }
+}
 
 async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
     let file = File::create(temp_path).await?;

src/job.rs

@@ -26,7 +26,6 @@ use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::EventType;
 use crate::imap::*;
 use crate::location;
-use crate::login_param::LoginParam;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
@@ -102,6 +101,10 @@ pub enum Action {
     MoveMsg = 200,
     DeleteMsgOnImap = 210,
 
+    // UID synchronization is high-priority to make sure correct UIDs
+    // are used by message moving/deletion.
+    ResyncFolders = 300,
+
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
     MaybeSendLocations = 5005, // low priority ...
     MaybeSendLocationsEnded = 5007,
@@ -124,6 +127,7 @@ impl From<Action> for Thread {
 
             Housekeeping => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
+            ResyncFolders => Thread::Imap,
             EmptyServer => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
@@ -338,12 +342,9 @@ impl Job {
 
     pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         let filename = job_try!(job_try!(self
@@ -486,12 +487,9 @@ impl Job {
         let recipients = vec![recipient];
 
         // connect to SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         self.smtp_send(context, recipients, body, self.job_id, smtp, || {
@@ -565,6 +563,11 @@ impl Job {
                     context,
                     "The message is deleted from the server when all parts are deleted.",
                 );
+            } else if cnt == 0 {
+                warn!(
+                    context,
+                    "The message {} has no UID on the server to delete", &msg.rfc724_mid
+                );
             } else {
                 /* if this is the last existing part of the message,
                 we delete the message from the server */
@@ -619,6 +622,44 @@ impl Job {
         }
     }
 
+    /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
+    ///
+    /// If a copy of the message is present in multiple folders, mvbox
+    /// is preferred to inbox, which is in turn preferred to
+    /// sentbox. This is because in the database it is impossible to
+    /// store multiple UIDs for one message, so we prefer to
+    /// automatically delete messages in the folders managed by Delta
+    /// Chat in contrast to the Sent folder, which is normally managed
+    /// by the user via webmail or another email client.
+    async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        if let Some(sentbox_folder) = &context.get_config(Config::ConfiguredSentboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, sentbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, inbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, mvbox_folder.to_string())
+                    .await
+            );
+        }
+        Status::Finished(Ok(()))
+    }
+
     async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.connect_configured(context).await {
             warn!(context, "could not connect: {:?}", err);
@@ -986,6 +1027,7 @@ async fn perform_job_action(
         }
         Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
+        Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
         Action::Housekeeping => {
@@ -1019,6 +1061,15 @@ async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     Ok(())
 }
 
+pub(crate) async fn schedule_resync(context: &Context) {
+    kill_action(context, Action::ResyncFolders).await;
+    add(
+        context,
+        Job::new(Action::ResyncFolders, 0, Params::new(), 0),
+    )
+    .await;
+}
+
 /// Creates a job.
 pub fn create(action: Action, foreign_id: i32, param: Params, delay_seconds: i64) -> Result<Job> {
     ensure!(
@@ -1043,6 +1094,7 @@ pub async fn add(context: &Context, job: Job) {
             Action::Housekeeping
             | Action::EmptyServer
             | Action::DeleteMsgOnImap
+            | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");

src/lib.rs

@@ -54,6 +54,7 @@ pub mod imex;
 mod scheduler;
 #[macro_use]
 pub mod job;
+mod format_flowed;
 pub mod key;
 mod keyring;
 pub mod location;

src/location.rs

@@ -469,7 +469,7 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
          <Document>\n\
          <Placemark>\
          <Timestamp><when>{}</when></Timestamp>\
-         <Point><coordinates>{:.2},{:.2}</coordinates></Point>\
+         <Point><coordinates>{},{}</coordinates></Point>\
          </Placemark>\n\
          </Document>\n\
          </kml>",
@@ -757,4 +757,22 @@ mod tests {
         assert!(locations_ref[1].accuracy < 2.6f64);
         assert_eq!(locations_ref[1].timestamp, 1544739072);
     }
+
+    #[async_std::test]
+    async fn test_get_message_kml() {
+        let context = TestContext::new().await;
+        let timestamp = 1598490000;
+
+        let xml = get_message_kml(timestamp, 51.423723f64, 8.552556f64);
+        let kml = Kml::parse(&context.ctx, xml.as_bytes()).expect("parsing failed");
+        let locations_ref = &kml.locations;
+        assert_eq!(locations_ref.len(), 1);
+
+        assert!(locations_ref[0].latitude >= 51.423723f64);
+        assert!(locations_ref[0].latitude < 51.423724f64);
+        assert!(locations_ref[0].longitude >= 8.552556f64);
+        assert!(locations_ref[0].longitude < 8.552557f64);
+        assert_eq!(locations_ref[0].accuracy, 0.0f64);
+        assert_eq!(locations_ref[0].timestamp, timestamp);
+    }
 }

src/login_param.rs

@@ -3,13 +3,16 @@
 use std::borrow::Cow;
 use std::fmt;
 
-use crate::context::Context;
+use crate::{context::Context, provider::Socket};
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(i32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
+    /// Same as AcceptInvalidCertificates unless overridden by
+    /// `strict_tls` setting in provider database.
     Automatic = 0,
+
     Strict = 1,
 
     /// Same as AcceptInvalidCertificates
@@ -25,30 +28,29 @@ impl Default for CertificateChecks {
     }
 }
 
+/// Login parameters for a single server, either IMAP or SMTP
+#[derive(Default, Debug, Clone)]
+pub struct ServerLoginParam {
+    pub server: String,
+    pub user: String,
+    pub password: String,
+    pub port: u16,
+    pub security: Socket,
+
+    /// TLS options: whether to allow invalid certificates and/or
+    /// invalid hostnames
+    pub certificate_checks: CertificateChecks,
+}
+
 #[derive(Default, Debug, Clone)]
 pub struct LoginParam {
     pub addr: String,
-    pub mail_server: String,
-    pub mail_user: String,
-    pub mail_pw: String,
-    pub mail_port: i32,
-    /// IMAP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub imap_certificate_checks: CertificateChecks,
-    pub send_server: String,
-    pub send_user: String,
-    pub send_pw: String,
-    pub send_port: i32,
-    /// SMTP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub smtp_certificate_checks: CertificateChecks,
+    pub imap: ServerLoginParam,
+    pub smtp: ServerLoginParam,
     pub server_flags: i32,
 }
 
 impl LoginParam {
-    /// Create a new `LoginParam` with default values.
-    pub fn new() -> Self {
-        Default::default()
-    }
-
     /// Read the login parameters from the database.
     pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
         let prefix = prefix.as_ref();
@@ -77,6 +79,13 @@ impl LoginParam {
         let key = format!("{}mail_pw", prefix);
         let mail_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}mail_security", prefix);
+        let mail_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}imap_certificate_checks", prefix);
         let imap_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -100,6 +109,13 @@ impl LoginParam {
         let key = format!("{}send_pw", prefix);
         let send_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}send_security", prefix);
+        let send_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}smtp_certificate_checks", prefix);
         let smtp_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -116,16 +132,22 @@ impl LoginParam {
 
         LoginParam {
             addr,
-            mail_server,
-            mail_user,
-            mail_pw,
-            mail_port,
-            imap_certificate_checks,
-            send_server,
-            send_user,
-            send_pw,
-            send_port,
-            smtp_certificate_checks,
+            imap: ServerLoginParam {
+                server: mail_server,
+                user: mail_user,
+                password: mail_pw,
+                port: mail_port as u16,
+                security: mail_security,
+                certificate_checks: imap_certificate_checks,
+            },
+            smtp: ServerLoginParam {
+                server: send_server,
+                user: send_user,
+                password: send_pw,
+                port: send_port as u16,
+                security: send_security,
+                certificate_checks: smtp_certificate_checks,
+            },
             server_flags,
         }
     }
@@ -143,41 +165,51 @@ impl LoginParam {
         sql.set_raw_config(context, key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_server))
+        sql.set_raw_config(context, key, Some(&self.imap.server))
             .await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(context, key, self.mail_port).await?;
+        sql.set_raw_config_int(context, key, self.imap.port as i32)
+            .await?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_user))
+        sql.set_raw_config(context, key, Some(&self.imap.user))
             .await?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_pw))
+        sql.set_raw_config(context, key, Some(&self.imap.password))
+            .await?;
+
+        let key = format!("{}mail_security", prefix);
+        sql.set_raw_config_int(context, key, self.imap.security as i32)
             .await?;
 
         let key = format!("{}imap_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.imap.certificate_checks as i32)
             .await?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_server))
+        sql.set_raw_config(context, key, Some(&self.smtp.server))
             .await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(context, key, self.send_port).await?;
+        sql.set_raw_config_int(context, key, self.smtp.port as i32)
+            .await?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_user))
+        sql.set_raw_config(context, key, Some(&self.smtp.user))
             .await?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_pw))
+        sql.set_raw_config(context, key, Some(&self.smtp.password))
+            .await?;
+
+        let key = format!("{}send_security", prefix);
+        sql.set_raw_config_int(context, key, self.smtp.security as i32)
             .await?;
 
         let key = format!("{}smtp_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.smtp.certificate_checks as i32)
             .await?;
 
         let key = format!("{}server_flags", prefix);
@@ -199,16 +231,24 @@ impl fmt::Display for LoginParam {
             f,
             "{} imap:{}:{}:{}:{}:cert_{} smtp:{}:{}:{}:{}:cert_{} {}",
             unset_empty(&self.addr),
-            unset_empty(&self.mail_user),
-            if !self.mail_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.mail_server),
-            self.mail_port,
-            self.imap_certificate_checks,
-            unset_empty(&self.send_user),
-            if !self.send_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.send_server),
-            self.send_port,
-            self.smtp_certificate_checks,
+            unset_empty(&self.imap.user),
+            if !self.imap.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.imap.server),
+            self.imap.port,
+            self.imap.certificate_checks,
+            unset_empty(&self.smtp.user),
+            if !self.smtp.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.smtp.server),
+            self.smtp.port,
+            self.smtp.certificate_checks,
             flags_readable,
         )
     }
@@ -237,30 +277,6 @@ fn get_readable_flags(flags: i32) -> String {
                 res += "AUTH_NORMAL ";
                 flag_added = true;
             }
-            if 1 << bit == 0x100 {
-                res += "IMAP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x200 {
-                res += "IMAP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x400 {
-                res += "IMAP_PLAIN ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x10000 {
-                res += "SMTP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x20000 {
-                res += "SMTP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x40000 {
-                res += "SMTP_PLAIN ";
-                flag_added = true;
-            }
             if flag_added {
                 res += &format!("{:#0x}", 1 << bit);
             }

src/message.rs

@@ -1096,30 +1096,70 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     let extension: &str = &path.extension()?.to_str()?.to_lowercase();
     let info = match extension {
+        // before using viewtype other than Viewtype::File,
+        // make sure, all target UIs support that type in the context of the used viewer/player.
+        // if in doubt, it is better to default to Viewtype::File that passes handing to an external app.
+        // (cmp. https://developer.android.com/guide/topics/media/media-formats )
         "3gp" => (Viewtype::Video, "video/3gpp"),
         "aac" => (Viewtype::Audio, "audio/aac"),
         "avi" => (Viewtype::Video, "video/x-msvideo"),
+        "doc" => (Viewtype::File, "application/msword"),
+        "docx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        ),
+        "epub" => (Viewtype::File, "application/epub+zip"),
         "flac" => (Viewtype::Audio, "audio/flac"),
         "gif" => (Viewtype::Gif, "image/gif"),
+        "html" => (Viewtype::File, "text/html"),
+        "htm" => (Viewtype::File, "text/html"),
+        "ico" => (Viewtype::File, "image/vnd.microsoft.icon"),
+        "jar" => (Viewtype::File, "application/java-archive"),
         "jpeg" => (Viewtype::Image, "image/jpeg"),
         "jpe" => (Viewtype::Image, "image/jpeg"),
         "jpg" => (Viewtype::Image, "image/jpeg"),
+        "json" => (Viewtype::File, "application/json"),
         "mov" => (Viewtype::Video, "video/quicktime"),
         "mp3" => (Viewtype::Audio, "audio/mpeg"),
         "mp4" => (Viewtype::Video, "video/mp4"),
+        "odp" => (
+            Viewtype::File,
+            "application/vnd.oasis.opendocument.presentation",
+        ),
+        "ods" => (
+            Viewtype::File,
+            "application/vnd.oasis.opendocument.spreadsheet",
+        ),
+        "odt" => (Viewtype::File, "application/vnd.oasis.opendocument.text"),
         "oga" => (Viewtype::Audio, "audio/ogg"),
         "ogg" => (Viewtype::Audio, "audio/ogg"),
-        "ogv" => (Viewtype::Video, "video/ogg"),
-        "opus" => (Viewtype::Audio, "audio/ogg"),
+        "ogv" => (Viewtype::File, "video/ogg"),
+        "opus" => (Viewtype::File, "audio/ogg"), // not supported eg. on Android 4
+        "otf" => (Viewtype::File, "font/otf"),
+        "pdf" => (Viewtype::File, "application/pdf"),
         "png" => (Viewtype::Image, "image/png"),
-        "spx" => (Viewtype::Audio, "audio/ogg"), // Ogg Speex Profile
-        "svg" => (Viewtype::Image, "image/svg+xml"),
+        "rar" => (Viewtype::File, "application/vnd.rar"),
+        "rtf" => (Viewtype::File, "application/rtf"),
+        "spx" => (Viewtype::File, "audio/ogg"), // Ogg Speex Profile
+        "svg" => (Viewtype::File, "image/svg+xml"),
         "tgs" => (Viewtype::Sticker, "application/x-tgsticker"),
+        "tiff" => (Viewtype::File, "image/tiff"),
+        "tif" => (Viewtype::File, "image/tiff"),
+        "ttf" => (Viewtype::File, "font/ttf"),
         "vcard" => (Viewtype::File, "text/vcard"),
         "vcf" => (Viewtype::File, "text/vcard"),
+        "wav" => (Viewtype::File, "audio/wav"),
+        "weba" => (Viewtype::File, "audio/webm"),
         "webm" => (Viewtype::Video, "video/webm"),
-        "webp" => (Viewtype::Image, "image/webp"),
+        "webp" => (Viewtype::Image, "image/webp"), // iOS via SDWebImage, Android since 4.0
         "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
+        "xhtml" => (Viewtype::File, "application/xhtml+xml"),
+        "xlsx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        ),
+        "xml" => (Viewtype::File, "application/vnd.ms-excel"),
+        "zip" => (Viewtype::File, "application/zip"),
         _ => {
             return None;
         }

src/mimefactory.rs

@@ -11,6 +11,7 @@ use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
+use crate::format_flowed::format_flowed;
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -910,11 +911,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        let flowed_text = format_flowed(final_text);
+
         let footer = &self.selfstatus;
         let message_text = format!(
             "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            escape_message_footer_marks(final_text),
+            escape_message_footer_marks(&flowed_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -926,7 +929,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         // Message is sent as text/plain, with charset = utf-8
         let main_part = PartBuilder::new()
-            .content_type(&mime::TEXT_PLAIN_UTF_8)
+            .header((
+                "Content-Type".to_string(),
+                "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+            ))
             .body(message_text);
         let mut parts = Vec::new();
 

src/mimeparser.rs

@@ -17,6 +17,7 @@ use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::error::{bail, Result};
 use crate::events::EventType;
+use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
@@ -604,7 +605,7 @@ impl MimeMessage {
                 skip the rest.  (see
                 https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
-                if let Some(first) = mail.subparts.iter().next() {
+                if let Some(first) = mail.subparts.get(0) {
                     any_part_added = self.parse_mime_recursive(context, first).await?;
                 }
             }
@@ -641,7 +642,7 @@ impl MimeMessage {
                             }
                         }
                         Some(_) => {
-                            if let Some(first) = mail.subparts.iter().next() {
+                            if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
                         }
@@ -715,6 +716,27 @@ impl MimeMessage {
                             simplify(out, self.has_chat_version())
                         };
 
+                        let is_format_flowed = if let Some(format) = mail.ctype.params.get("format")
+                        {
+                            format.as_str().to_ascii_lowercase() == "flowed"
+                        } else {
+                            false
+                        };
+
+                        let simplified_txt = if mime_type.type_() == mime::TEXT
+                            && mime_type.subtype() == mime::PLAIN
+                            && is_format_flowed
+                        {
+                            let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                delsp.as_str().to_ascii_lowercase() == "yes"
+                            } else {
+                                false
+                            };
+                            unformat_flowed(&simplified_txt, delsp)
+                        } else {
+                            simplified_txt
+                        };
+
                         if !simplified_txt.is_empty() {
                             let mut part = Part::default();
                             part.typ = Viewtype::Text;
@@ -1022,6 +1044,28 @@ impl MimeMessage {
             message::handle_ndn(context, failure_report, error).await
         }
     }
+
+    /// Returns timestamp of the parent message.
+    ///
+    /// If there is no parent message or it is not found in the
+    /// database, returns None.
+    pub async fn get_parent_timestamp(&self, context: &Context) -> Result<Option<i64>> {
+        let parent_timestamp = if let Some(field) = self
+            .get(HeaderDef::InReplyTo)
+            .and_then(|msgid| parse_message_id(msgid).ok())
+        {
+            context
+                .sql
+                .query_get_value_result(
+                    "SELECT timestamp FROM msgs WHERE rfc724_mid=?",
+                    paramsv![field],
+                )
+                .await?
+        } else {
+            None
+        };
+        Ok(parent_timestamp)
+    }
 }
 
 async fn update_gossip_peerstates(
@@ -1391,6 +1435,39 @@ mod tests {
         assert!(mimeparser.chat_disposition_notification_to.is_none());
     }
 
+    #[async_std::test]
+    async fn test_get_parent_timestamp() {
+        let context = TestContext::new().await;
+        let raw = b"From: foo@example.org\n\
+                    Content-Type: text/plain\n\
+                    Chat-Version: 1.0\n\
+                    In-Reply-To: <Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org>\n\
+                    \n\
+                    Some reply\n\
+                    ";
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            mimeparser.get_parent_timestamp(&context.ctx).await.unwrap(),
+            None
+        );
+        let timestamp = 1570435529;
+        context
+            .ctx
+            .sql
+            .execute(
+                "INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)",
+                paramsv!["Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org", timestamp],
+            )
+            .await
+            .expect("Failed to write to the database");
+        assert_eq!(
+            mimeparser.get_parent_timestamp(&context.ctx).await.unwrap(),
+            Some(timestamp)
+        );
+    }
+
     #[async_std::test]
     async fn test_mimeparser_with_context() {
         let context = TestContext::new().await;
@@ -2029,4 +2106,13 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         let test = parse_message_ids("  < ").unwrap();
         assert!(test.is_empty());
     }
+
+    #[test]
+    fn test_mime_parse_format_flowed() {
+        let mime_type = "text/plain; charset=utf-8; Format=Flowed; DelSp=No"
+            .parse::<mime::Mime>()
+            .unwrap();
+        let format_param = mime_type.get_param("format").unwrap();
+        assert_eq!(format_param.as_str().to_ascii_lowercase(), "flowed");
+    }
 }

src/param.rs

@@ -95,6 +95,12 @@ pub enum Param {
     Recipients = b'R',
 
     /// For Groups
+    ///
+    /// An unpromoted group has not had any messages sent to it and thus only exists on the
+    /// creator's device.  Any changes made to an unpromoted group do not need to send
+    /// system messages to the group members to update them of the changes.  Once a message
+    /// has been sent to a group it is promoted and group changes require sending system
+    /// messages to all members.
     Unpromoted = b'U',
 
     /// For Groups and Contacts

src/provider/mod.rs

@@ -6,7 +6,7 @@ use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
-#[derive(Debug, Copy, Clone, PartialEq, ToPrimitive)]
+#[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Status {
     OK = 1,
@@ -14,21 +14,29 @@ pub enum Status {
     BROKEN = 3,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Display, PartialEq, Copy, Clone, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Protocol {
     SMTP = 1,
     IMAP = 2,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Display, PartialEq, Copy, Clone, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Socket {
-    STARTTLS = 1,
-    SSL = 2,
+    Automatic = 0,
+    SSL = 1,
+    STARTTLS = 2,
+    Plain = 3,
 }
 
-#[derive(Debug, PartialEq)]
+impl Default for Socket {
+    fn default() -> Self {
+        Socket::Automatic
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
 #[repr(u8)]
 pub enum UsernamePattern {
     EMAIL = 1,
@@ -42,7 +50,7 @@ pub enum Oauth2Authorizer {
     Gmail = 2,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Server {
     pub protocol: Protocol,
     pub socket: Socket,
@@ -51,20 +59,6 @@ pub struct Server {
     pub username_pattern: UsernamePattern,
 }
 
-impl Server {
-    pub fn apply_username_pattern(&self, addr: String) -> String {
-        match self.username_pattern {
-            UsernamePattern::EMAIL => addr,
-            UsernamePattern::EMAILLOCALPART => {
-                if let Some(at) = addr.find('@') {
-                    return addr.split_at(at).0.to_string();
-                }
-                addr
-            }
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct ConfigDefault {
     pub key: Config,
@@ -83,25 +77,6 @@ pub struct Provider {
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-impl Provider {
-    pub fn get_server(&self, protocol: Protocol) -> Option<&Server> {
-        for record in self.server.iter() {
-            if record.protocol == protocol {
-                return Some(record);
-            }
-        }
-        None
-    }
-
-    pub fn get_imap_server(&self) -> Option<&Server> {
-        self.get_server(Protocol::IMAP)
-    }
-
-    pub fn get_smtp_server(&self) -> Option<&Server> {
-        self.get_server(Protocol::SMTP)
-    }
-}
-
 pub fn get_provider_info(addr: &str) -> Option<&Provider> {
     let domain = match addr.parse::<EmailAddress>() {
         Ok(addr) => addr.domain,
@@ -137,15 +112,16 @@ mod tests {
         let provider = get_provider_info("nauta.cu"); // this is no email address
         assert!(provider.is_none());
 
-        let provider = get_provider_info("user@nauta.cu").unwrap();
+        let addr = "user@nauta.cu";
+        let provider = get_provider_info(addr).unwrap();
         assert!(provider.status == Status::OK);
-        let server = provider.get_imap_server().unwrap();
+        let server = &provider.server[0];
         assert_eq!(server.protocol, Protocol::IMAP);
         assert_eq!(server.socket, Socket::STARTTLS);
         assert_eq!(server.hostname, "imap.nauta.cu");
         assert_eq!(server.port, 143);
         assert_eq!(server.username_pattern, UsernamePattern::EMAIL);
-        let server = provider.get_smtp_server().unwrap();
+        let server = &provider.server[1];
         assert_eq!(server.protocol, Protocol::SMTP);
         assert_eq!(server.socket, Socket::STARTTLS);
         assert_eq!(server.hostname, "smtp.nauta.cu");

src/securejoin.rs

@@ -15,7 +15,7 @@ use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::lot::LotState;
+use crate::lot::{Lot, LotState};
 use crate::message::Message;
 use crate::mimeparser::*;
 use crate::param::*;
@@ -67,6 +67,58 @@ macro_rules! get_qr_attr {
     };
 }
 
+#[derive(Debug, PartialEq)]
+pub(crate) enum BobStatus {
+    Error,
+    Success,
+}
+
+impl Default for BobStatus {
+    fn default() -> Self {
+        Self::Error
+    }
+}
+
+/// State for setup-contact/secure-join protocol joiner's side.
+///
+/// The setup-contact protocol needs to carry state for both the inviter (Alice) and the
+/// joiner/invitee (Bob).  For Alice this state is minimal and in the `tokens` table in the
+/// database.  For Bob this state is only carried live on the [Context] in this struct.
+#[derive(Debug, Default)]
+pub(crate) struct Bob {
+    /// The next message expected by the protocol.
+    expects: SecureJoinStep,
+    /// The final status of the last-run setup-contact/secure-join protocol.
+    ///
+    /// This is only meaningful if you know you have exited the protocol but have not
+    /// started a new one.
+    pub status: BobStatus,
+    /// The QR-scanned information of the currently running protocol.
+    pub qr_scan: Option<Lot>,
+}
+
+/// The next message expected by [Bob] in the setup-contact/secure-join protocol.
+#[derive(Debug, PartialEq)]
+enum SecureJoinStep {
+    /// No setup-contact protocol running.
+    NotActive,
+    /// Expecting the auth-required message.
+    ///
+    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
+    AuthRequired,
+    /// Expecting the contact-confirm message.
+    ///
+    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
+    /// 6b.
+    ContactConfirm,
+}
+
+impl Default for SecureJoinStep {
+    fn default() -> Self {
+        Self::NotActive
+    }
+}
+
 pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<String> {
     /*=======================================================
     ====             Alice - the inviter side            ====
@@ -159,8 +211,8 @@ async fn cleanup(
     join_vg: bool,
 ) -> ChatId {
     let mut bob = context.bob.write().await;
-    bob.expects = 0;
-    let ret_chat_id: ChatId = if bob.status == DC_BOB_SUCCESS {
+    bob.expects = SecureJoinStep::NotActive;
+    let ret_chat_id: ChatId = if bob.status == BobStatus::Success {
         if join_vg {
             chat::get_chat_id_by_grpid(
                 context,
@@ -223,7 +275,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
     join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
     {
         let mut bob = context.bob.write().await;
-        bob.status = 0;
+        bob.status = BobStatus::Error;
         bob.qr_scan = Some(qr_scan);
     }
     if fingerprint_equals_sender(
@@ -245,7 +297,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         // the scanned fingerprint matches Alice's key,
         // we can proceed to step 4b) directly and save two mails
         info!(context, "Taking protocol shortcut.");
-        context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+        context.bob.write().await.expects = SecureJoinStep::ContactConfirm;
         joiner_progress!(
             context,
             chat_id_2_contact_id(context, contact_chat_id).await,
@@ -276,7 +328,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
             return cleanup(&context, contact_chat_id, true, join_vg).await;
         }
     } else {
-        context.bob.write().await.expects = DC_VC_AUTH_REQUIRED;
+        context.bob.write().await.expects = SecureJoinStep::AuthRequired;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -521,7 +573,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 let bob = context.bob.read().await;
                 let scan = bob.qr_scan.as_ref();
                 scan.is_none()
-                    || bob.expects != DC_VC_AUTH_REQUIRED
+                    || bob.expects != SecureJoinStep::AuthRequired
                     || join_vg && scan.unwrap().state != LotState::QrAskVerifyGroup
             };
 
@@ -545,7 +597,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     },
                 )
                 .await;
-                context.bob.write().await.status = 0; // secure-join failed
+                context.bob.write().await.status = BobStatus::Error; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
@@ -558,14 +610,14 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Fingerprint mismatch on joiner-side.",
                 )
                 .await;
-                context.bob.write().await.status = 0; // secure-join failed
+                context.bob.write().await.status = BobStatus::Error; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
             let own_fingerprint = get_self_fingerprint(context).await.unwrap();
             joiner_progress!(context, contact_id, 400);
-            context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+            context.bob.write().await.expects = SecureJoinStep::ContactConfirm;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -709,7 +761,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 HandshakeMessage::Ignore
             };
 
-            if context.bob.read().await.expects != DC_VC_CONTACT_CONFIRM {
+            if context.bob.read().await.expects != SecureJoinStep::ContactConfirm {
                 info!(context, "Message belongs to a different handshake.",);
                 return Ok(abort_retval);
             }
@@ -756,7 +808,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Contact confirm message not encrypted.",
                 )
                 .await;
-                context.bob.write().await.status = 0;
+                context.bob.write().await.status = BobStatus::Error;
                 return Ok(abort_retval);
             }
 
@@ -788,7 +840,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(abort_retval);
             }
             secure_connection_established(context, contact_chat_id).await;
-            context.bob.write().await.expects = 0;
+            context.bob.write().await.expects = SecureJoinStep::NotActive;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -805,7 +857,7 @@ pub(crate) async fn handle_securejoin_handshake(
             )
             .await?;
 
-            context.bob.write().await.status = 1;
+            context.bob.write().await.status = BobStatus::Success;
             context.stop_ongoing().await;
             Ok(if join_vg {
                 HandshakeMessage::Propagate

src/smtp/mod.rs

@@ -10,9 +10,9 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::EventType;
-use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
 use crate::oauth2::*;
-use crate::provider::get_provider_info;
+use crate::provider::{get_provider_info, Socket};
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -94,31 +94,53 @@ impl Smtp {
             .unwrap_or_default()
     }
 
+    /// Connect using configured parameters.
+    pub async fn connect_configured(&mut self, context: &Context) -> Result<()> {
+        if self.is_connected().await {
+            return Ok(());
+        }
+
+        let lp = LoginParam::from_database(context, "configured_").await;
+        self.connect(
+            context,
+            &lp.smtp,
+            &lp.addr,
+            lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+        )
+        .await
+    }
+
     /// Connect using the provided login params.
-    pub async fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
+    pub async fn connect(
+        &mut self,
+        context: &Context,
+        lp: &ServerLoginParam,
+        addr: &str,
+        oauth2: bool,
+    ) -> Result<()> {
         if self.is_connected().await {
             warn!(context, "SMTP already connected.");
             return Ok(());
         }
 
-        if lp.send_server.is_empty() || lp.send_port == 0 {
+        if lp.server.is_empty() || lp.port == 0 {
             context.emit_event(EventType::ErrorNetwork("SMTP bad parameters.".into()));
             return Err(Error::BadParameters);
         }
 
         let from =
-            EmailAddress::new(lp.addr.clone()).map_err(|err| Error::InvalidLoginAddress {
-                address: lp.addr.clone(),
+            EmailAddress::new(addr.to_string()).map_err(|err| Error::InvalidLoginAddress {
+                address: addr.to_string(),
                 error: err,
             })?;
 
         self.from = Some(from);
 
-        let domain = &lp.send_server;
-        let port = lp.send_port as u16;
+        let domain = &lp.server;
+        let port = lp.port;
 
-        let provider = get_provider_info(&lp.addr);
-        let strict_tls = match lp.smtp_certificate_checks {
+        let provider = get_provider_info(addr);
+        let strict_tls = match lp.certificate_checks {
             CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
             CertificateChecks::Strict => true,
             CertificateChecks::AcceptInvalidCertificates
@@ -127,17 +149,16 @@ impl Smtp {
         let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
-        let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
+        let (creds, mechanism) = if oauth2 {
             // oauth2
-            let addr = &lp.addr;
-            let send_pw = &lp.send_pw;
+            let send_pw = &lp.password;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false).await;
             if access_token.is_none() {
                 return Err(Error::Oauth2Error {
                     address: addr.to_string(),
                 });
             }
-            let user = &lp.send_user;
+            let user = &lp.user;
             (
                 smtp::authentication::Credentials::new(
                     user.to_string(),
@@ -147,8 +168,8 @@ impl Smtp {
             )
         } else {
             // plain
-            let user = lp.send_user.clone();
-            let pw = lp.send_pw.clone();
+            let user = lp.user.clone();
+            let pw = lp.password.clone();
             (
                 smtp::authentication::Credentials::new(user, pw),
                 vec![
@@ -158,12 +179,9 @@ impl Smtp {
             )
         };
 
-        let security = if 0
-            != lp.server_flags & (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_PLAIN) as i32
-        {
-            smtp::ClientSecurity::Opportunistic(tls_parameters)
-        } else {
-            smtp::ClientSecurity::Wrapper(tls_parameters)
+        let security = match lp.security {
+            Socket::STARTTLS | Socket::Plain => smtp::ClientSecurity::Opportunistic(tls_parameters),
+            _ => smtp::ClientSecurity::Wrapper(tls_parameters),
         };
 
         let client = smtp::SmtpClient::with_security((domain.as_str(), port), security)
@@ -196,7 +214,7 @@ impl Smtp {
 
         context.emit_event(EventType::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
-            lp.send_user,
+            lp.user,
         )));
 
         Ok(())

src/sql.rs

@@ -1288,6 +1288,43 @@ async fn open(
             update_icons = true;
             sql.set_raw_config_int(context, "dbversion", 66).await?;
         }
+        if dbversion < 67 {
+            info!(context, "[migration] v67");
+            for prefix in &["", "configured_"] {
+                if let Some(server_flags) = sql
+                    .get_raw_config_int(context, format!("{}server_flags", prefix))
+                    .await
+                {
+                    let imap_socket_flags = server_flags & 0x700;
+                    let key = format!("{}mail_security", prefix);
+                    match imap_socket_flags {
+                        0x100 => sql.set_raw_config_int(context, key, 2).await?, // STARTTLS
+                        0x200 => sql.set_raw_config_int(context, key, 1).await?, // SSL/TLS
+                        0x400 => sql.set_raw_config_int(context, key, 3).await?, // Plain
+                        _ => sql.set_raw_config_int(context, key, 0).await?,
+                    }
+                    let smtp_socket_flags = server_flags & 0x70000;
+                    let key = format!("{}send_security", prefix);
+                    match smtp_socket_flags {
+                        0x10000 => sql.set_raw_config_int(context, key, 2).await?, // STARTTLS
+                        0x20000 => sql.set_raw_config_int(context, key, 1).await?, // SSL/TLS
+                        0x40000 => sql.set_raw_config_int(context, key, 3).await?, // Plain
+                        _ => sql.set_raw_config_int(context, key, 0).await?,
+                    }
+                }
+            }
+            sql.set_raw_config_int(context, "dbversion", 67).await?;
+        }
+        if dbversion < 68 {
+            info!(context, "[migration] v68");
+            // the index is used to speed up get_fresh_msg_cnt(), see comment there for more details
+            sql.execute(
+                "CREATE INDEX IF NOT EXISTS msgs_index7 ON msgs (state, hidden, chat_id);",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 68).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)

test-data/autoconfig/lakenet.ch.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<clientConfig version="1.1">
+  <emailProvider id="lakenet.ch">
+    <domain>%EMAILDOMAIN%</domain>
+    <displayName>%EMAILDOMAIN% Mail</displayName>
+    <displayShortName>%EMAILDOMAIN%</displayShortName>
+    <incomingServer type="imap">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="imap">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>143</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="pop3">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="pop3">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>110</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url="https://www.lakenet.ch/">
+      <descr lang="it">Impostazioni per le e-mail LakeNet</descr>
+      <descr lang="fr">Reglages pour le courriel e-mail LakeNet</descr>
+      <descr lang="en">Settings for LakeNet's e-mail accounts</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url="https://lakenet.ch/webmail/" />
+    <loginPageInfo url="https://lakenet.ch/webmail/">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id="rcmloginuser" name="_user"/>
+      <passwordField id="rcmloginpwd" name="_pass"/>
+      <loginButton id="rcmloginsubmit"/>
+    </loginPageInfo>
+  </webMail>
+  <clientConfigUpdate url="https://lakenet.ch/.well-known/autoconfig/mail/config-v1.1.xml" />
+</clientConfig>

test-data/autoconfig/outlook.com.xml

@@ -0,0 +1,71 @@
+<clientConfig version="1.1">
+  <!-- Retrieved from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11 -->
+  <emailProvider id="outlook.com">
+    <domain>hotmail.com</domain>
+    <domain>hotmail.co.uk</domain>
+    <domain>hotmail.co.jp</domain>
+    <domain>hotmail.com.br</domain>
+    <domain>hotmail.de</domain>
+    <domain>hotmail.fr</domain>
+    <domain>hotmail.it</domain>
+    <domain>hotmail.es</domain>
+    <domain>live.com</domain>
+    <domain>live.co.uk</domain>
+    <domain>live.co.jp</domain>
+    <domain>live.de</domain>
+    <domain>live.fr</domain>
+    <domain>live.it</domain>
+    <domain>live.jp</domain>
+    <domain>msn.com</domain>
+    <domain>outlook.com</domain>
+    <displayName>Outlook.com (Microsoft)</displayName>
+    <displayShortName>Outlook</displayShortName>
+    <incomingServer type="exchange">
+      <hostname>outlook.office365.com</hostname>
+      <port>443</port>
+      <username>%EMAILADDRESS%</username>
+      <socketType>SSL</socketType>
+      <authentication>OAuth2</authentication>
+      <owaURL>https://outlook.office365.com/owa/</owaURL>
+      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
+      <useGlobalPreferredServer>true</useGlobalPreferredServer>
+    </incomingServer>
+    <incomingServer type="imap">
+      <hostname>outlook.office365.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="pop3">
+      <hostname>outlook.office365.com</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <pop3>
+        <leaveMessagesOnServer>true</leaveMessagesOnServer>
+        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
+      </pop3>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname>smtp.office365.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url="http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app">
+      <descr lang="en">Set up an email app with Outlook.com</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url="https://www.outlook.com/"/>
+    <loginPageInfo url="https://www.outlook.com/">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id="i0116" name="login"/>
+      <passwordField id="i0118" name="passwd"/>
+      <loginButton id="idSIButton9" name="SI"/>
+    </loginPageInfo>
+  </webMail>
+</clientConfig>";

test-data/message/wrong-html.eml

@@ -0,0 +1,114 @@
+Return-Path: <camping@otherdomain.eu>
+X-Original-To: alice@example.com
+Delivered-To: m045a7e8@dd37930.kasserver.com
+Received: from mout.kundenserver.de (mout.kundenserver.de [212.227.126.131])
+	by dd37930.kasserver.com (Postfix) with ESMTPS id 271F34B4258C
+	for <holger@somedomain.de>; Thu,  6 Aug 2020 18:40:32 +0200 (CEST)
+Received: from oxbsltgw18.schlund.de ([172.19.249.35]) by
+ mrelayeu.kundenserver.de (mreue009 [213.165.67.103]) with ESMTPSA (Nemesis)
+ id 1MpDRv-1kW93Y0lGZ-00qjvh for <holger@somedomain.de>; Thu, 06 Aug 2020
+ 18:40:31 +0200
+Date: Thu, 6 Aug 2020 18:40:30 +0200 (CEST)
+From: Camping <camping@otherdomain.eu>
+Reply-To: Camping <camping@otherdomain.eu>
+To: Alice <alice@example.com>
+Message-ID: <512278196.1287440.1596732031020@email.ionos.fr>
+Subject: Re: subj?
+MIME-Version: 1.0
+Content-Type: multipart/related;
+	boundary="----=_Part_1287438_2124736777.1596732031007"
+X-Mailer: Open-Xchange Mailer v7.10.1-Rev32
+X-Originating-Client: open-xchange-appsuite
+
+------=_Part_1287438_2124736777.1596732031007
+MIME-Version: 1.0
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+
+<!doctype html>
+<html>
+ <head>=20
+  <meta charset=3D"UTF-8">=20
+ </head>
+ <body>
+  <div class=3D"default-style">
+   Guten Abend,
+   <br>
+  </div>
+  <div class=3D"default-style">
+   <br>
+  </div>
+  <div class=3D"default-style">
+   Lots of text
+   <br>
+  </div>
+  <div class=3D"default-style">
+   text with Umlaut =C3=A4...
+   <br>
+  </div>
+  <div class=3D"default-style">
+   MfG
+   <br>
+  </div>
+  <div class=3D"default-style">
+   <br>
+  </div>
+  <div class=3D"io-ox-signature">
+   <p>--------------------------------------<br></p>
+   <p style=3D"text-align: left;"><a href=3D"https://example.com=
+/">Camping </a><br></p>
+   <p>someaddress<br></p>
+   <p>sometown</p>
+   <p><img alt=3D"" class=3D"aspect-ratio" style=3D"width: 505px; height: 1=
+68px; max-width: 100%;" id=3D"d5cd260f-1b1f-4bfa-81d7-c0a74923b979" src=3D"=
+cid:d5cd260f-1b1f-4bfa-81d7-c0a74923b979" width=3D"505" height=3D"168"><br>=
+</p>
+  </div>
+  <blockquote type=3D"cite">
+   <div>
+    Le 5 ao=C3=BBt 2020 =C3=A0 10:46, holger &lt;
+    <a href=3D"mailto:holger@somedomain.de">holger@somedomain.de</a>&gt; a =
+=C3=A9crit&nbsp;:
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    Bonjour,
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    --
+   </div>
+   <div>
+    Sent with my Delta Chat Messenger:=20
+    <a href=3D"https://delta.chat" rel=3D"noopener" target=3D"_blank">https=
+://delta.chat</a>
+    <br>
+   </div>
+  </blockquote>=20
+ </body>
+</html>
+------=_Part_1287438_2124736777.1596732031007
+Content-Type: image/png
+Content-Transfer-Encoding: base64
+Content-ID: <d5cd260f-1b1f-4bfa-81d7-c0a74923b979>
+Content-Disposition: inline
+
+iVBORw0KGgoAAAANSUhEUgAAAlgAAADICAYAAAA0n5+2AAAgAElEQVR4nCzaZ3sjhIGo7fyMPSch
+MN29S7YsW71avfcuWZJlS3K33HuvY08fpjEVhoGhDh0CSSAESEhCgDRSKNns5mx2k8AM5Dznw/v+
+[scrubbed]
+/kNZ/B9Uub32fzngSwAAAABJRU5ErkJggg==
+------=_Part_1287438_2124736777.1596732031007--
+