updates

https://github.com/stjepang/polling/pull/10

.circleci/config.yml

@@ -138,6 +138,12 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+
   remote_tests_python:
     machine: true
     steps:
@@ -172,6 +178,11 @@ workflows:
     jobs:
       # - cargo_fetch
 
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
       - remote_tests_python:
           filters:
             tags:

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all
+          args: --workspace --tests --examples
 
 
   build_and_test:

CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog 
 
+## 1.46.0
+
+- breaking change: `dc_configure()` report errors in
+  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
+  #1886 #1905
+
+- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
+  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
+
+- parse multiple servers in Mozilla autoconfig #1860
+
+- try multiple servers for each protocol #1871
+
+- do IMAP and SMTP configuration in parallel #1891
+
+- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
+
+- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
+
+- do not reset peerstate on encrypted messages,
+  ignore reordered autocrypt headers #1885 #1890
+
+- always sort message replies after parent message #1852
+
+- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
+
+- improve mimetype guessing for PDF and many other formats #1857 #1861
+
+- improve accepting invalid html #1851
+
+- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
+
+- tweak HELO command #1908
+
+- make `dc_accounts_get_all()` return accounts sorted #1909
+
+- fix KML coordinates precision used for location streaming #1872
+
+- fix cancelling import/export #1855
+
+
 ## 1.45.0
 
 - add `dc_accounts_t` account manager object and related api functions #1784

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.45.0"
+version = "1.46.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -17,10 +17,10 @@ hex = "0.4.0"
 sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+surf = { version = "=2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = "0.3"
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.4.0"
@@ -102,3 +102,6 @@ repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 
+[patch.crates-io]
+polling = { git = "https://github.com/oblique/polling", branch = "master"}
+async-std = { git = "https://github.com/async-rs/async-std", branch = "master"}

ci_scripts/remote_python_packaging.sh

@@ -30,6 +30,7 @@ set +x
 
 ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     set +x -e
+    shopt -s huponexit
     cd $BUILDDIR
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_python.sh

@@ -24,6 +24,11 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     # let's share the target dir with our last run on this branch/job-type
     # cargo will make sure to block/unblock us properly 

ci_scripts/remote_tests_rust.sh

@@ -20,6 +20,9 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     # let's share the target dir with our last run on this branch/job-type
     # cargo will make sure to block/unblock us properly 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.45.0"
+version = "1.46.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -274,10 +274,12 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `mail_user`    = IMAP-username, guessed if left out
  * - `mail_pw`      = IMAP-password (always needed)
  * - `mail_port`    = IMAP-port, guessed if left out
+ * - `mail_security`= IMAP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `send_server`  = SMTP-server, guessed if left out
  * - `send_user`    = SMTP-user, guessed if left out
  * - `send_pw`      = SMTP-password, guessed if left out
  * - `send_port`    = SMTP-port, guessed if left out
+ * - `send_security`= SMTP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
@@ -1490,17 +1492,6 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
-/*
- * Empty IMAP server folder: delete all messages.
- * Deprecated, use dc_set_config() with the key "delete_server_after" instead.
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param flags What to delete, a combination of the @ref DC_EMPTY flags
- * @return None.
- */
-void            dc_empty_server              (dc_context_t* context, uint32_t flags);
-
 
 /**
  * Forward messages to another chat.
@@ -2369,7 +2360,7 @@ dc_context_t*  dc_accounts_get_account          (dc_accounts_t* accounts, uint32
 
 /**
  * Get the currently selected account.
- * If there is at least once account in the account-manager,
+ * If there is at least one account in the account-manager,
  * there is always a selected one.
  * To change the selected account, use dc_accounts_select_account();
  * also adding/importing/migrating accounts may change the selection.
@@ -4167,6 +4158,44 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_MSG_VIDEOCHAT_INVITATION 70
 
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_SOCKET DC_SOCKET
+ *
+ * These constants configure socket security.
+ * To set socket security, use dc_set_config() with the keys "mail_security" and/or "send_security".
+ * If no socket-configuration is explicitly specified, #DC_SOCKET_AUTO is used.
+ *
+ * @addtogroup DC_SOCKET
+ * @{
+ */
+
+/**
+ * Choose socket security automatically.
+ */
+#define DC_SOCKET_AUTO 0
+
+
+/**
+ * Connect via SSL/TLS.
+ */
+#define DC_SOCKET_SSL 1
+
+
+/**
+ * Connect via STARTTLS.
+ */
+#define DC_SOCKET_STARTTLS 2
+
+
+/**
+ * Connect unencrypted, this should not be used.
+ */
+#define DC_SOCKET_PLAIN 3
 
 /**
  * @}
@@ -4201,54 +4230,11 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_LP_AUTH_NORMAL                0x4
 
 
-/**
- * Connect to IMAP via STARTTLS.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_IMAP_SOCKET_STARTTLS     0x100
-
-
-/**
- * Connect to IMAP via SSL.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_IMAP_SOCKET_SSL          0x200
-
-
-/**
- * Connect to IMAP unencrypted, this should not be used.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_IMAP_SOCKET_PLAIN        0x400
-
-
-/**
- * Connect to SMTP via STARTTLS.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_SMTP_SOCKET_STARTTLS   0x10000
-
-
-/**
- * Connect to SMTP via SSL.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_SMTP_SOCKET_SSL        0x20000
-
-
-/**
- * Connect to SMTP unencrypted, this should not be used.
- * If this flag is set, automatic configuration is skipped.
- */
-#define DC_LP_SMTP_SOCKET_PLAIN      0x40000 ///<
-
 /**
  * @}
  */
 
 #define DC_LP_AUTH_FLAGS        (DC_LP_AUTH_OAUTH2|DC_LP_AUTH_NORMAL) // if none of these flags are set, the default is chosen
-#define DC_LP_IMAP_SOCKET_FLAGS (DC_LP_IMAP_SOCKET_STARTTLS|DC_LP_IMAP_SOCKET_SSL|DC_LP_IMAP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
-#define DC_LP_SMTP_SOCKET_FLAGS (DC_LP_SMTP_SOCKET_STARTTLS|DC_LP_SMTP_SOCKET_SSL|DC_LP_SMTP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 
 /**
  * @defgroup DC_CERTCK DC_CERTCK
@@ -4284,10 +4270,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
-#define DC_EMPTY_MVBOX 0x01 // Deprecated, flag for dc_empty_server(): Clear all mvbox messages
-#define DC_EMPTY_INBOX 0x02 // Deprecated, flag for dc_empty_server(): Clear all INBOX messages
-
-
 /**
  * @class dc_event_emitter_t
  *
@@ -4509,14 +4491,6 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
-/**
- * Emitted when an IMAP folder was emptied.
- *
- * @param data1 0
- * @param data2 (char*) Folder name.
- */
-#define DC_EVENT_IMAP_FOLDER_EMPTIED  106
-
 /**
  * Emitted when a new blob file was successfully written
  *
@@ -4695,7 +4669,7 @@ void dc_event_unref(dc_event_t* event);
  * Inform about the configuration progress started by dc_configure().
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
- * @param data2 0
+ * @param data2 (char*) progress comment, error message or NULL if not applicable
  */
 #define DC_EVENT_CONFIGURE_PROGRESS       2041
 
@@ -4758,18 +4732,8 @@ void dc_event_unref(dc_event_t* event);
  */
 
 
-#define DC_EVENT_FILE_COPIED         2055 // not used anymore
-#define DC_EVENT_IS_OFFLINE          2081 // not used anymore
-#define DC_EVENT_GET_STRING          2091 // not used anymore, use dc_set_stock_translation()
-#define DC_ERROR_SEE_STRING          0    // not used anymore
-#define DC_ERROR_SELF_NOT_IN_GROUP   1    // not used anymore
-#define DC_STR_SELFNOTINGRP          21   // not used anymore
 #define DC_EVENT_DATA1_IS_STRING(e)  0    // not used anymore 
-#define DC_EVENT_DATA2_IS_STRING(e)  ((e)==DC_EVENT_IMEX_FILE_WRITTEN || ((e)>=100 && (e)<=499))
-#define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
-#define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
-#define dc_archive_chat(a,b,c)  dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore
-#define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0)      // not used anymore
+#define DC_EVENT_DATA2_IS_STRING(e)  ((e)==DC_EVENT_CONFIGURE_PROGRESS || (e)==DC_EVENT_IMEX_FILE_WRITTEN || ((e)>=100 && (e)<=499))
 
 
 /*
@@ -4960,8 +4924,9 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_EPHEMERAL_FOUR_WEEKS       81
 #define DC_STR_VIDEOCHAT_INVITATION       82
 #define DC_STR_VIDEOCHAT_INVITE_MSG_BODY  83
+#define DC_STR_CONFIGURATION_FAILED       84
 
-#define DC_STR_COUNT                      83
+#define DC_STR_COUNT                      84
 
 /*
  * @}

deltachat-ffi/src/lib.rs

@@ -355,7 +355,6 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SmtpMessageSent(_)
         | EventType::ImapMessageDeleted(_)
         | EventType::ImapMessageMoved(_)
-        | EventType::ImapFolderEmptied(_)
         | EventType::NewBlobFile(_)
         | EventType::DeletedBlobFile(_)
         | EventType::Warning(_)
@@ -373,7 +372,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
             let id = id.unwrap_or_default();
             id as libc::c_int
         }
-        EventType::ConfigureProgress(progress) | EventType::ImexProgress(progress) => {
+        EventType::ConfigureProgress { progress, .. } | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
         EventType::ImexFileWritten(_) => 0,
@@ -398,7 +397,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::SmtpMessageSent(_)
         | EventType::ImapMessageDeleted(_)
         | EventType::ImapMessageMoved(_)
-        | EventType::ImapFolderEmptied(_)
         | EventType::NewBlobFile(_)
         | EventType::DeletedBlobFile(_)
         | EventType::Warning(_)
@@ -407,7 +405,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ErrorSelfNotInGroup(_)
         | EventType::ContactsChanged(_)
         | EventType::LocationChanged(_)
-        | EventType::ConfigureProgress(_)
+        | EventType::ConfigureProgress { .. }
         | EventType::ImexProgress(_)
         | EventType::ImexFileWritten(_)
         | EventType::ChatModified(_) => 0,
@@ -438,7 +436,6 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SmtpMessageSent(msg)
         | EventType::ImapMessageDeleted(msg)
         | EventType::ImapMessageMoved(msg)
-        | EventType::ImapFolderEmptied(msg)
         | EventType::NewBlobFile(msg)
         | EventType::DeletedBlobFile(msg)
         | EventType::Warning(msg)
@@ -456,11 +453,17 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::ChatModified(_)
         | EventType::ContactsChanged(_)
         | EventType::LocationChanged(_)
-        | EventType::ConfigureProgress(_)
         | EventType::ImexProgress(_)
         | EventType::SecurejoinInviterProgress { .. }
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
+        EventType::ConfigureProgress { comment, .. } => {
+            if let Some(comment) = comment {
+                comment.to_c_string().unwrap_or_default().into_raw()
+            } else {
+                ptr::null_mut()
+            }
+        }
         EventType::ImexFileWritten(file) => {
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
@@ -1423,17 +1426,6 @@ pub unsafe extern "C" fn dc_delete_msgs(
     block_on(message::delete_msgs(&ctx, &msg_ids))
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_empty_server(context: *mut dc_context_t, flags: u32) {
-    if context.is_null() || flags == 0 {
-        eprintln!("ignoring careless call to dc_empty_server()");
-        return;
-    }
-    let ctx = &*context;
-
-    block_on(message::dc_empty_server(&ctx, flags))
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_forward_msgs(
     context: *mut dc_context_t,
@@ -1891,8 +1883,13 @@ pub unsafe extern "C" fn dc_join_securejoin(
     }
     let ctx = &*context;
 
-    block_on(async move { securejoin::dc_join_securejoin(&ctx, &to_string_lossy(qr)).await })
-        .to_u32()
+    block_on(async move {
+        securejoin::dc_join_securejoin(&ctx, &to_string_lossy(qr))
+            .await
+            .map(|chatid| chatid.to_u32())
+            .log_err(ctx, "failed dc_join_securejoin() call")
+            .unwrap_or_default()
+    })
 }
 
 #[no_mangle]

examples/repl/cmdline.rs

@@ -405,7 +405,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
-                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -1092,11 +1091,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 seconds, device_cnt, server_cnt
             );
         }
-        "emptyserver" => {
-            ensure!(!arg1.is_empty(), "Argument <flags> missing");
-
-            message::dc_empty_server(&context, arg1.parse()?).await;
-        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -80,11 +80,21 @@ fn receive_event(event: EventType) {
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        EventType::ConfigureProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-            );
+        EventType::ConfigureProgress { progress, comment } => {
+            if let Some(comment) = comment {
+                info!(
+                    "{}",
+                    yellow.paint(format!(
+                        "Received CONFIGURE_PROGRESS({} ‰, {})",
+                        progress, comment
+                    ))
+                );
+            } else {
+                info!(
+                    "{}",
+                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                );
+            }
         }
         EventType::ImexProgress(progress) => {
             info!(
@@ -410,7 +420,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await;
+                dc_join_securejoin(&ctx, arg1).await?;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -10,7 +10,7 @@ use deltachat::EventType;
 
 fn cb(event: EventType) {
     match event {
-        EventType::ConfigureProgress(progress) => {
+        EventType::ConfigureProgress { progress, .. } => {
             log::info!("progress: {}", progress);
         }
         EventType::Info(msg) => {

python/src/deltachat/account.py

@@ -179,17 +179,6 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def empty_server_folders(self, inbox=False, mvbox=False):
-        """ empty server folders. """
-        flags = 0
-        if inbox:
-            flags |= const.DC_EMPTY_INBOX
-        if mvbox:
-            flags |= const.DC_EMPTY_MVBOX
-        if not flags:
-            raise ValueError("no flags set")
-        lib.dc_empty_server(self._dc_context, flags)
-
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -216,7 +205,7 @@ class Account(object):
     def create_contact(self, obj, name=None):
         """ create a (new) Contact or return an existing one.
 
-        Calling this method will always resulut in the same
+        Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
         with that e-mail address, it is unblocked and its display
         `name` is updated if specified.

python/src/deltachat/contact.py

@@ -51,6 +51,10 @@ class Contact(object):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
+    def set_blocked(self, block=True):
+        """ Block or unblock a contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, block)
+
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/cutil.py

@@ -17,7 +17,8 @@ def iter_array(dc_array_t, constructor):
 
 
 def from_dc_charpointer(obj):
-    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    if obj != ffi.NULL:
+        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:

python/tests/test_account.py

@@ -819,18 +819,12 @@ class TestOnlineAccount:
         assert msg_in.text == "message2"
         assert msg_in.is_forwarded()
 
-    def test_send_self_message_and_empty_folder(self, acfactory, lp):
+    def test_send_self_message(self, acfactory, lp):
         ac1 = acfactory.get_one_online_account(mvbox=True, move=True)
         lp.sec("ac1: create self chat")
         chat = ac1.get_self_contact().create_chat()
         chat.send_text("hello")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
-        ac1.empty_server_folders(inbox=True, mvbox=True)
-        ev1 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
-        ev2 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
-        boxes = [ev1.data2, ev2.data2]
-        boxes.remove("INBOX")
-        assert len(boxes) == 1 and boxes[0].endswith("DeltaChat")
 
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -986,7 +980,10 @@ class TestOnlineAccount:
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
         lp.sec("sending multi-line non-unicode message from ac1 to ac2")
-        text1 = "hello\nworld"
+        text1 = (
+            "hello\nworld\nthis is a very long message that should be"
+            + " wrapped using format=flowed and unwrapped on the receiver"
+        )
         msg_out = chat.send_text(text1)
         assert not msg_out.is_encrypted()
 
@@ -1402,6 +1399,46 @@ class TestOnlineAccount:
         assert ev.action == "removed"
         assert ev.message.get_sender_contact().addr == ac1_addr
 
+    def test_system_group_msg_from_blocked_user(self, acfactory, lp):
+        """
+        Tests that a blocked user removes you from a group.
+        The message has to be fetched even though the user is blocked
+        to avoid inconsistent group state.
+        Also tests blocking in general.
+        """
+        lp.sec("Create a group chat with ac1 and ac2")
+        (ac1, ac2) = acfactory.get_two_online_accounts()
+        acfactory.introduce_each_other((ac1, ac2))
+        chat_on_ac1 = ac1.create_group_chat("title", contacts=[ac2])
+        chat_on_ac1.send_text("First group message")
+        chat_on_ac2 = ac2._evtracker.wait_next_incoming_message().chat
+
+        lp.sec("ac1 blocks ac2")
+        contact = ac1.create_contact(ac2)
+        contact.set_blocked()
+        assert contact.is_blocked()
+
+        lp.sec("ac2 sends a message to ac1 that does not arrive because it is blocked")
+        ac2.create_chat(ac1).send_text("This will not arrive!")
+
+        lp.sec("ac2 sends a group message to ac1 that arrives")
+        # Groups would be hardly usable otherwise: If you have blocked some
+        # users, they write messages and you only see replies to them without context
+        chat_on_ac2.send_text("This will arrive")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "This will arrive"
+        message_texts = [m.text for m in chat_on_ac1.get_messages()]
+        assert len(message_texts) == 2
+        assert "First group message" in message_texts
+        assert "This will arrive" in message_texts
+
+        lp.sec("ac2 removes ac1 from their group")
+        assert ac1.get_self_contact() in chat_on_ac1.get_contacts()
+        assert contact.is_blocked()
+        chat_on_ac2.remove_contact(ac1)
+        ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert not ac1.get_self_contact() in chat_on_ac1.get_contacts()
+
     def test_set_get_group_image(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1690,6 +1727,46 @@ class TestOnlineAccount:
 
         assert len(imap2.get_all_messages()) == 1
 
+    def test_name_changes(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("displayname", "Account 1")
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        contact = None
+
+        def update_name():
+            """Send a message from ac1 to ac2 to update the name"""
+            nonlocal contact
+            chat12.send_text("Hello")
+            msg = ac2._evtracker.wait_next_incoming_message()
+            contact = msg.get_sender_contact()
+            return contact.name
+
+        assert update_name() == "Account 1"
+
+        ac1.set_config("displayname", "Account 1 revision 2")
+        assert update_name() == "Account 1 revision 2"
+
+        # Explicitly rename contact on ac2 to "Renamed"
+        ac2.create_contact(contact, name="Renamed")
+        assert contact.name == "Renamed"
+
+        # ac1 also renames itself into "Renamed"
+        assert update_name() == "Renamed"
+        ac1.set_config("displayname", "Renamed")
+        assert update_name() == "Renamed"
+
+        # Contact name was set to "Renamed" explicitly before,
+        # so it should not be changed.
+        ac1.set_config("displayname", "Renamed again")
+        updated_name = update_name()
+        if updated_name == "Renamed again":
+            # Known bug, mark as XFAIL
+            pytest.xfail("Contact was renamed after explicit rename")
+        else:
+            # No renames should happen after explicit rename
+            assert updated_name == "Renamed"
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):
@@ -1815,8 +1892,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "cannot login" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_user(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -1824,8 +1900,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "cannot login" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_domain(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -1833,5 +1908,4 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "could not connect" in ev.data2.lower()
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")

src/accounts.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::collections::BTreeMap;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::task::{Context as TaskContext, Poll};
@@ -20,7 +20,7 @@ use crate::events::Event;
 pub struct Accounts {
     dir: PathBuf,
     config: Config,
-    accounts: Arc<RwLock<HashMap<u32, Context>>>,
+    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
 }
 
 impl Accounts {
@@ -342,9 +342,9 @@ impl Config {
         })
     }
 
-    pub async fn load_accounts(&self) -> Result<HashMap<u32, Context>> {
+    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
         let cfg = &*self.inner.read().await;
-        let mut accounts = HashMap::with_capacity(cfg.accounts.len());
+        let mut accounts = BTreeMap::new();
         for account_config in &cfg.accounts {
             let ctx = Context::new(
                 cfg.os_name.clone(),
@@ -530,4 +530,23 @@ mod tests {
             ctx.get_config(crate::config::Config::Addr).await.unwrap()
         );
     }
+
+    /// Tests that accounts are sorted by ID.
+    #[async_std::test]
+    async fn test_accounts_sorted() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        for expected_id in 2..10 {
+            let id = accounts.add_account().await.unwrap();
+            assert_eq!(id, expected_id);
+        }
+
+        let ids = accounts.get_all().await;
+        for (i, expected_id) in (1..10).enumerate() {
+            assert_eq!(ids.get(i), Some(&expected_id));
+        }
+    }
 }

src/blob.rs

@@ -685,7 +685,7 @@ mod tests {
 
         let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
         assert!(stem.contains("foo"));
-        assert!(!stem.contains("?"));
+        assert!(!stem.contains('?'));
         assert_eq!(ext, ".bar");
 
         let (stem, ext) = BlobObject::sanitise_name("no-extension");
@@ -698,10 +698,10 @@ mod tests {
         assert!(!stem.contains("ignored"));
         assert!(stem.contains("this"));
         assert!(stem.contains("forbidden"));
-        assert!(!stem.contains("/"));
-        assert!(!stem.contains("\\"));
-        assert!(!stem.contains(":"));
-        assert!(!stem.contains("*"));
-        assert!(!stem.contains("?"));
+        assert!(!stem.contains('/'));
+        assert!(!stem.contains('\\'));
+        assert!(!stem.contains(':'));
+        assert!(!stem.contains('*'));
+        assert!(!stem.contains('?'));
     }
 }

src/chat.rs

@@ -375,6 +375,16 @@ impl ChatId {
     }
 
     pub async fn get_fresh_msg_cnt(self, context: &Context) -> usize {
+        // this function is typically used to show a badge counter beside _each_ chatlist item.
+        // to make this as fast as possible, esp. on older devices, we added an combined index over the rows used for querying.
+        // so if you alter the query here, you may want to alter the index over `(state, hidden, chat_id)` in `sql.rs`.
+        //
+        // the impact of the index is significant once the database grows:
+        // - on an older android4 with 18k messages, query-time decreased from 110ms to 2ms
+        // - on an mid-class moto-g or iphone7 with 50k messages, query-time decreased from 26ms or 6ms to 0-1ms
+        // the times are average, no matter if there are fresh messages or not -
+        // and have to be multiplied by the number of items shown at once on the chatlist,
+        // so savings up to 2 seconds are possible on older devices - newer ones will feel "snappier" :)
         context
             .sql
             .query_get_value::<i32>(
@@ -832,7 +842,11 @@ impl Chat {
             /* check if we want to encrypt this message.  If yes and circumstances change
             so that E2EE is no longer available at a later point (reset, changed settings),
             we might not send the message out at all */
-            if msg.param.get_int(Param::ForcePlaintext).unwrap_or_default() == 0 {
+            if !msg
+                .param
+                .get_bool(Param::ForcePlaintext)
+                .unwrap_or_default()
+            {
                 let mut can_encrypt = true;
                 let mut all_mutual = context.get_config_bool(Config::E2eeEnabled).await;
 
@@ -1156,7 +1170,7 @@ pub async fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<ChatId
 }
 
 /// Create a normal chat with a single user.  To create group chats,
-/// see dc_create_group_chat().
+/// see [Chat::create_group_chat].
 ///
 /// If a chat already exists, this ID is returned, otherwise a new chat is created;
 /// this new chat may already contain messages, eg. from the deaddrop, to get the
@@ -1408,7 +1422,9 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
                 message::guess_msgtype_from_suffix(&blob.to_abs_path())
             {
                 msg.viewtype = better_type;
-                msg.param.set(Param::MimeType, better_mime);
+                if !msg.param.exists(Param::MimeType) {
+                    msg.param.set(Param::MimeType, better_mime);
+                }
             }
         } else if !msg.param.exists(Param::MimeType) {
             if let Some((_, mime)) = message::guess_msgtype_from_suffix(&blob.to_abs_path()) {
@@ -1952,7 +1968,8 @@ pub async fn create_group_chat(
     verified: VerifiedStatus,
     chat_name: impl AsRef<str>,
 ) -> Result<ChatId, Error> {
-    ensure!(!chat_name.as_ref().is_empty(), "Invalid chat name");
+    let chat_name = improve_single_line_input(chat_name);
+    ensure!(!chat_name.is_empty(), "Invalid chat name");
 
     let draft_txt = context
         .stock_string_repl_str(StockMessage::NewGroupDraft, &chat_name)
@@ -1967,7 +1984,7 @@ pub async fn create_group_chat(
             } else {
                 Chattype::Group
             },
-            chat_name.as_ref().to_string(),
+            chat_name,
             grpid,
             time(),
         ],
@@ -2432,17 +2449,18 @@ pub async fn set_chat_name(
     chat_id: ChatId,
     new_name: impl AsRef<str>,
 ) -> Result<(), Error> {
+    let new_name = improve_single_line_input(new_name);
     /* the function only sets the names of group chats; normal chats get their names from the contacts */
     let mut success = false;
 
-    ensure!(!new_name.as_ref().is_empty(), "Invalid name");
+    ensure!(!new_name.is_empty(), "Invalid name");
     ensure!(!chat_id.is_special(), "Invalid chat ID");
 
     let chat = Chat::load_from_db(context, chat_id).await?;
     let mut msg = Message::default();
 
     if real_group_exists(context, chat_id).await {
-        if chat.name == new_name.as_ref() {
+        if chat.name == new_name {
             success = true;
         } else if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await {
             emit_event!(
@@ -2455,7 +2473,7 @@ pub async fn set_chat_name(
                 .sql
                 .execute(
                     "UPDATE chats SET name=? WHERE id=?;",
-                    paramsv![new_name.as_ref().to_string(), chat_id],
+                    paramsv![new_name.to_string(), chat_id],
                 )
                 .await
                 .is_ok()
@@ -2467,7 +2485,7 @@ pub async fn set_chat_name(
                             .stock_system_msg(
                                 StockMessage::MsgGrpName,
                                 &chat.name,
-                                new_name.as_ref(),
+                                &new_name,
                                 DC_CONTACT_ID_SELF,
                             )
                             .await,
@@ -2683,6 +2701,7 @@ pub(crate) async fn get_chat_cnt(context: &Context) -> usize {
     }
 }
 
+/// Returns a tuple of `(chatid, is_verified, blocked)`.
 pub(crate) async fn get_chat_id_by_grpid(
     context: &Context,
     grpid: impl AsRef<str>,

src/config.rs

@@ -9,6 +9,7 @@ use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::events::EventType;
+use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
@@ -24,11 +25,13 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    MailSecurity,
     ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SendSecurity,
     SmtpCertificateChecks,
     ServerFlags,
 
@@ -232,6 +235,15 @@ impl Context {
                 });
                 ret
             }
+            Config::Displayname => {
+                let value = value.map(improve_single_line_input);
+                self.sql.set_raw_config(self, key, value.as_deref()).await
+            }
+            Config::DeleteServerAfter => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                job::schedule_resync(self).await;
+                ret
+            }
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }

src/configure/auto_mozilla.rs

@@ -1,32 +1,31 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
+use quick_xml::events::{BytesStart, Event};
+
+use std::io::BufRead;
+use std::str::FromStr;
 
-use crate::constants::*;
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
 #[derive(Debug)]
-struct MozAutoconfigure<'a> {
-    pub in_emailaddr: &'a str,
-    pub in_emaildomain: &'a str,
-    pub in_emaillocalpart: &'a str,
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub tag_server: MozServer,
-    pub tag_config: MozConfigTag,
+struct Server {
+    pub typ: String,
+    pub hostname: String,
+    pub port: u16,
+    pub sockettype: Socket,
+    pub username: String,
 }
 
-#[derive(Debug, PartialEq)]
-enum MozServer {
-    Undefined,
-    Imap,
-    Smtp,
+#[derive(Debug)]
+struct MozAutoconfigure {
+    pub incoming_servers: Vec<Server>,
+    pub outgoing_servers: Vec<Server>,
 }
 
 #[derive(Debug)]
@@ -38,10 +37,147 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
-    let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+impl Default for MozConfigTag {
+    fn default() -> Self {
+        Self::Undefined
+    }
+}
 
+impl FromStr for MozConfigTag {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.trim().to_lowercase().as_ref() {
+            "hostname" => Ok(MozConfigTag::Hostname),
+            "port" => Ok(MozConfigTag::Port),
+            "sockettype" => Ok(MozConfigTag::Sockettype),
+            "username" => Ok(MozConfigTag::Username),
+            _ => Err(()),
+        }
+    }
+}
+
+/// Parses a single IncomingServer or OutgoingServer section.
+fn parse_server<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+    server_event: &BytesStart,
+) -> Result<Option<Server>, quick_xml::Error> {
+    let end_tag = String::from_utf8_lossy(server_event.name())
+        .trim()
+        .to_lowercase();
+
+    let typ = server_event
+        .attributes()
+        .find(|attr| {
+            attr.as_ref()
+                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
+                .unwrap_or_default()
+        })
+        .map(|typ| {
+            typ.unwrap()
+                .unescape_and_decode_value(reader)
+                .unwrap_or_default()
+                .to_lowercase()
+        })
+        .unwrap_or_default();
+
+    let mut hostname = None;
+    let mut port = None;
+    let mut sockettype = Socket::Automatic;
+    let mut username = None;
+
+    let mut tag_config = MozConfigTag::Undefined;
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                tag_config = String::from_utf8_lossy(event.name())
+                    .parse()
+                    .unwrap_or_default();
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == end_tag {
+                    break;
+                }
+            }
+            Event::Text(ref event) => {
+                let val = event
+                    .unescape_and_decode(reader)
+                    .unwrap_or_default()
+                    .trim()
+                    .to_owned();
+
+                match tag_config {
+                    MozConfigTag::Hostname => hostname = Some(val),
+                    MozConfigTag::Port => port = Some(val.parse().unwrap_or_default()),
+                    MozConfigTag::Username => username = Some(val),
+                    MozConfigTag::Sockettype => {
+                        sockettype = match val.to_lowercase().as_ref() {
+                            "ssl" => Socket::SSL,
+                            "starttls" => Socket::STARTTLS,
+                            "plain" => Socket::Plain,
+                            _ => Socket::Automatic,
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+    }
+
+    if let (Some(hostname), Some(port), Some(username)) = (hostname, port, username) {
+        Ok(Some(Server {
+            typ,
+            hostname,
+            port,
+            sockettype,
+            username,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<MozAutoconfigure, quick_xml::Error> {
+    let mut incoming_servers = Vec::new();
+    let mut outgoing_servers = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == "incomingserver" {
+                    if let Some(incoming_server) = parse_server(reader, event)? {
+                        incoming_servers.push(incoming_server);
+                    }
+                } else if tag == "outgoingserver" {
+                    if let Some(outgoing_server) = parse_server(reader, event)? {
+                        outgoing_servers.push(outgoing_server);
+                    }
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+        buf.clear();
+    }
+
+    Ok(MozAutoconfigure {
+        incoming_servers,
+        outgoing_servers,
+    })
+}
+
+/// Parses XML and fills in address and domain placeholders.
+fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoconfigure, Error> {
     // Split address into local part and domain part.
     let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
     let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
@@ -49,59 +185,78 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
         _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
     };
 
-    let mut moz_ac = MozAutoconfigure {
-        in_emailaddr,
-        in_emaildomain,
-        in_emaillocalpart,
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        tag_server: MozServer::Undefined,
-        tag_config: MozConfigTag::Undefined,
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
+    reader.trim_text(true);
+
+    let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })?;
+
+    let fill_placeholders = |val: &str| -> String {
+        val.replace("%EMAILADDRESS%", in_emailaddr)
+            .replace("%EMAILLOCALPART%", in_emaillocalpart)
+            .replace("%EMAILDOMAIN%", in_emaildomain)
     };
 
-    let mut buf = Vec::new();
-    loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                moz_autoconfigure_starttag_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::End(ref e) => moz_autoconfigure_endtag_cb(e, &mut moz_ac),
-            quick_xml::events::Event::Text(ref e) => {
-                moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::Eof => break,
-            _ => (),
+    let fill_server_placeholders = |server: Server| -> Server {
+        Server {
+            typ: server.typ,
+            hostname: fill_placeholders(&server.hostname),
+            port: server.port,
+            sockettype: server.sockettype,
+            username: fill_placeholders(&server.username),
         }
-        buf.clear();
-    }
+    };
 
-    if moz_ac.out.mail_server.is_empty()
-        || moz_ac.out.mail_port == 0
-        || moz_ac.out.send_server.is_empty()
-        || moz_ac.out.send_port == 0
-    {
-        Err(Error::IncompleteAutoconfig(moz_ac.out))
-    } else {
-        Ok(moz_ac.out)
-    }
+    Ok(MozAutoconfigure {
+        incoming_servers: moz_ac
+            .incoming_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+        outgoing_servers: moz_ac
+            .outgoing_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+    })
 }
 
-pub async fn moz_autoconfigure(
+/// Parses XML into `ServerParams` vector.
+fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerParams>, Error> {
+    let moz_ac = parse_xml_with_address(in_emailaddr, xml_raw)?;
+
+    let res = moz_ac
+        .incoming_servers
+        .into_iter()
+        .chain(moz_ac.outgoing_servers.into_iter())
+        .filter_map(|server| {
+            let protocol = match server.typ.as_ref() {
+                "imap" => Some(Protocol::IMAP),
+                "smtp" => Some(Protocol::SMTP),
+                _ => None,
+            };
+            Some(ServerParams {
+                protocol: protocol?,
+                socket: server.sockettype,
+                hostname: server.hostname,
+                port: server.port,
+                username: server.username,
+            })
+        })
+        .collect();
+    Ok(res)
+}
+
+pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_xml(&param_in.addr, &xml_raw);
+    let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,
@@ -111,212 +266,62 @@ pub async fn moz_autoconfigure(
     res
 }
 
-fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
-    event: &BytesText,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let val = event.unescape_and_decode(reader).unwrap_or_default();
-
-    let addr = moz_ac.in_emailaddr;
-    let email_local = moz_ac.in_emaillocalpart;
-    let email_domain = moz_ac.in_emaildomain;
-
-    let val = val
-        .trim()
-        .replace("%EMAILADDRESS%", addr)
-        .replace("%EMAILLOCALPART%", email_local)
-        .replace("%EMAILDOMAIN%", email_domain);
-
-    match moz_ac.tag_server {
-        MozServer::Imap => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
-            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.mail_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Smtp => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.send_server = val,
-            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.send_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Undefined => {}
-    }
-}
-
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        if moz_ac.tag_server == MozServer::Imap {
-            moz_ac.out_imap_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        if moz_ac.tag_server == MozServer::Smtp {
-            moz_ac.out_smtp_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else {
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    }
-}
-
-fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
-    event: &BytesStart,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        moz_ac.tag_server = if let Some(typ) = event.attributes().find(|attr| {
-            attr.as_ref()
-                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
-                .unwrap_or_default()
-        }) {
-            let typ = typ
-                .unwrap()
-                .unescape_and_decode_value(reader)
-                .unwrap_or_default()
-                .to_lowercase();
-
-            if typ == "imap" && !moz_ac.out_imap_set {
-                MozServer::Imap
-            } else {
-                MozServer::Undefined
-            }
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if !moz_ac.out_smtp_set {
-            MozServer::Smtp
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "hostname" {
-        moz_ac.tag_config = MozConfigTag::Hostname;
-    } else if tag == "port" {
-        moz_ac.tag_config = MozConfigTag::Port;
-    } else if tag == "sockettype" {
-        moz_ac.tag_config = MozConfigTag::Sockettype;
-    } else if tag == "username" {
-        moz_ac.tag_config = MozConfigTag::Username;
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
     fn test_parse_outlook_autoconfig() {
-        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
-        let xml_raw =
-"<clientConfig version=\"1.1\">
-  <emailProvider id=\"outlook.com\">
-    <domain>hotmail.com</domain>
-    <domain>hotmail.co.uk</domain>
-    <domain>hotmail.co.jp</domain>
-    <domain>hotmail.com.br</domain>
-    <domain>hotmail.de</domain>
-    <domain>hotmail.fr</domain>
-    <domain>hotmail.it</domain>
-    <domain>hotmail.es</domain>
-    <domain>live.com</domain>
-    <domain>live.co.uk</domain>
-    <domain>live.co.jp</domain>
-    <domain>live.de</domain>
-    <domain>live.fr</domain>
-    <domain>live.it</domain>
-    <domain>live.jp</domain>
-    <domain>msn.com</domain>
-    <domain>outlook.com</domain>
-    <displayName>Outlook.com (Microsoft)</displayName>
-    <displayShortName>Outlook</displayShortName>
-    <incomingServer type=\"exchange\">
-      <hostname>outlook.office365.com</hostname>
-      <port>443</port>
-      <username>%EMAILADDRESS%</username>
-      <socketType>SSL</socketType>
-      <authentication>OAuth2</authentication>
-      <owaURL>https://outlook.office365.com/owa/</owaURL>
-      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
-      <useGlobalPreferredServer>true</useGlobalPreferredServer>
-    </incomingServer>
-    <incomingServer type=\"imap\">
-      <hostname>outlook.office365.com</hostname>
-      <port>993</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </incomingServer>
-    <incomingServer type=\"pop3\">
-      <hostname>outlook.office365.com</hostname>
-      <port>995</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-      <pop3>
-        <leaveMessagesOnServer>true</leaveMessagesOnServer>
-        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
-      </pop3>
-    </incomingServer>
-    <outgoingServer type=\"smtp\">
-      <hostname>smtp.office365.com</hostname>
-      <port>587</port>
-      <socketType>STARTTLS</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
-    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
-      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
-    </documentation>
-  </emailProvider>
-  <webMail>
-    <loginPage url=\"https://www.outlook.com/\"/>
-    <loginPageInfo url=\"https://www.outlook.com/\">
-      <username>%EMAILADDRESS%</username>
-      <usernameField id=\"i0116\" name=\"login\"/>
-      <passwordField id=\"i0118\" name=\"passwd\"/>
-      <loginButton id=\"idSIButton9\" name=\"SI\"/>
-    </loginPageInfo>
-  </webMail>
-</clientConfig>";
-        let res = parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
-        assert_eq!(res.mail_server, "outlook.office365.com");
-        assert_eq!(res.mail_port, 993);
-        assert_eq!(res.send_server, "smtp.office365.com");
-        assert_eq!(res.send_port, 587);
+        let xml_raw = include_str!("../../test-data/autoconfig/outlook.com.xml");
+        let res = parse_serverparams("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res[0].protocol, Protocol::IMAP);
+        assert_eq!(res[0].hostname, "outlook.office365.com");
+        assert_eq!(res[0].port, 993);
+        assert_eq!(res[1].protocol, Protocol::SMTP);
+        assert_eq!(res[1].hostname, "smtp.office365.com");
+        assert_eq!(res[1].port, 587);
+    }
+
+    #[test]
+    fn test_parse_lakenet_autoconfig() {
+        let xml_raw = include_str!("../../test-data/autoconfig/lakenet.ch.xml");
+        let res =
+            parse_xml_with_address("example@lakenet.ch", xml_raw).expect("XML parsing failed");
+
+        assert_eq!(res.incoming_servers.len(), 4);
+
+        assert_eq!(res.incoming_servers[0].typ, "imap");
+        assert_eq!(res.incoming_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[0].port, 993);
+        assert_eq!(res.incoming_servers[0].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[0].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[1].typ, "imap");
+        assert_eq!(res.incoming_servers[1].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[1].port, 143);
+        assert_eq!(res.incoming_servers[1].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[1].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[2].typ, "pop3");
+        assert_eq!(res.incoming_servers[2].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[2].port, 995);
+        assert_eq!(res.incoming_servers[2].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[2].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[3].typ, "pop3");
+        assert_eq!(res.incoming_servers[3].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[3].port, 110);
+        assert_eq!(res.incoming_servers[3].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[3].username, "example@lakenet.ch");
+
+        assert_eq!(res.outgoing_servers.len(), 1);
+
+        assert_eq!(res.outgoing_servers[0].typ, "smtp");
+        assert_eq!(res.outgoing_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.outgoing_servers[0].port, 587);
+        assert_eq!(res.outgoing_servers[0].sockettype, Socket::STARTTLS);
+        assert_eq!(res.outgoing_servers[0].username, "example@lakenet.ch");
     }
 }

src/configure/auto_outlook.rs

@@ -1,122 +1,194 @@
-//! Outlook's Autodiscover
+//! # Outlook's Autodiscover
+//!
+//! This module implements autoconfiguration via POX (Plain Old XML) interface to Autodiscover
+//! Service. Newer SOAP interface, introduced in Exchange 2010, is not used.
 
-use quick_xml::events::BytesEnd;
+use quick_xml::events::Event;
+
+use std::io::BufRead;
 
-use crate::constants::*;
 use crate::context::Context;
-use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
-struct OutlookAutodiscover {
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub config_type: Option<String>,
-    pub config_server: String,
-    pub config_port: i32,
-    pub config_ssl: String,
-    pub config_redirecturl: Option<String>,
+/// Result of parsing a single `Protocol` tag.
+///
+/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
+#[derive(Debug)]
+struct ProtocolTag {
+    /// Server type, such as "IMAP", "SMTP" or "POP3".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
+    pub typ: String,
+
+    /// Server identifier, hostname or IP address for IMAP and SMTP.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
+    pub server: String,
+
+    /// Network port.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
+    pub port: u16,
+
+    /// Whether connection should be secure, "on" or "off", default is "on".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
+    pub ssl: bool,
 }
 
 enum ParsingResult {
-    LoginParam(LoginParam),
+    Protocols(Vec<ProtocolTag>),
+
+    /// XML redirect via `RedirectUrl` tag.
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut outlk_ad = OutlookAutodiscover {
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        config_type: None,
-        config_server: String::new(),
-        config_port: 0,
-        config_ssl: String::new(),
-        config_redirecturl: None,
-    };
-
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
-    reader.trim_text(true);
+/// Parses a single Protocol section.
+fn parse_protocol<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<Option<ProtocolTag>, quick_xml::Error> {
+    let mut protocol_type = None;
+    let mut protocol_server = None;
+    let mut protocol_port = None;
+    let mut protocol_ssl = true;
 
     let mut buf = Vec::new();
 
     let mut current_tag: Option<String> = None;
-
     loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
-
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                current_tag = Some(String::from_utf8_lossy(event.name()).trim().to_lowercase());
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
                 if tag == "protocol" {
-                    outlk_ad.config_type = None;
-                    outlk_ad.config_server = String::new();
-                    outlk_ad.config_port = 0;
-                    outlk_ad.config_ssl = String::new();
-                    outlk_ad.config_redirecturl = None;
-
+                    break;
+                }
+                if Some(tag) == current_tag {
                     current_tag = None;
-                } else {
-                    current_tag = Some(tag);
                 }
             }
-            quick_xml::events::Event::End(ref e) => {
-                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
-                current_tag = None;
-            }
-            quick_xml::events::Event::Text(ref e) => {
+            Event::Text(ref e) => {
                 let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
-                        "type" => {
-                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        "type" => protocol_type = Some(val.trim().to_string()),
+                        "server" => protocol_server = Some(val.trim().to_string()),
+                        "port" => protocol_port = Some(val.trim().parse().unwrap_or_default()),
+                        "ssl" => {
+                            protocol_ssl = match val.trim() {
+                                "on" => true,
+                                "off" => false,
+                                _ => true,
+                            }
                         }
-                        "server" => outlk_ad.config_server = val.trim().to_string(),
-                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
-                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
-                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
                         _ => {}
                     };
                 }
             }
-            quick_xml::events::Event::Eof => break,
+            Event::Eof => break,
+            _ => {}
+        }
+    }
+
+    if let (Some(protocol_type), Some(protocol_server), Some(protocol_port)) =
+        (protocol_type, protocol_server, protocol_port)
+    {
+        Ok(Some(ProtocolTag {
+            typ: protocol_type,
+            server: protocol_server,
+            port: protocol_port,
+            ssl: protocol_ssl,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Parses `RedirectUrl` tag.
+fn parse_redirecturl<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<String, quick_xml::Error> {
+    let mut buf = Vec::new();
+    match reader.read_event(&mut buf)? {
+        Event::Text(ref e) => {
+            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            Ok(val.trim().to_string())
+        }
+        _ => Ok("".to_string()),
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<ParsingResult, quick_xml::Error> {
+    let mut protocols = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref e) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
+
+                if tag == "protocol" {
+                    if let Some(protocol) = parse_protocol(reader)? {
+                        protocols.push(protocol);
+                    }
+                } else if tag == "redirecturl" {
+                    let redirecturl = parse_redirecturl(reader)?;
+                    return Ok(ParsingResult::RedirectUrl(redirecturl));
+                }
+            }
+            Event::Eof => break,
             _ => (),
         }
         buf.clear();
     }
 
-    // XML redirect via redirecturl
-    let res = if outlk_ad.config_redirecturl.is_none()
-        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
-    {
-        if outlk_ad.out.mail_server.is_empty()
-            || outlk_ad.out.mail_port == 0
-            || outlk_ad.out.send_server.is_empty()
-            || outlk_ad.out.send_port == 0
-        {
-            return Err(Error::IncompleteAutoconfig(outlk_ad.out));
-        }
-        ParsingResult::LoginParam(outlk_ad.out)
-    } else {
-        ParsingResult::RedirectUrl(outlk_ad.config_redirecturl.unwrap())
-    };
-    Ok(res)
+    Ok(ParsingResult::Protocols(protocols))
 }
 
-pub async fn outlk_autodiscover(
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })
+}
+
+fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
+    protocols
+        .into_iter()
+        .filter_map(|protocol| {
+            Some(ServerParams {
+                protocol: match protocol.typ.to_lowercase().as_ref() {
+                    "imap" => Some(Protocol::IMAP),
+                    "smtp" => Some(Protocol::SMTP),
+                    _ => None,
+                }?,
+                socket: match protocol.ssl {
+                    true => Socket::Automatic,
+                    false => Socket::Plain,
+                },
+                hostname: protocol.server,
+                port: protocol.port,
+                username: String::new(),
+            })
+        })
+        .collect()
+}
+
+pub(crate) async fn outlk_autodiscover(
     context: &Context,
     url: &str,
-    _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
@@ -127,47 +199,18 @@ pub async fn outlk_autodiscover(
         }
         match res? {
             ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
-            ParsingResult::LoginParam(login_param) => return Ok(login_param),
+            ParsingResult::Protocols(protocols) => {
+                return Ok(protocols_to_serverparams(protocols));
+            }
         }
     }
     Err(Error::RedirectionError)
 }
 
-fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodiscover) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "protocol" {
-        if let Some(type_) = &outlk_ad.config_type {
-            let port = outlk_ad.config_port;
-            let ssl_on = outlk_ad.config_ssl == "on";
-            let ssl_off = outlk_ad.config_ssl == "off";
-            if type_ == "imap" && !outlk_ad.out_imap_set {
-                outlk_ad.out.mail_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.mail_port = port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_imap_set = true
-            } else if type_ == "smtp" && !outlk_ad.out_smtp_set {
-                outlk_ad.out.send_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.send_port = outlk_ad.config_port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_smtp_set = true
-            }
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
@@ -184,16 +227,13 @@ mod tests {
     </Response>
   </Autodiscover>
  ").expect("XML is not parsed successfully");
-        match res {
-            ParsingResult::LoginParam(_lp) => {
-                panic!("redirecturl is not found");
-            }
-            ParsingResult::RedirectUrl(url) => {
-                assert_eq!(
-                    url,
-                    "https://mail.example.com/autodiscover/autodiscover.xml"
-                );
-            }
+        if let ParsingResult::RedirectUrl(url) = res {
+            assert_eq!(
+                url,
+                "https://mail.example.com/autodiscover/autodiscover.xml"
+            );
+        } else {
+            panic!("redirecturl is not found");
         }
     }
 
@@ -228,11 +268,16 @@ mod tests {
         .expect("XML is not parsed successfully");
 
         match res {
-            ParsingResult::LoginParam(lp) => {
-                assert_eq!(lp.mail_server, "example.com");
-                assert_eq!(lp.mail_port, 993);
-                assert_eq!(lp.send_server, "smtp.example.com");
-                assert_eq!(lp.send_port, 25);
+            ParsingResult::Protocols(protocols) => {
+                assert_eq!(protocols[0].typ, "IMAP");
+                assert_eq!(protocols[0].server, "example.com");
+                assert_eq!(protocols[0].port, 993);
+                assert_eq!(protocols[0].ssl, true);
+
+                assert_eq!(protocols[1].typ, "SMTP");
+                assert_eq!(protocols[1].server, "smtp.example.com");
+                assert_eq!(protocols[1].port, 25);
+                assert_eq!(protocols[1].ssl, false);
             }
             ParsingResult::RedirectUrl(_) => {
                 panic!("RedirectUrl is not expected");

src/configure/mod.rs

@@ -3,9 +3,11 @@
 mod auto_mozilla;
 mod auto_outlook;
 mod read_url;
+mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::prelude::*;
+use async_std::task;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::config::Config;
@@ -13,22 +15,31 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::imap::Imap;
-use crate::login_param::{CertificateChecks, LoginParam};
+use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
 use crate::oauth2::*;
+use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
+use crate::stock::StockMessage;
 use crate::{chat, e2ee, provider};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
+use server_params::{expand_param_vector, ServerParams};
 
 macro_rules! progress {
-    ($context:tt, $progress:expr) => {
+    ($context:tt, $progress:expr, $comment:expr) => {
         assert!(
             $progress <= 1000,
             "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
         );
-        $context.emit_event($crate::events::EventType::ConfigureProgress($progress));
+        $context.emit_event($crate::events::EventType::ConfigureProgress {
+            progress: $progress,
+            comment: $comment,
+        });
+    };
+    ($context:tt, $progress:expr) => {
+        progress!($context, $progress, None);
     };
 }
 
@@ -107,7 +118,17 @@ impl Context {
                 Ok(())
             }
             Err(err) => {
-                progress!(self, 0);
+                progress!(
+                    self,
+                    0,
+                    Some(
+                        self.stock_string_repl_str(
+                            StockMessage::ConfigurationFailed,
+                            err.to_string(),
+                        )
+                        .await
+                    )
+                );
                 Err(err)
             }
         }
@@ -115,20 +136,37 @@ impl Context {
 }
 
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
-    let mut param_autoconfig: Option<LoginParam> = None;
-    let mut keep_flags = 0;
-
-    // Read login parameters from the database
     progress!(ctx, 1);
+
+    // Check basic settings.
     ensure!(!param.addr.is_empty(), "Please enter an email address.");
 
+    // Only check for IMAP password, SMTP password is an "advanced" setting.
+    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
+    if param.smtp.password.is_empty() {
+        param.smtp.password = param.imap.password.clone()
+    }
+
+    // Normalize authentication flags.
+    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
+        DC_LP_AUTH_OAUTH2 => true,
+        DC_LP_AUTH_NORMAL => false,
+        _ => false,
+    };
+    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
+    param.server_flags |= if oauth2 {
+        DC_LP_AUTH_OAUTH2 as i32
+    } else {
+        DC_LP_AUTH_NORMAL as i32
+    };
+
     // Step 1: Load the parameters and check email-address and password
 
-    if 0 != param.server_flags & DC_LP_AUTH_OAUTH2 {
+    if oauth2 {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
-        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.mail_pw)
+        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
             .await
             .and_then(|e| e.parse().ok())
         {
@@ -149,95 +187,128 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // Step 2: Autoconfig
     progress!(ctx, 200);
 
-    // param.mail_user.is_empty() -- the user can enter a loginname which is used by autoconfig then
-    // param.send_pw.is_empty()   -- the password cannot be auto-configured and is no criterion for
-    //                               autoconfig or not
-    if param.mail_server.is_empty()
-        && param.mail_port == 0
-        && param.send_server.is_empty()
-        && param.send_port == 0
-        && param.send_user.is_empty()
-        && (param.server_flags & !DC_LP_AUTH_OAUTH2) == 0
+    let param_autoconfig;
+    if param.imap.server.is_empty()
+        && param.imap.port == 0
+        && param.imap.security == Socket::Automatic
+        && param.imap.user.is_empty()
+        && param.smtp.server.is_empty()
+        && param.smtp.port == 0
+        && param.smtp.security == Socket::Automatic
+        && param.smtp.user.is_empty()
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
-        keep_flags = param.server_flags & DC_LP_AUTH_OAUTH2;
-        if let Some(new_param) = get_offline_autoconfig(ctx, &param) {
-            // got parameters from our provider-database, skip Autoconfig, preserve the OAuth2 setting
-            param_autoconfig = Some(new_param);
-        }
 
-        if param_autoconfig.is_none() {
+        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
+            param_autoconfig = Some(servers);
+        } else {
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
+    } else {
+        param_autoconfig = None;
     }
 
-    // C.  Do we have any autoconfig result?
     progress!(ctx, 500);
-    if let Some(ref cfg) = param_autoconfig {
-        info!(ctx, "Got autoconfig: {}", &cfg);
-        if !cfg.mail_user.is_empty() {
-            param.mail_user = cfg.mail_user.clone();
-        }
-        // all other values are always NULL when entering autoconfig
-        param.mail_server = cfg.mail_server.clone();
-        param.mail_port = cfg.mail_port;
-        param.send_server = cfg.send_server.clone();
-        param.send_port = cfg.send_port;
-        param.send_user = cfg.send_user.clone();
-        param.server_flags = cfg.server_flags;
-        // although param_autoconfig's data are no longer needed from,
-        // it is used to later to prevent trying variations of port/server/logins
-    }
-    param.server_flags |= keep_flags;
 
-    // Step 3: Fill missing fields with defaults
-    if param.send_user.is_empty() {
-        param.send_user = param.mail_user.clone();
-    }
-    if param.send_pw.is_empty() {
-        param.send_pw = param.mail_pw.clone()
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_IMAP_SOCKET_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & (DC_LP_SMTP_SOCKET_FLAGS as i32)) {
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-    }
-    if !dc_exactly_one_bit_set(param.server_flags & DC_LP_AUTH_FLAGS as i32) {
-        param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-        param.server_flags |= DC_LP_AUTH_NORMAL as i32
-    }
-
-    // do we have a complete configuration?
-    ensure!(
-        !param.mail_pw.is_empty() && !param.send_pw.is_empty(),
-        "Account settings incomplete."
+    let servers = expand_param_vector(
+        param_autoconfig.unwrap_or_else(|| {
+            vec![
+                ServerParams {
+                    protocol: Protocol::IMAP,
+                    hostname: param.imap.server.clone(),
+                    port: param.imap.port,
+                    socket: param.imap.security,
+                    username: param.imap.user.clone(),
+                },
+                ServerParams {
+                    protocol: Protocol::SMTP,
+                    hostname: param.smtp.server.clone(),
+                    port: param.smtp.port,
+                    socket: param.smtp.security,
+                    username: param.smtp.user.clone(),
+                },
+            ]
+        }),
+        &param.addr,
+        &param_domain,
     );
 
+    progress!(ctx, 550);
+
+    // Spawn SMTP configuration task
+    let mut smtp = Smtp::new();
+
+    let context_smtp = ctx.clone();
+    let mut smtp_param = param.smtp.clone();
+    let smtp_addr = param.addr.clone();
+    let smtp_servers: Vec<ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::SMTP)
+        .cloned()
+        .collect();
+
+    let smtp_config_task = task::spawn(async move {
+        let mut smtp_configured = false;
+        for smtp_server in smtp_servers {
+            smtp_param.user = smtp_server.username.clone();
+            smtp_param.server = smtp_server.hostname.clone();
+            smtp_param.port = smtp_server.port;
+            smtp_param.security = smtp_server.socket;
+
+            if try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp).await {
+                smtp_configured = true;
+                break;
+            }
+        }
+
+        if smtp_configured {
+            Some(smtp_param)
+        } else {
+            None
+        }
+    });
+
     progress!(ctx, 600);
-    // try to connect to IMAP - if we did not got an autoconfig,
-    // do some further tries with different settings and username variations
+
+    // Configure IMAP
     let (_s, r) = async_std::sync::channel(1);
     let mut imap = Imap::new(r);
 
-    if param_autoconfig.is_some() {
-        if try_imap_one_param(ctx, &param, &mut imap).await.is_err() {
-            bail!("IMAP autoconfig did not succeed");
-        }
-    } else {
-        *param = try_imap_hostnames(ctx, param.clone(), &mut imap).await?;
-    }
-    progress!(ctx, 750);
+    let mut imap_configured = false;
+    let imap_servers: Vec<&ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::IMAP)
+        .collect();
+    let imap_servers_count = imap_servers.len();
+    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
+        param.imap.user = imap_server.username.clone();
+        param.imap.server = imap_server.hostname.clone();
+        param.imap.port = imap_server.port;
+        param.imap.security = imap_server.socket;
 
-    let mut smtp = Smtp::new();
-    if param_autoconfig.is_some() {
-        if try_smtp_one_param(ctx, &param, &mut smtp).await.is_err() {
-            bail!("SMTP autoconfig did not succeed");
+        if try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+            imap_configured = true;
+            break;
         }
-    } else {
-        *param = try_smtp_hostnames(ctx, param.clone(), &mut smtp).await?;
+        progress!(
+            ctx,
+            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
+        );
     }
+    if !imap_configured {
+        bail!("IMAP autoconfig did not succeed");
+    }
+
+    progress!(ctx, 850);
+
+    // Wait for SMTP configuration
+    if let Some(smtp_param) = smtp_config_task.await {
+        param.smtp = smtp_param;
+    } else {
+        bail!("SMTP autoconfig did not succeed");
+    }
+
     progress!(ctx, 900);
 
     let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
@@ -306,8 +377,8 @@ impl AutoconfigSource {
             AutoconfigSource {
                 provider: AutoconfigProvider::Outlook,
                 url: format!(
-                    "https://{}{}/autodiscover/autodiscover.xml",
-                    "autodiscover.", domain
+                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
+                    domain
                 ),
             },
             // always SSL for Thunderbird's database
@@ -318,10 +389,10 @@ impl AutoconfigSource {
         ]
     }
 
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<LoginParam> {
+    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
         let params = match self.provider {
             AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url, &param).await?,
+            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
         };
 
         Ok(params)
@@ -337,7 +408,7 @@ async fn get_autoconfig(
     param: &LoginParam,
     param_domain: &str,
     param_addr_urlencoded: &str,
-) -> Option<LoginParam> {
+) -> Option<Vec<ServerParams>> {
     let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
 
     let mut progress = 300;
@@ -353,301 +424,97 @@ async fn get_autoconfig(
     None
 }
 
-fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
+fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
     info!(
         context,
         "checking internal provider-info for offline autoconfig"
     );
 
-    if let Some(provider) = provider::get_provider_info(&param.addr) {
+    if let Some(provider) = provider::get_provider_info(&addr) {
         match provider.status {
             provider::Status::OK | provider::Status::PREPARATION => {
-                let imap = provider.get_imap_server();
-                let smtp = provider.get_smtp_server();
-                // clippy complains about these is_some()/unwrap() settings,
-                // however, rewriting the code to "if let" would make things less obvious,
-                // esp. if we allow more combinations of servers (pop, jmap).
-                // therefore, #[allow(clippy::unnecessary_unwrap)] is added above.
-                if let Some(imap) = imap {
-                    if let Some(smtp) = smtp {
-                        let mut p = LoginParam::new();
-                        p.addr = param.addr.clone();
-
-                        p.mail_server = imap.hostname.to_string();
-                        p.mail_user = imap.apply_username_pattern(param.addr.clone());
-                        p.mail_port = imap.port as i32;
-                        p.imap_certificate_checks = CertificateChecks::Automatic;
-                        p.server_flags |= match imap.socket {
-                            provider::Socket::STARTTLS => DC_LP_IMAP_SOCKET_STARTTLS,
-                            provider::Socket::SSL => DC_LP_IMAP_SOCKET_SSL,
-                        };
-
-                        p.send_server = smtp.hostname.to_string();
-                        p.send_user = smtp.apply_username_pattern(param.addr.clone());
-                        p.send_port = smtp.port as i32;
-                        p.smtp_certificate_checks = CertificateChecks::Automatic;
-                        p.server_flags |= match smtp.socket {
-                            provider::Socket::STARTTLS => DC_LP_SMTP_SOCKET_STARTTLS as i32,
-                            provider::Socket::SSL => DC_LP_SMTP_SOCKET_SSL as i32,
-                        };
-
-                        info!(context, "offline autoconfig found: {}", p);
-                        return Some(p);
-                    }
+                if provider.server.is_empty() {
+                    info!(context, "offline autoconfig found, but no servers defined");
+                    None
+                } else {
+                    info!(context, "offline autoconfig found");
+                    let servers = provider
+                        .server
+                        .iter()
+                        .map(|s| ServerParams {
+                            protocol: s.protocol,
+                            socket: s.socket,
+                            hostname: s.hostname.to_string(),
+                            port: s.port,
+                            username: match s.username_pattern {
+                                UsernamePattern::EMAIL => addr.to_string(),
+                                UsernamePattern::EMAILLOCALPART => {
+                                    if let Some(at) = addr.find('@') {
+                                        addr.split_at(at).0.to_string()
+                                    } else {
+                                        addr.to_string()
+                                    }
+                                }
+                            },
+                        })
+                        .collect();
+                    Some(servers)
                 }
-                info!(context, "offline autoconfig found, but no servers defined");
-                return None;
             }
             provider::Status::BROKEN => {
                 info!(context, "offline autoconfig found, provider is broken");
-                return None;
+                None
             }
         }
-    }
-    info!(context, "no offline autoconfig found");
-    None
-}
-
-async fn try_imap_hostnames(
-    context: &Context,
-    mut param: LoginParam,
-    imap: &mut Imap,
-) -> Result<LoginParam> {
-    if param.mail_server.is_empty() {
-        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
-        let param_domain = parsed.domain;
-
-        param.mail_server = param_domain.clone();
-        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
-            return Ok(param);
-        }
-
-        progress!(context, 650);
-        param.mail_server = "imap.".to_string() + &param_domain;
-        if let Ok(param) = try_imap_ports(context, param.clone(), imap).await {
-            return Ok(param);
-        }
-
-        progress!(context, 700);
-        param.mail_server = "mail.".to_string() + &param_domain;
-        try_imap_ports(context, param, imap).await
     } else {
-        progress!(context, 700);
-        try_imap_ports(context, param, imap).await
+        info!(context, "no offline autoconfig found");
+        None
     }
 }
 
-// Try various IMAP ports and corresponding TLS settings.
-async fn try_imap_ports(
+async fn try_imap_one_param(
     context: &Context,
-    mut param: LoginParam,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
     imap: &mut Imap,
-) -> Result<LoginParam> {
-    // Try to infer port from socket security.
-    if param.mail_port == 0 {
-        if 0 != param.server_flags & DC_LP_IMAP_SOCKET_SSL {
-            param.mail_port = 993
-        }
-        if 0 != param.server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN) {
-            param.mail_port = 143
-        }
-    }
-
-    if param.mail_port == 0 {
-        // Neither port nor security is set.
-        //
-        // Try common secure combinations.
-
-        // Try TLS over port 993
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
-        param.mail_port = 993;
-        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
-            return Ok(login_param);
-        }
-
-        // Try STARTTLS over port 143
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
-        param.mail_port = 143;
-        try_imap_usernames(context, param, imap).await
-    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
-        // Try TLS over user-provided port.
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32;
-        if let Ok(login_param) = try_imap_usernames(context, param.clone(), imap).await {
-            return Ok(login_param);
-        }
-
-        // Try STARTTLS over user-provided port.
-        param.server_flags &= !(DC_LP_IMAP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32;
-        try_imap_usernames(context, param, imap).await
-    } else {
-        try_imap_usernames(context, param, imap).await
-    }
-}
-
-async fn try_imap_usernames(
-    context: &Context,
-    mut param: LoginParam,
-    imap: &mut Imap,
-) -> Result<LoginParam> {
-    if param.mail_user.is_empty() {
-        param.mail_user = param.addr.clone();
-        if let Err(e) = try_imap_one_param(context, &param, imap).await {
-            if let Some(at) = param.mail_user.find('@') {
-                param.mail_user = param.mail_user.split_at(at).0.to_string();
-                try_imap_one_param(context, &param, imap).await?;
-            } else {
-                return Err(e);
-            }
-        }
-        Ok(param)
-    } else {
-        try_imap_one_param(context, &param, imap).await?;
-        Ok(param)
-    }
-}
-
-async fn try_imap_one_param(context: &Context, param: &LoginParam, imap: &mut Imap) -> Result<()> {
+) -> bool {
     let inf = format!(
-        "imap: {}@{}:{} flags=0x{:x} certificate_checks={}",
-        param.mail_user,
-        param.mail_server,
-        param.mail_port,
-        param.server_flags,
-        param.imap_certificate_checks
+        "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if imap.connect(context, &param).await {
+    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        false
+    } else {
         info!(context, "success: {}", inf);
-        return Ok(());
+        true
     }
-
-    bail!("Could not connect: {}", inf);
 }
 
-async fn try_smtp_hostnames(
+async fn try_smtp_one_param(
     context: &Context,
-    mut param: LoginParam,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
     smtp: &mut Smtp,
-) -> Result<LoginParam> {
-    if param.send_server.is_empty() {
-        let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
-        let param_domain = parsed.domain;
-
-        param.send_server = param_domain.clone();
-        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
-            return Ok(param);
-        }
-
-        progress!(context, 800);
-        param.send_server = "smtp.".to_string() + &param_domain;
-        if let Ok(param) = try_smtp_ports(context, param.clone(), smtp).await {
-            return Ok(param);
-        }
-
-        progress!(context, 850);
-        param.send_server = "mail.".to_string() + &param_domain;
-        try_smtp_ports(context, param, smtp).await
-    } else {
-        progress!(context, 850);
-        try_smtp_ports(context, param, smtp).await
-    }
-}
-
-// Try various SMTP ports and corresponding TLS settings.
-async fn try_smtp_ports(
-    context: &Context,
-    mut param: LoginParam,
-    smtp: &mut Smtp,
-) -> Result<LoginParam> {
-    // Try to infer port from socket security.
-    if param.send_port == 0 {
-        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_STARTTLS as i32 {
-            param.send_port = 587;
-        }
-        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32 {
-            param.send_port = 25;
-        }
-        if 0 != param.server_flags & DC_LP_SMTP_SOCKET_SSL as i32 {
-            param.send_port = 465;
-        }
-    }
-
-    if param.send_port == 0 {
-        // Neither port nor security is set.
-        //
-        // Try common secure combinations.
-
-        // Try TLS over port 465.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
-        param.send_port = 465;
-        if let Ok(login_param) = try_smtp_usernames(context, param.clone(), smtp).await {
-            return Ok(login_param);
-        }
-
-        // Try STARTTLS over port 587.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-        param.send_port = 587;
-        try_smtp_usernames(context, param, smtp).await
-    } else if 0 == param.server_flags & DC_LP_SMTP_SOCKET_FLAGS as i32 {
-        // Try TLS over user-provided port.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32;
-        if let Ok(param) = try_smtp_usernames(context, param.clone(), smtp).await {
-            return Ok(param);
-        }
-
-        // Try STARTTLS over user-provided port.
-        param.server_flags &= !(DC_LP_SMTP_SOCKET_FLAGS as i32);
-        param.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32;
-        try_smtp_usernames(context, param, smtp).await
-    } else {
-        try_smtp_usernames(context, param, smtp).await
-    }
-}
-
-async fn try_smtp_usernames(
-    context: &Context,
-    mut param: LoginParam,
-    smtp: &mut Smtp,
-) -> Result<LoginParam> {
-    if param.send_user.is_empty() {
-        param.send_user = param.addr.clone();
-        if let Err(e) = try_smtp_one_param(context, &param, smtp).await {
-            if let Some(at) = param.send_user.find('@') {
-                param.send_user = param.send_user.split_at(at).0.to_string();
-                try_smtp_one_param(context, &param, smtp).await?;
-            } else {
-                return Err(e);
-            }
-        }
-        Ok(param)
-    } else {
-        try_smtp_one_param(context, &param, smtp).await?;
-        Ok(param)
-    }
-}
-
-async fn try_smtp_one_param(context: &Context, param: &LoginParam, smtp: &mut Smtp) -> Result<()> {
+) -> bool {
     let inf = format!(
-        "smtp: {}@{}:{} flags: 0x{:x}",
-        param.send_user, param.send_server, param.send_port, param.server_flags
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, &param).await {
-        bail!("could not connect: {}", err);
+    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        false
+    } else {
+        info!(context, "success: {}", inf);
+        smtp.disconnect().await;
+        true
     }
-
-    info!(context, "success: {}", inf);
-    smtp.disconnect().await;
-    Ok(())
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -662,9 +529,6 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
-    #[error("Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
     #[error("Failed to get URL")]
     ReadUrlError(#[from] self::read_url::Error),
 
@@ -674,6 +538,7 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
 
     use super::*;
     use crate::config::*;
@@ -697,23 +562,15 @@ mod tests {
     async fn test_get_offline_autoconfig() {
         let context = TestContext::new().await.ctx;
 
-        let mut params = LoginParam::new();
-        params.addr = "someone123@example.org".to_string();
-        assert!(get_offline_autoconfig(&context, &params).is_none());
+        let addr = "someone123@example.org";
+        assert!(get_offline_autoconfig(&context, addr).is_none());
 
-        let mut params = LoginParam::new();
-        params.addr = "someone123@nauta.cu".to_string();
-        let found_params = get_offline_autoconfig(&context, &params).unwrap();
-        assert_eq!(found_params.mail_server, "imap.nauta.cu".to_string());
-        assert_eq!(found_params.send_server, "smtp.nauta.cu".to_string());
-
-        assert_eq!(
-            found_params.imap_certificate_checks,
-            CertificateChecks::Automatic
-        );
-        assert_eq!(
-            found_params.smtp_certificate_checks,
-            CertificateChecks::Automatic
-        );
+        let addr = "someone123@nauta.cu";
+        let found_params = get_offline_autoconfig(&context, addr).unwrap();
+        assert_eq!(found_params.len(), 2);
+        assert_eq!(found_params[0].protocol, Protocol::IMAP);
+        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
+        assert_eq!(found_params[1].protocol, Protocol::SMTP);
+        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
     }
 }

src/configure/server_params.rs

@@ -0,0 +1,164 @@
+//! Variable server parameters lists
+
+use crate::provider::{Protocol, Socket};
+
+/// Set of variable parameters to try during configuration.
+///
+/// Can be loaded from offline provider database, online configuraiton
+/// or derived from user entered parameters.
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct ServerParams {
+    /// Protocol, such as IMAP or SMTP.
+    pub protocol: Protocol,
+
+    /// Server hostname, empty if unknown.
+    pub hostname: String,
+
+    /// Server port, zero if unknown.
+    pub port: u16,
+
+    /// Socket security, such as TLS or STARTTLS, Socket::Automatic if unknown.
+    pub socket: Socket,
+
+    /// Username, empty if unknown.
+    pub username: String,
+}
+
+impl ServerParams {
+    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+
+        if self.username.is_empty() {
+            self.username = addr.to_string();
+            res.push(self.clone());
+
+            if let Some(at) = addr.find('@') {
+                self.username = addr.split_at(at).0.to_string();
+                res.push(self);
+            }
+        } else {
+            res.push(self)
+        }
+        res
+    }
+
+    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+        if self.hostname.is_empty() {
+            self.hostname = param_domain.to_string();
+            res.push(self.clone());
+
+            self.hostname = match self.protocol {
+                Protocol::IMAP => "imap.".to_string() + param_domain,
+                Protocol::SMTP => "smtp.".to_string() + param_domain,
+            };
+            res.push(self.clone());
+
+            self.hostname = "mail.".to_string() + param_domain;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+
+    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+        // Try to infer port from socket security.
+        if self.port == 0 {
+            self.port = match self.socket {
+                Socket::SSL => match self.protocol {
+                    Protocol::IMAP => 993,
+                    Protocol::SMTP => 465,
+                },
+                Socket::STARTTLS | Socket::Plain => match self.protocol {
+                    Protocol::IMAP => 143,
+                    Protocol::SMTP => 587,
+                },
+                Socket::Automatic => 0,
+            }
+        }
+
+        let mut res = Vec::new();
+        if self.port == 0 {
+            // Neither port nor security is set.
+            //
+            // Try common secure combinations.
+
+            // Try STARTTLS
+            self.socket = Socket::STARTTLS;
+            self.port = match self.protocol {
+                Protocol::IMAP => 143,
+                Protocol::SMTP => 587,
+            };
+            res.push(self.clone());
+
+            // Try TLS
+            self.socket = Socket::SSL;
+            self.port = match self.protocol {
+                Protocol::IMAP => 993,
+                Protocol::SMTP => 465,
+            };
+            res.push(self);
+        } else if self.socket == Socket::Automatic {
+            // Try TLS over user-provided port.
+            self.socket = Socket::SSL;
+            res.push(self.clone());
+
+            // Try STARTTLS over user-provided port.
+            self.socket = Socket::STARTTLS;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+}
+
+/// Expands vector of `ServerParams`, replacing placeholders with
+/// variants to try.
+pub(crate) fn expand_param_vector(
+    v: Vec<ServerParams>,
+    addr: &str,
+    domain: &str,
+) -> Vec<ServerParams> {
+    v.into_iter()
+        // The order of expansion is important: ports are expanded the
+        // last, so they are changed the first. Username is only
+        // changed if default value (address with domain) didn't work
+        // for all available hosts and ports.
+        .flat_map(|params| params.expand_usernames(addr).into_iter())
+        .flat_map(|params| params.expand_hostnames(domain).into_iter())
+        .flat_map(|params| params.expand_ports().into_iter())
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_expand_param_vector() {
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 0,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 993,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+        );
+    }
+}

src/constants.rs

@@ -1,6 +1,4 @@
 //! # Constants
-#![allow(dead_code)]
-
 use deltachat_derive::*;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
@@ -9,13 +7,6 @@ lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
 }
 
-// some defaults
-const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
-const DC_INBOX_WATCH_DEFAULT: i32 = 1;
-const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
-
 #[derive(
     Debug,
     Display,
@@ -118,11 +109,11 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
-pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
+pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: u32 = 3;
 /// a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
+pub const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
 /// virtual chat showing all messages flagged with msgs.starred=2
 pub const DC_CHAT_ID_STARRED: u32 = 5;
 /// only an indicator in a chatlist
@@ -166,9 +157,9 @@ pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
-const DC_MAX_GET_TEXT_LEN: usize = 30000;
+pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -199,44 +190,8 @@ pub const DC_LP_AUTH_OAUTH2: i32 = 0x2;
 /// If this flag is set, automatic configuration is skipped.
 pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 
-/// Connect to IMAP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_STARTTLS: i32 = 0x100;
-
-/// Connect to IMAP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_SSL: i32 = 0x200;
-
-/// Connect to IMAP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_PLAIN: i32 = 0x400;
-
-/// Connect to SMTP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_STARTTLS: usize = 0x10000;
-
-/// Connect to SMTP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
-
-/// Connect to SMTP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
-
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
-
-// QR code scanning (view from Bob, the joiner)
-pub const DC_VC_AUTH_REQUIRED: i32 = 2;
-pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
-pub const DC_BOB_ERROR: i32 = 0;
-pub const DC_BOB_SUCCESS: i32 = 1;
 
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
@@ -331,16 +286,6 @@ mod tests {
     }
 }
 
-// These constants are used as events
-// reported to the callback given to dc_context_new().
-// If you do not want to handle an event, it is always safe to return 0,
-// so there is no need to add a "case" for every event.
-
-const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
-const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
-const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
-const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]

src/contact.rs

@@ -20,6 +20,7 @@ use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
 use crate::param::*;
 use crate::peerstate::*;
+use crate::provider::Socket;
 use crate::stock::StockMessage;
 
 /// An object representing a single contact in memory.
@@ -235,6 +236,7 @@ impl Contact {
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
     ) -> Result<u32> {
+        let name = improve_single_line_input(name);
         ensure!(
             !addr.as_ref().is_empty(),
             "Cannot create contact with empty address"
@@ -728,8 +730,8 @@ impl Contact {
                     );
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
                 }
-            } else if 0 == loginparam.server_flags & DC_LP_IMAP_SOCKET_PLAIN as i32
-                && 0 == loginparam.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32
+            } else if loginparam.imap.security != Socket::Plain
+                && loginparam.smtp.security != Socket::Plain
             {
                 ret += &context.stock_str(StockMessage::EncrTransp).await;
             } else {

src/context.rs

@@ -17,9 +17,9 @@ use crate::error::*;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::lot::Lot;
 use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
+use crate::securejoin::Bob;
 use crate::sql::Sql;
 use std::time::SystemTime;
 
@@ -44,7 +44,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<BobStatus>,
+    pub(crate) bob: RwLock<Bob>,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -508,13 +508,6 @@ impl Default for RunningState {
     }
 }
 
-#[derive(Debug, Default)]
-pub(crate) struct BobStatus {
-    pub expects: i32,
-    pub status: i32,
-    pub qr_scan: Option<Lot>,
-}
-
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }
@@ -580,7 +573,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 

src/dc_receive_imf.rs

@@ -111,7 +111,7 @@ pub async fn dc_receive_imf(
     // or if From: is equal to SELF (in this case, it is any outgoing messages,
     // we do not check Return-Path any more as this is unreliable, see
     // https://github.com/deltachat/deltachat-core/issues/150)
-    let (from_id, from_id_blocked, incoming_origin) =
+    let (from_id, _from_id_blocked, incoming_origin) =
         from_field_to_contact_id(context, &mime_parser.from).await?;
 
     let incoming = from_id != DC_CONTACT_ID_SELF;
@@ -162,7 +162,6 @@ pub async fn dc_receive_imf(
             &rfc724_mid,
             &mut sent_timestamp,
             from_id,
-            from_id_blocked,
             &mut hidden,
             &mut chat_id,
             seen,
@@ -329,7 +328,6 @@ async fn add_parts(
     rfc724_mid: &str,
     sent_timestamp: &mut i64,
     from_id: u32,
-    from_id_blocked: bool,
     hidden: &mut bool,
     chat_id: &mut ChatId,
     seen: bool,
@@ -418,8 +416,6 @@ async fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
-
-                    context.bob.write().await.status = 0; // secure-join failed
                     context.stop_ongoing().await;
                     warn!(context, "Error in Secure-Join message handling: {}", err);
                     return Ok(());
@@ -432,6 +428,8 @@ async fn add_parts(
                 .await
                 .unwrap_or_default();
 
+        // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
+        // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() && mime_parser.failure_report.is_some() {
             *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
             info!(
@@ -440,11 +438,8 @@ async fn add_parts(
             );
         }
 
-        // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
-        // it might also be blocked and displayed in the deaddrop as a result
         if chat_id.is_unset() {
             // try to create a group
-            // (groups appear automatically only if the _sender_ is known, see core issue #54)
 
             let create_blocked =
                 if !test_normal_chat_id.is_unset() && test_normal_chat_id_blocked == Blocked::Not {
@@ -701,6 +696,20 @@ async fn add_parts(
     let in_fresh = state == MessageState::InFresh;
     let rcvd_timestamp = time();
     let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, in_fresh).await;
+
+    // Ensure replies to messages are sorted after the parent message.
+    //
+    // This is useful in a case where sender clocks are not
+    // synchronized and parent message has a Date: header with a
+    // timestamp higher than reply timestamp.
+    //
+    // This does not help if parent message arrives later than the
+    // reply.
+    let parent_timestamp = mime_parser.get_parent_timestamp(context).await?;
+    let sort_timestamp = parent_timestamp.map_or(sort_timestamp, |parent_timestamp| {
+        std::cmp::max(sort_timestamp, parent_timestamp)
+    });
+
     *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
@@ -840,9 +849,7 @@ async fn add_parts(
     if chat_id.is_trash() || *hidden {
         *create_event_to_send = None;
     } else if incoming && state == MessageState::InFresh {
-        if from_id_blocked {
-            *create_event_to_send = None;
-        } else if Blocked::Not != chat_id_blocked {
+        if Blocked::Not != chat_id_blocked {
             *create_event_to_send = Some(CreateEvent::MsgsChanged);
         } else {
             *create_event_to_send = Some(CreateEvent::IncomingMsg);
@@ -2202,7 +2209,7 @@ mod tests {
         } else {
             panic!("Wrong item type");
         };
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
@@ -2253,7 +2260,7 @@ mod tests {
             chat::get_chat_msgs(&t.ctx, group_id, 0, None).await.len(),
             1
         );
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.state, MessageState::OutMdnRcvd);
@@ -2341,7 +2348,7 @@ mod tests {
         } else {
             panic!("Wrong item type");
         };
-        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone())
+        let msg = message::Message::load_from_db(&t.ctx, *msg_id)
             .await
             .unwrap();
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
@@ -2623,4 +2630,26 @@ mod tests {
         );
         assert_eq!(last_msg.from_id, DC_CONTACT_ID_INFO);
     }
+
+    #[async_std::test]
+    async fn test_html_only_mail() {
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+        dc_receive_imf(
+            &t.ctx,
+            include_bytes!("../test-data/message/wrong-html.eml"),
+            "INBOX",
+            0,
+            false,
+        )
+        .await
+        .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
+        let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
+        assert_eq!(msg.text.unwrap(), "   Guten Abend,   \n\n   Lots of text   \n\n   text with Umlaut ä...   \n\n   MfG    [...]");
+    }
 }

src/dc_tools.rs

@@ -9,7 +9,9 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
 use async_std::{fs, io};
+
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
@@ -17,10 +19,6 @@ use crate::context::Context;
 use crate::error::{bail, Error};
 use crate::events::EventType;
 
-pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
-    0 != v && 0 == v & (v - 1)
-}
-
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
 #[allow(clippy::indexing_slicing)]
@@ -296,6 +294,23 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     }
 }
 
+pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
+    match async_std::fs::read_dir(path).await {
+        Ok(mut read_dir) => {
+            while let Some(entry) = read_dir.next().await {
+                match entry {
+                    Ok(file) => {
+                        dc_delete_file(context, file.file_name()).await;
+                    }
+                    Err(e) => warn!(context, "Could not read file to delete: {}", e),
+                }
+            }
+        }
+
+        Err(e) => warn!(context, "Could not read dir to delete: {}", e),
+    }
+}
+
 pub(crate) async fn dc_copy_file(
     context: &Context,
     src_path: impl AsRef<Path>,
@@ -612,8 +627,20 @@ pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
     (listflags & bitindex) == bitindex
 }
 
+/// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
+pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+    input
+        .as_ref()
+        .replace("\n", " ")
+        .replace("\r", " ")
+        .trim()
+        .to_string()
+}
+
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use std::convert::TryInto;
 
@@ -960,4 +987,10 @@ mod tests {
         assert_eq!(w, 100);
         assert_eq!(h, 50);
     }
+
+    #[test]
+    fn test_improve_single_line_input() {
+        assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
+        assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
+    }
 }

src/dehtml.rs

@@ -25,7 +25,19 @@ enum AddText {
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
 pub fn dehtml(buf: &str) -> String {
-    let buf = buf.trim();
+    let s = dehtml_quick_xml(buf);
+    if !s.trim().is_empty() {
+        return s;
+    }
+    let s = dehtml_manually(buf);
+    if !s.trim().is_empty() {
+        return s;
+    }
+    buf.to_string()
+}
+
+pub fn dehtml_quick_xml(buf: &str) -> String {
+    let buf = buf.trim().trim_start_matches("<!doctype html>");
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
@@ -171,9 +183,28 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     }
 }
 
+pub fn dehtml_manually(buf: &str) -> String {
+    // Just strip out everything between "<" and ">"
+    let mut strbuilder = String::new();
+    let mut show_next_chars = true;
+    for c in buf.chars() {
+        match c {
+            '<' => show_next_chars = false,
+            '>' => show_next_chars = true,
+            _ => {
+                if show_next_chars {
+                    strbuilder.push(c)
+                }
+            }
+        }
+    }
+    strbuilder
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::simplify::simplify;
 
     #[test]
     fn test_dehtml() {
@@ -182,20 +213,23 @@ mod tests {
                 "<a href='https://example.com'> Foo </a>",
                 "[ Foo ](https://example.com)",
             ),
-            ("<img href='/foo.png'>", ""),
             ("<b> bar </b>", "* bar *"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
-            // Note missing '
-            ("<a href='/foo.png>Hi</a> ", ""),
+            // Despite missing ', this should be shown:
+            ("<a href='/foo.png>Hi</a> ", "Hi "),
             (
                 "<a href='https://get.delta.chat/'/>",
                 "[](https://get.delta.chat/)",
             ),
             ("", ""),
+            ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
+            // Invalid html (at least DC should show the text if the html is invalid):
+            ("<!some invalid html code>\n<b>some text</b>", "some text"),
+            ("<This text is in brackets>", "<This text is in brackets>"),
         ];
         for (input, output) in cases {
-            assert_eq!(dehtml(input), output);
+            assert_eq!(simplify(dehtml(input), true).0, output);
         }
     }
 

src/e2ee.rs

@@ -135,41 +135,31 @@ pub async fn try_decrypt(
         .map(|from| from.addr)
         .unwrap_or_default();
 
-    let mut peerstate = None;
-    let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
-
-    if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await?;
+    let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
+    // Apply Autocrypt header
+    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            if let Some(ref header) = autocryptheader {
-                peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
-                peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            }
-        } else if let Some(ref header) = autocryptheader {
+            peerstate.apply_header(&header, message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        } else {
             let p = Peerstate::from_header(context, header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
     }
 
-    /* possibly perform decryption */
+    // Possibly perform decryption
     let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
     let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
 
-    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-        peerstate = Peerstate::from_addr(&context, &from).await?;
-    }
-    if let Some(peerstate) = peerstate {
+    if let Some(ref mut peerstate) = peerstate {
         peerstate.handle_fingerprint_change(context).await?;
-        if let Some(key) = peerstate.public_key {
-            public_keyring_for_validate.add(key);
-        } else if let Some(key) = peerstate.gossip_key {
-            public_keyring_for_validate.add(key);
+        if let Some(key) = &peerstate.public_key {
+            public_keyring_for_validate.add(key.clone());
+        } else if let Some(key) = &peerstate.gossip_key {
+            public_keyring_for_validate.add(key.clone());
         }
     }
 
@@ -181,6 +171,18 @@ pub async fn try_decrypt(
         &mut signatures,
     )
     .await?;
+
+    if let Some(mut peerstate) = peerstate {
+        // If message is not encrypted and it is not a read receipt, degrade encryption.
+        if out_mail.is_none()
+            && message_time > peerstate.last_seen_autocrypt
+            && !contains_report(mail)
+        {
+            peerstate.degrade_encryption(message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        }
+    }
+
     Ok((out_mail, signatures))
 }
 
@@ -319,6 +321,11 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
 mod tests {
     use super::*;
 
+    use crate::chat;
+    use crate::constants::Viewtype;
+    use crate::contact::{Contact, Origin};
+    use crate::message::Message;
+    use crate::param::Param;
     use crate::test_utils::*;
 
     mod ensure_secret_key_exists {
@@ -379,4 +386,106 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let data = b"blas";
         assert_eq!(has_decrypted_pgp_armor(data), false);
     }
+
+    #[async_std::test]
+    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let (contact_alice_id, _modified) = Contact::add_or_lookup(
+            &bob.ctx,
+            "Alice",
+            "alice@example.com",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+
+        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
+        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+
+        // Alice sends unencrypted message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        // Bob receives unencrypted message from Alice
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+
+        // Parsing a message is enough to update peerstate
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Bob sends encrypted message to Alice
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        let sent = bob.pop_sent_msg().await;
+
+        // Alice receives encrypted message from Bob
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+
+        let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_bob.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Now Alice and Bob have established keys.
+
+        // Alice sends encrypted message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message with Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);
+
+        Ok(())
+    }
 }

src/events.rs

@@ -119,10 +119,6 @@ pub enum EventType {
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
-    /// Emitted when an IMAP folder was emptied
-    #[strum(props(id = "106"))]
-    ImapFolderEmptied(String),
-
     /// Emitted when an new file in the $BLOBDIR was created
     #[strum(props(id = "150"))]
     NewBlobFile(String),
@@ -237,10 +233,16 @@ pub enum EventType {
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
-    ///
-    /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     #[strum(props(id = "2041"))]
-    ConfigureProgress(usize),
+    ConfigureProgress {
+        /// Progress.
+        ///
+        /// 0=error, 1-999=progress in permille, 1000=success and done
+        progress: usize,
+
+        /// Progress comment or error, something to display to the user.
+        comment: Option<String>,
+    },
 
     /// Inform about the import/export progress started by imex().
     ///

src/format_flowed.rs

@@ -0,0 +1,136 @@
+///! # format=flowed support
+///!
+///! Format=flowed is defined in
+///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+///!
+///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+///! during formatting, i.e., DelSp parameter introduced in RFC 3676
+///! is assumed to be set to "no".
+///!
+///! For received messages, DelSp parameter is honoured.
+
+/// Wraps line to 72 characters using format=flowed soft breaks.
+///
+/// 72 characters is the limit recommended by RFC 3676.
+///
+/// The function breaks line only after SP and before non-whitespace
+/// characters. It also does not insert breaks before ">" to avoid the
+/// need to do space stuffing (see RFC 3676) for quotes.
+///
+/// If there are long words, line may still exceed the limits on line
+/// length. However, this should be rare and should not result in
+/// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
+/// and Spam Assassin limit is 78 characters.
+fn format_line_flowed(line: &str) -> String {
+    let mut result = String::new();
+    let mut buffer = String::new();
+    let mut after_space = false;
+
+    for c in line.chars() {
+        if c == ' ' {
+            buffer.push(c);
+            after_space = true;
+        } else {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() && c != '>' {
+                // Flush the buffer and insert soft break (SP CRLF).
+                result += &buffer;
+                result += "\r\n";
+                buffer = String::new();
+            }
+            buffer.push(c);
+            after_space = false;
+        }
+    }
+    result + &buffer
+}
+
+/// Returns text formatted according to RFC 3767 (format=flowed).
+///
+/// This function accepts text separated by LF, but returns text
+/// separated by CRLF.
+///
+/// RFC 2646 technique is used to insert soft line breaks, so DelSp
+/// SHOULD be set to "no" when sending.
+pub fn format_flowed(text: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if line.len() > 78 {
+            result += &format_line_flowed(line);
+        } else {
+            result += line;
+        }
+    }
+    result
+}
+
+/// Joins lines in format=flowed text.
+///
+/// Lines must be separated by single LF.
+///
+/// Quote processing is not supported, it is assumed that they are
+/// deleted during simplification.
+///
+/// Signature separator line is not processed here, it is assumed to
+/// be stripped beforehand.
+pub fn unformat_flowed(text: &str, delsp: bool) -> String {
+    let mut result = String::new();
+    let mut skip_newline = true;
+
+    for line in text.split('\n') {
+        // Revert space-stuffing
+        let line = line.strip_prefix(" ").unwrap_or(line);
+
+        if !skip_newline {
+            result.push('\n');
+        }
+
+        if let Some(line) = line.strip_suffix(" ") {
+            // Flowed line
+            result += line;
+            if !delsp {
+                result.push(' ');
+            }
+            skip_newline = true;
+        } else {
+            // Fixed line
+            result += line;
+            skip_newline = false;
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_flowed() {
+        let text = "Foo bar baz";
+        assert_eq!(format_flowed(text), "Foo bar baz");
+
+        let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    \n\
+                    To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "This is the Autocrypt Setup Message used to transfer your key between clients.\r\n\
+                        \r\n\
+                        To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
+    }
+
+    #[test]
+    fn test_unformat_flowed() {
+        let text = "this is a very long message that should be wrapped using format=flowed and \n\
+            unwrapped on the receiver";
+        let expected =
+            "this is a very long message that should be wrapped using format=flowed and \
+                        unwrapped on the receiver";
+        assert_eq!(unformat_flowed(text, false), expected);
+    }
+}

src/headerdef.rs

@@ -3,7 +3,6 @@ use mailparse::{MailHeader, MailHeaderMap};
 
 #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
 #[strum(serialize_all = "kebab_case")]
-#[allow(dead_code)]
 pub enum HeaderDef {
     MessageId,
     Subject,

src/imap/idle.rs

@@ -5,31 +5,11 @@ use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
+use crate::error::{bail, format_err, Result};
 use crate::{context::Context, scheduler::InterruptInfo};
 
-use super::select_folder;
 use super::session::Session;
 
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete: {0}")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
-
-    #[error("IMAP IDLE protocol timed out: {0}")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
-
-    #[error("IMAP server does not have IDLE capability")]
-    IdleAbilityMissing,
-
-    #[error("IMAP select folder error: {0}")]
-    SelectFolderError(#[from] select_folder::Error),
-
-    #[error("Setup handle error: {0}")]
-    SetupHandleError(#[from] super::Error),
-}
-
 impl Imap {
     pub fn can_idle(&self) -> bool {
         self.config.can_idle
@@ -43,9 +23,9 @@ impl Imap {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
-            return Err(Error::IdleAbilityMissing);
+            bail!("IMAP server does not have IDLE capability");
         }
-        self.setup_handle_if_needed(context).await?;
+        self.setup_handle(context).await?;
 
         self.select_folder(context, watch_folder.clone()).await?;
 
@@ -72,7 +52,7 @@ impl Imap {
 
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
-                return Err(Error::IdleProtocolFailed(err));
+                bail!("IMAP IDLE protocol failed to init/complete: {}", err);
             }
 
             let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
@@ -115,7 +95,7 @@ impl Imap {
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(Error::IdleTimeout)??;
+                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
             self.session = Some(Session { inner: session });
         } else {
             warn!(context, "Attempted to idle without a session");

src/imap/mod.rs

@@ -19,70 +19,30 @@ use crate::context::Context;
 use crate::dc_receive_imf::{
     dc_receive_imf, from_field_to_contact_id, is_msgrmsg_rfc724_mid_in_list,
 };
+use crate::error::{bail, format_err, Result};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
-use crate::login_param::{CertificateChecks, LoginParam};
+use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{self, update_server_uid, MessageState};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::provider::get_provider_info;
-use crate::{chat, scheduler::InterruptInfo, stock::StockMessage};
+use crate::provider::{get_provider_info, Socket};
+use crate::{
+    chat, dc_tools::dc_extract_grpid_from_rfc724_mid, scheduler::InterruptInfo, stock::StockMessage,
+};
 
 mod client;
 mod idle;
 pub mod select_folder;
 mod session;
 
+use chat::get_chat_id_by_grpid;
 use client::Client;
 use message::Message;
 use session::Session;
 
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("IMAP Connect without configured params")]
-    ConnectWithoutConfigure,
-
-    #[error("IMAP Connection Failed params: {0}")]
-    ConnectionFailed(String),
-
-    #[error("IMAP No Connection established")]
-    NoConnection,
-
-    #[error("IMAP Could not get OAUTH token")]
-    OauthError,
-
-    #[error("IMAP Could not login as {0}")]
-    LoginFailed(String),
-
-    #[error("IMAP Could not fetch")]
-    FetchFailed(#[from] async_imap::error::Error),
-
-    #[error("IMAP operation attempted while it is torn down")]
-    InTeardown,
-
-    #[error("IMAP operation attempted while it is torn down")]
-    SqlError(#[from] crate::sql::Error),
-
-    #[error("IMAP got error from elsewhere")]
-    WrappedError(#[from] crate::error::Error),
-
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
-
-    #[error("Mail parse error")]
-    MailParseError(#[from] mailparse::MailParseError),
-
-    #[error("No mailbox selected, folder: {0}")]
-    NoMailbox(String),
-
-    #[error("IMAP other error: {0}")]
-    Other(String),
-}
-
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapActionResult {
     Failed,
@@ -105,9 +65,9 @@ const PREFETCH_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
                               AUTOCRYPT-SETUP-MESSAGE\
                               )])";
 const DELETE_CHECK_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])";
+const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])";
 const JUST_UID: &str = "(UID)";
 const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
-const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
@@ -147,12 +107,9 @@ enum FolderMeaning {
 #[derive(Debug)]
 struct ImapConfig {
     pub addr: String,
-    pub imap_server: String,
-    pub imap_port: u16,
-    pub imap_user: String,
-    pub imap_pw: String,
+    pub lp: ServerLoginParam,
     pub strict_tls: bool,
-    pub server_flags: usize,
+    pub oauth2: bool,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
     pub selected_folder_needs_expunge: bool,
@@ -167,12 +124,9 @@ impl Default for ImapConfig {
     fn default() -> Self {
         ImapConfig {
             addr: "".into(),
-            imap_server: "".into(),
-            imap_port: 0,
-            imap_user: "".into(),
-            imap_pw: "".into(),
+            lp: Default::default(),
             strict_tls: false,
-            server_flags: 0,
+            oauth2: false,
             selected_folder: None,
             selected_mailbox: None,
             selected_folder_needs_expunge: false,
@@ -207,9 +161,13 @@ impl Imap {
         self.should_reconnect = true;
     }
 
-    async fn setup_handle_if_needed(&mut self, context: &Context) -> Result<()> {
-        if self.config.imap_server.is_empty() {
-            return Err(Error::InTeardown);
+    /// Connects or reconnects if needed.
+    ///
+    /// It is safe to call this function if already connected, actions
+    /// are performed only as needed.
+    async fn try_setup_handle(&mut self, context: &Context) -> Result<()> {
+        if self.config.lp.server.is_empty() {
+            bail!("IMAP operation attempted while it is torn down");
         }
 
         if self.should_reconnect() {
@@ -219,40 +177,40 @@ impl Imap {
             return Ok(());
         }
 
-        let server_flags = self.config.server_flags as i32;
+        let oauth2 = self.config.oauth2;
 
-        let connection_res: ImapResult<Client> =
-            if (server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN)) != 0 {
-                let config = &mut self.config;
-                let imap_server: &str = config.imap_server.as_ref();
-                let imap_port = config.imap_port;
+        let connection_res: ImapResult<Client> = if self.config.lp.security == Socket::STARTTLS
+            || self.config.lp.security == Socket::Plain
+        {
+            let config = &mut self.config;
+            let imap_server: &str = config.lp.server.as_ref();
+            let imap_port = config.lp.port;
 
-                match Client::connect_insecure((imap_server, imap_port)).await {
-                    Ok(client) => {
-                        if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.strict_tls).await
-                        } else {
-                            Ok(client)
-                        }
+            match Client::connect_insecure((imap_server, imap_port)).await {
+                Ok(client) => {
+                    if config.lp.security == Socket::STARTTLS {
+                        client.secure(imap_server, config.strict_tls).await
+                    } else {
+                        Ok(client)
                     }
-                    Err(err) => Err(err),
                 }
-            } else {
-                let config = &self.config;
-                let imap_server: &str = config.imap_server.as_ref();
-                let imap_port = config.imap_port;
+                Err(err) => Err(err),
+            }
+        } else {
+            let config = &self.config;
+            let imap_server: &str = config.lp.server.as_ref();
+            let imap_port = config.lp.port;
 
-                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
-                    .await
-            };
+            Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls).await
+        };
 
         let login_res = match connection_res {
             Ok(client) => {
                 let config = &self.config;
-                let imap_user: &str = config.imap_user.as_ref();
-                let imap_pw: &str = config.imap_pw.as_ref();
+                let imap_user: &str = config.lp.user.as_ref();
+                let imap_pw: &str = config.lp.password.as_ref();
 
-                if (server_flags & DC_LP_AUTH_OAUTH2) != 0 {
+                if oauth2 {
                     let addr: &str = config.addr.as_ref();
 
                     if let Some(token) =
@@ -264,7 +222,7 @@ impl Imap {
                         };
                         client.authenticate("XOAUTH2", auth).await
                     } else {
-                        return Err(Error::OauthError);
+                        bail!("IMAP Could not get OAUTH token");
                     }
                 } else {
                     client.login(imap_user, imap_pw).await
@@ -273,8 +231,8 @@ impl Imap {
             Err(err) => {
                 let message = {
                     let config = &self.config;
-                    let imap_server: &str = config.imap_server.as_ref();
-                    let imap_port = config.imap_port;
+                    let imap_server: &str = config.lp.server.as_ref();
+                    let imap_port = config.lp.port;
                     context
                         .stock_string_repl_str2(
                             StockMessage::ServerResponse,
@@ -283,9 +241,7 @@ impl Imap {
                         )
                         .await
                 };
-                // IMAP connection failures are reported to users
-                emit_event!(context, EventType::ErrorNetwork(message));
-                return Err(Error::ConnectionFailed(err.to_string()));
+                bail!("{}: {}", message, err);
             }
         };
 
@@ -301,13 +257,12 @@ impl Imap {
             }
 
             Err((err, _)) => {
-                let imap_user = self.config.imap_user.to_owned();
+                let imap_user = self.config.lp.user.to_owned();
                 let message = context
                     .stock_string_repl_str(StockMessage::CannotLogin, &imap_user)
                     .await;
 
                 warn!(context, "{} ({})", message, err);
-                emit_event!(context, EventType::ErrorNetwork(message.clone()));
 
                 let lock = context.wrong_pw_warning_mutex.lock().await;
                 if self.login_failed_once
@@ -319,7 +274,7 @@ impl Imap {
                     drop(lock);
 
                     let mut msg = Message::new(Viewtype::Text);
-                    msg.text = Some(message);
+                    msg.text = Some(message.clone());
                     if let Err(e) =
                         chat::add_device_msg_with_importance(context, None, Some(&mut msg), true)
                             .await
@@ -331,11 +286,24 @@ impl Imap {
                 }
 
                 self.trigger_reconnect();
-                Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
+                Err(format_err!("{}: {}", message, err))
             }
         }
     }
 
+    /// Connects or reconnects if not already connected.
+    ///
+    /// This function emits network error if it fails.  It should not
+    /// be used during configuration to avoid showing failed attempt
+    /// errors to the user.
+    async fn setup_handle(&mut self, context: &Context) -> Result<()> {
+        let res = self.try_setup_handle(context).await;
+        if let Err(ref err) = res {
+            emit_event!(context, EventType::ErrorNetwork(err.to_string()));
+        }
+        res
+    }
+
     async fn unsetup_handle(&mut self, context: &Context) {
         // Close folder if messages should be expunged
         if let Err(err) = self.close_folder(context).await {
@@ -357,56 +325,64 @@ impl Imap {
         let mut cfg = &mut self.config;
 
         cfg.addr = "".into();
-        cfg.imap_server = "".into();
-        cfg.imap_user = "".into();
-        cfg.imap_pw = "".into();
-        cfg.imap_port = 0;
+        cfg.lp = Default::default();
 
         cfg.can_idle = false;
         cfg.can_move = false;
     }
 
-    /// Connects to imap account using already-configured parameters.
+    /// Connects to IMAP account using already-configured parameters.
+    ///
+    /// Emits network error if connection fails.
     pub async fn connect_configured(&mut self, context: &Context) -> Result<()> {
         if self.is_connected() && !self.should_reconnect() {
             return Ok(());
         }
         if !context.is_configured().await {
-            return Err(Error::ConnectWithoutConfigure);
+            bail!("IMAP Connect without configured params");
         }
 
         let param = LoginParam::from_database(context, "configured_").await;
         // the trailing underscore is correct
 
-        if self.connect(context, &param).await {
-            self.ensure_configured_folders(context, true).await
+        if let Err(err) = self
+            .connect(
+                context,
+                &param.imap,
+                &param.addr,
+                param.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+            )
+            .await
+        {
+            bail!("IMAP Connection Failed with params {}: {}", param, err);
         } else {
-            Err(Error::ConnectionFailed(format!("{}", param)))
+            self.ensure_configured_folders(context, true).await
         }
     }
 
     /// Tries connecting to imap account using the specific login parameters.
-    pub async fn connect(&mut self, context: &Context, lp: &LoginParam) -> bool {
-        if lp.mail_server.is_empty() || lp.mail_user.is_empty() || lp.mail_pw.is_empty() {
-            return false;
+    ///
+    /// `addr` is used to renew token if OAuth2 authentication is used.
+    ///
+    /// Does not emit network errors, can be used to try various
+    /// parameters during autoconfiguration.
+    pub async fn connect(
+        &mut self,
+        context: &Context,
+        lp: &ServerLoginParam,
+        addr: &str,
+        oauth2: bool,
+    ) -> Result<()> {
+        if lp.server.is_empty() || lp.user.is_empty() || lp.password.is_empty() {
+            bail!("Incomplete IMAP connection parameters");
         }
 
         {
-            let addr = &lp.addr;
-            let imap_server = &lp.mail_server;
-            let imap_port = lp.mail_port as u16;
-            let imap_user = &lp.mail_user;
-            let imap_pw = &lp.mail_pw;
-            let server_flags = lp.server_flags as usize;
-
             let mut config = &mut self.config;
             config.addr = addr.to_string();
-            config.imap_server = imap_server.to_string();
-            config.imap_port = imap_port;
-            config.imap_user = imap_user.to_string();
-            config.imap_pw = imap_pw.to_string();
-            let provider = get_provider_info(&lp.addr);
-            config.strict_tls = match lp.imap_certificate_checks {
+            config.lp = lp.clone();
+            let provider = get_provider_info(&addr);
+            config.strict_tls = match lp.certificate_checks {
                 CertificateChecks::Automatic => {
                     provider.map_or(false, |provider| provider.strict_tls)
                 }
@@ -414,20 +390,20 @@ impl Imap {
                 CertificateChecks::AcceptInvalidCertificates
                 | CertificateChecks::AcceptInvalidCertificates2 => false,
             };
-            config.server_flags = server_flags;
+            config.oauth2 = oauth2;
         }
 
-        if let Err(err) = self.setup_handle_if_needed(context).await {
-            warn!(context, "failed to setup imap handle: {}", err);
+        if let Err(err) = self.try_setup_handle(context).await {
+            warn!(context, "try_setup_handle: {}", err);
             self.free_connect_params().await;
-            return false;
+            return Err(err);
         }
 
         let teardown = match &mut self.session {
             Some(ref mut session) => match session.capabilities().await {
                 Ok(caps) => {
                     if !context.sql.is_open().await {
-                        warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.mail_user,);
+                        warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.user,);
                         true
                     } else {
                         let can_idle = caps.has_str("IDLE");
@@ -447,7 +423,7 @@ impl Imap {
                             context,
                             EventType::ImapConnected(format!(
                                 "IMAP-LOGIN as {}, capabilities: {}",
-                                lp.mail_user, caps_list,
+                                lp.user, caps_list,
                             ))
                         );
                         false
@@ -464,10 +440,12 @@ impl Imap {
         if teardown {
             self.disconnect(context).await;
 
-            false
-        } else {
-            true
+            warn!(
+                context,
+                "IMAP disconnected immediately after connecting due to error"
+            );
         }
+        Ok(())
     }
 
     pub async fn disconnect(&mut self, context: &Context) {
@@ -478,9 +456,9 @@ impl Imap {
     pub async fn fetch(&mut self, context: &Context, watch_folder: &str) -> Result<()> {
         if !context.sql.is_open().await {
             // probably shutdown
-            return Err(Error::InTeardown);
+            bail!("IMAP operation attempted while it is torn down");
         }
-        self.setup_handle_if_needed(context).await?;
+        self.setup_handle(context).await?;
 
         while self.fetch_new_messages(context, &watch_folder).await? {
             // We fetch until no more new messages are there.
@@ -514,6 +492,81 @@ impl Imap {
         }
     }
 
+    /// Synchronizes UIDs in the database with UIDs on the server.
+    ///
+    /// It is assumed that no operations are taking place on the same
+    /// folder at the moment. Make sure to run it in the same
+    /// thread/task as other network operations on this folder to
+    /// avoid race conditions.
+    pub(crate) async fn resync_folder_uids(
+        &mut self,
+        context: &Context,
+        folder: String,
+    ) -> Result<()> {
+        // Collect pairs of UID and Message-ID.
+        let mut msg_ids = BTreeMap::new();
+
+        self.select_folder(context, Some(&folder)).await?;
+
+        let session = if let Some(ref mut session) = &mut self.session {
+            session
+        } else {
+            bail!("IMAP No Connection established");
+        };
+
+        match session.uid_fetch("1:*", RFC724MID_UID).await {
+            Ok(mut list) => {
+                while let Some(fetch) = list.next().await {
+                    let msg = fetch?;
+
+                    // Get Message-ID
+                    let message_id = get_fetch_headers(&msg)
+                        .and_then(|headers| prefetch_get_message_id(&headers))
+                        .ok();
+
+                    if let (Some(uid), Some(rfc724_mid)) = (msg.uid, message_id) {
+                        msg_ids.insert(uid, rfc724_mid);
+                    }
+                }
+            }
+            Err(err) => {
+                bail!("Can't resync folder {}: {}", folder, err);
+            }
+        }
+
+        info!(
+            context,
+            "Resync: collected {} message IDs in folder {}",
+            msg_ids.len(),
+            &folder
+        );
+
+        // Write collected UIDs to SQLite database.
+        context
+            .sql
+            .with_conn(move |mut conn| {
+                let conn2 = &mut conn;
+                let tx = conn2.transaction()?;
+                tx.execute(
+                    "UPDATE msgs SET server_uid=0 WHERE server_folder=?",
+                    params![folder],
+                )?;
+                for (uid, rfc724_mid) in &msg_ids {
+                    // This may detect previously undetected moved
+                    // messages, so we update server_folder too.
+                    tx.execute(
+                        "UPDATE msgs \
+                         SET server_folder=?,server_uid=? WHERE rfc724_mid=?",
+                        params![folder, uid, rfc724_mid],
+                    )?;
+                }
+                tx.commit()?;
+                Ok(())
+            })
+            .await?;
+        Ok(())
+    }
+
     /// return Result with (uid_validity, last_seen_uid) tuple.
     pub(crate) async fn select_with_uidvalidity(
         &mut self,
@@ -529,13 +582,12 @@ impl Imap {
         let mailbox = config
             .selected_mailbox
             .as_ref()
-            .ok_or_else(|| Error::NoMailbox(folder.to_string()))?;
+            .ok_or_else(|| format_err!("No mailbox selected, folder: {}", folder))?;
 
         let new_uid_validity = match mailbox.uid_validity {
             Some(v) => v,
             None => {
-                let s = format!("No UIDVALIDITY for folder {:?}", folder);
-                return Err(Error::Other(s));
+                bail!("No UIDVALIDITY for folder {:?}", folder);
             }
         };
 
@@ -582,21 +634,24 @@ impl Imap {
                             if let Some(new_last_seen_uid) = new_last_seen_uid {
                                 new_last_seen_uid
                             } else {
-                                return Err(Error::Other("failed to fetch".into()));
+                                bail!("failed to fetch");
                             }
                         }
                         Err(err) => {
-                            return Err(Error::FetchFailed(err));
+                            bail!("IMAP Could not fetch: {}", err);
                         }
                     }
                 } else {
-                    return Err(Error::NoConnection);
+                    bail!("IMAP No Connection established");
                 }
             }
         };
 
         self.set_config_last_seen_uid(context, &folder, new_uid_validity, new_last_seen_uid)
             .await;
+        if uid_validity != 0 || last_seen_uid != 0 {
+            job::schedule_resync(context).await;
+        }
         info!(
             context,
             "uid/validity change: new {}/{} current {}/{}",
@@ -693,7 +748,7 @@ impl Imap {
         uid: u32,
     ) -> Result<BTreeMap<u32, async_imap::types::Fetch>> {
         if self.session.is_none() {
-            return Err(Error::NoConnection);
+            bail!("IMAP No Connection established");
         }
 
         let session = self.session.as_mut().unwrap();
@@ -704,11 +759,11 @@ impl Imap {
         let mut list = session
             .uid_fetch(set, PREFETCH_FLAGS)
             .await
-            .map_err(Error::FetchFailed)?;
+            .map_err(|err| format_err!("IMAP Could not fetch: {}", err))?;
 
         let mut msgs = BTreeMap::new();
         while let Some(fetch) = list.next().await {
-            let msg = fetch.map_err(|err| Error::Other(err.to_string()))?;
+            let msg = fetch?;
             if let Some(msg_uid) = msg.uid {
                 msgs.insert(msg_uid, msg);
             }
@@ -1170,14 +1225,14 @@ impl Imap {
 
     pub async fn configure_folders(&mut self, context: &Context, create_mvbox: bool) -> Result<()> {
         if !self.is_connected() {
-            return Err(Error::NoConnection);
+            bail!("IMAP No Connection established");
         }
 
         if let Some(ref mut session) = &mut self.session {
             let mut folders = match session.list(Some(""), Some("*")).await {
                 Ok(f) => f,
                 Err(err) => {
-                    return Err(Error::Other(format!("list_folders failed {:?}", err)));
+                    bail!("list_folders failed: {}", err);
                 }
             };
 
@@ -1188,7 +1243,7 @@ impl Imap {
             let mut fallback_folder = get_fallback_folder(&delimiter);
 
             while let Some(folder) = folders.next().await {
-                let folder = folder.map_err(|err| Error::Other(err.to_string()))?;
+                let folder = folder?;
                 info!(context, "Scanning folder: {:?}", folder);
 
                 // Update the delimiter iff there is a different one, but only once.
@@ -1282,60 +1337,6 @@ impl Imap {
         info!(context, "FINISHED configuring IMAP-folders.");
         Ok(())
     }
-
-    pub async fn empty_folder(&mut self, context: &Context, folder: &str) {
-        info!(context, "emptying folder {}", folder);
-
-        // we want to report all error to the user
-        // (no retry should be attempted)
-        if folder.is_empty() {
-            error!(context, "cannot perform empty, folder not set");
-            return;
-        }
-        if let Err(err) = self.setup_handle_if_needed(context).await {
-            error!(context, "could not setup imap connection: {:?}", err);
-            return;
-        }
-        if let Err(err) = self.select_folder(context, Some(&folder)).await {
-            error!(
-                context,
-                "Could not select {} for expunging: {:?}", folder, err
-            );
-            return;
-        }
-
-        if !self
-            .add_flag_finalized_with_set(context, SELECT_ALL, "\\Deleted")
-            .await
-        {
-            error!(context, "Cannot mark messages for deletion {}", folder);
-            return;
-        }
-
-        // we now trigger expunge to actually delete messages
-        self.config.selected_folder_needs_expunge = true;
-        match self.select_folder::<String>(context, None).await {
-            Ok(()) => {
-                emit_event!(context, EventType::ImapFolderEmptied(folder.to_string()));
-            }
-            Err(err) => {
-                error!(context, "expunge failed {}: {:?}", folder, err);
-            }
-        }
-        if let Err(err) = context
-            .sql
-            .execute(
-                "UPDATE msgs SET server_folder='',server_uid=0 WHERE server_folder=?",
-                paramsv![folder],
-            )
-            .await
-        {
-            warn!(
-                context,
-                "Failed to reset server_uid and server_folder for deleted messages: {}", err
-            );
-        }
-    }
 }
 
 /// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
@@ -1473,7 +1474,7 @@ fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result<String>
     if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId) {
         Ok(crate::mimeparser::parse_message_id(&message_id)?)
     } else {
-        Err(Error::Other("prefetch: No message ID found".to_string()))
+        bail!("prefetch: No message ID found");
     }
 }
 
@@ -1501,6 +1502,18 @@ pub(crate) async fn prefetch_should_download(
     headers: &[mailparse::MailHeader<'_>],
     show_emails: ShowEmails,
 ) -> Result<bool> {
+    if let Some(rfc724_mid) = headers.get_header_value(HeaderDef::MessageId) {
+        if let Some(group_id) = dc_extract_grpid_from_rfc724_mid(&rfc724_mid) {
+            if let Ok((chat_id, _, _)) = get_chat_id_by_grpid(context, group_id).await {
+                if !chat_id.is_unset() {
+                    // This might be a group command, like removing a group member.
+                    // We really need to fetch this to avoid inconsistent group state.
+                    return Ok(true);
+                }
+            }
+        }
+    }
+
     let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
     let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await;
 

src/imex.rs

@@ -82,18 +82,46 @@ pub async fn imex(
     what: ImexMode,
     param1: Option<impl AsRef<Path>>,
 ) -> Result<()> {
-    use futures::future::FutureExt;
-
     let cancel = context.alloc_ongoing().await?;
-    let res = imex_inner(context, what, param1)
-        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
-        .await;
+
+    let res = async {
+        let success = imex_inner(context, what, param1).await;
+        match success {
+            Ok(()) => {
+                info!(context, "IMEX successfully completed");
+                context.emit_event(EventType::ImexProgress(1000));
+                Ok(())
+            }
+            Err(err) => {
+                cleanup_aborted_imex(context, what).await;
+                error!(context, "{}", err);
+                context.emit_event(EventType::ImexProgress(0));
+                bail!("IMEX FAILED to complete: {}", err);
+            }
+        }
+    }
+    .race(async {
+        cancel.recv().await.ok();
+        cleanup_aborted_imex(context, what).await;
+        Err(format_err!("canceled"))
+    })
+    .await;
 
     context.free_ongoing().await;
 
     res
 }
 
+async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
+    if what == ImexMode::ImportBackup {
+        dc_delete_file(context, context.get_dbfile()).await;
+        dc_delete_files_in_dir(context, context.get_blobdir()).await;
+    }
+    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
+        context.sql.open(context, context.get_dbfile(), false).await;
+    }
+}
+
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
@@ -192,10 +220,8 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(
-        Param::ForcePlaintext,
-        ForcePlaintext::NoAutocryptHeader as i32,
-    );
+    msg.param.set_int(Param::ForcePlaintext, 1);
+    msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
@@ -425,7 +451,7 @@ async fn imex_inner(
         }
     }
 
-    let success = match what {
+    match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
@@ -434,18 +460,6 @@ async fn imex_inner(
         ImexMode::ExportBackup => export_backup_old(context, path).await,
         // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
-    };
-
-    match success {
-        Ok(()) => {
-            info!(context, "IMEX successfully completed");
-            context.emit_event(EventType::ImexProgress(1000));
-            Ok(())
-        }
-        Err(err) => {
-            context.emit_event(EventType::ImexProgress(0));
-            bail!("IMEX FAILED to complete: {}", err);
-        }
     }
 }
 
@@ -632,6 +646,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
     let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let _d = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
@@ -670,13 +685,19 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         }
         Err(e) => {
             error!(context, "backup failed: {}", e);
-            // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-            fs::remove_file(temp_path).await?;
         }
     }
 
     res
 }
+struct DeleteOnDrop(PathBuf);
+impl Drop for DeleteOnDrop {
+    fn drop(&mut self) {
+        let file = self.0.clone();
+        // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
+    }
+}
 
 async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
     let file = File::create(temp_path).await?;

src/job.rs

@@ -17,7 +17,6 @@ use async_smtp::smtp::response::Detail;
 use crate::blob::BlobObject;
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -26,7 +25,6 @@ use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::EventType;
 use crate::imap::*;
 use crate::location;
-use crate::login_param::LoginParam;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
@@ -94,7 +92,6 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
-    EmptyServer = 107,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -102,6 +99,10 @@ pub enum Action {
     MoveMsg = 200,
     DeleteMsgOnImap = 210,
 
+    // UID synchronization is high-priority to make sure correct UIDs
+    // are used by message moving/deletion.
+    ResyncFolders = 300,
+
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
     MaybeSendLocations = 5005, // low priority ...
     MaybeSendLocationsEnded = 5007,
@@ -124,7 +125,7 @@ impl From<Action> for Thread {
 
             Housekeeping => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
-            EmptyServer => Thread::Imap,
+            ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
 
@@ -338,12 +339,9 @@ impl Job {
 
     pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         let filename = job_try!(job_try!(self
@@ -486,12 +484,9 @@ impl Job {
         let recipients = vec![recipient];
 
         // connect to SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         self.smtp_send(context, recipients, body, self.job_id, smtp, || {
@@ -565,6 +560,11 @@ impl Job {
                     context,
                     "The message is deleted from the server when all parts are deleted.",
                 );
+            } else if cnt == 0 {
+                warn!(
+                    context,
+                    "The message {} has no UID on the server to delete", &msg.rfc724_mid
+                );
             } else {
                 /* if this is the last existing part of the message,
                 we delete the message from the server */
@@ -619,21 +619,40 @@ impl Job {
         }
     }
 
-    async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
+    /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
+    ///
+    /// If a copy of the message is present in multiple folders, mvbox
+    /// is preferred to inbox, which is in turn preferred to
+    /// sentbox. This is because in the database it is impossible to
+    /// store multiple UIDs for one message, so we prefer to
+    /// automatically delete messages in the folders managed by Delta
+    /// Chat in contrast to the Sent folder, which is normally managed
+    /// by the user via webmail or another email client.
+    async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.connect_configured(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
 
-        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
-                imap.empty_folder(context, &mvbox_folder).await;
-            }
+        if let Some(sentbox_folder) = &context.get_config(Config::ConfiguredSentboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, sentbox_folder.to_string())
+                    .await
+            );
         }
-        if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
-                imap.empty_folder(context, &inbox_folder).await;
-            }
+
+        if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, inbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, mvbox_folder.to_string())
+                    .await
+            );
         }
         Status::Finished(Ok(()))
     }
@@ -984,8 +1003,8 @@ async fn perform_job_action(
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
-        Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
+        Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
         Action::Housekeeping => {
@@ -1019,6 +1038,15 @@ async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     Ok(())
 }
 
+pub(crate) async fn schedule_resync(context: &Context) {
+    kill_action(context, Action::ResyncFolders).await;
+    add(
+        context,
+        Job::new(Action::ResyncFolders, 0, Params::new(), 0),
+    )
+    .await;
+}
+
 /// Creates a job.
 pub fn create(action: Action, foreign_id: i32, param: Params, delay_seconds: i64) -> Result<Job> {
     ensure!(
@@ -1041,8 +1069,8 @@ pub async fn add(context: &Context, job: Job) {
         match action {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
-            | Action::EmptyServer
             | Action::DeleteMsgOnImap
+            | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::MoveMsg => {
                 info!(context, "interrupt: imap");

src/key.rs

@@ -355,7 +355,7 @@ pub async fn store_self_keypair(
 }
 
 /// A key fingerprint
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+#[derive(Clone, Eq, PartialEq, Hash)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
@@ -375,6 +375,14 @@ impl Fingerprint {
     }
 }
 
+impl fmt::Debug for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Fingerprint")
+            .field("hex", &self.hex())
+            .finish()
+    }
+}
+
 /// Make a human-readable fingerprint.
 impl fmt::Display for Fingerprint {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -529,11 +537,11 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-        for i in 0..4096 {
-            bad_data[i] = (i & 0xff) as u8;
+        for (i, v) in bad_data.iter_mut().enumerate() {
+            *v = (i & 0xff) as u8;
         }
         for j in 0..(4096 / 40) {
-            let slice = &bad_data[j..j + 4096 / 2 + j];
+            let slice = &bad_data.get(j..j + 4096 / 2 + j).unwrap();
             assert!(SignedPublicKey::from_slice(slice).is_err());
             assert!(SignedSecretKey::from_slice(slice).is_err());
         }
@@ -593,7 +601,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let ctx0 = ctx.clone();
         let thr0 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx.clone();
+        let ctx1 = ctx;
         let thr1 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
         let res0 = thr0.join().unwrap();

src/lib.rs

@@ -54,6 +54,7 @@ pub mod imex;
 mod scheduler;
 #[macro_use]
 pub mod job;
+mod format_flowed;
 pub mod key;
 mod keyring;
 pub mod location;

src/location.rs

@@ -469,7 +469,7 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
          <Document>\n\
          <Placemark>\
          <Timestamp><when>{}</when></Timestamp>\
-         <Point><coordinates>{:.2},{:.2}</coordinates></Point>\
+         <Point><coordinates>{},{}</coordinates></Point>\
          </Placemark>\n\
          </Document>\n\
          </kml>",
@@ -723,6 +723,8 @@ pub(crate) async fn job_maybe_send_locations_ended(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use crate::test_utils::TestContext;
 
@@ -757,4 +759,22 @@ mod tests {
         assert!(locations_ref[1].accuracy < 2.6f64);
         assert_eq!(locations_ref[1].timestamp, 1544739072);
     }
+
+    #[async_std::test]
+    async fn test_get_message_kml() {
+        let context = TestContext::new().await;
+        let timestamp = 1598490000;
+
+        let xml = get_message_kml(timestamp, 51.423723f64, 8.552556f64);
+        let kml = Kml::parse(&context.ctx, xml.as_bytes()).expect("parsing failed");
+        let locations_ref = &kml.locations;
+        assert_eq!(locations_ref.len(), 1);
+
+        assert!(locations_ref[0].latitude >= 51.423723f64);
+        assert!(locations_ref[0].latitude < 51.423724f64);
+        assert!(locations_ref[0].longitude >= 8.552556f64);
+        assert!(locations_ref[0].longitude < 8.552557f64);
+        assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
+        assert_eq!(locations_ref[0].timestamp, timestamp);
+    }
 }

src/login_param.rs

@@ -3,13 +3,16 @@
 use std::borrow::Cow;
 use std::fmt;
 
-use crate::context::Context;
+use crate::{context::Context, provider::Socket};
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(i32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
+    /// Same as AcceptInvalidCertificates unless overridden by
+    /// `strict_tls` setting in provider database.
     Automatic = 0,
+
     Strict = 1,
 
     /// Same as AcceptInvalidCertificates
@@ -25,30 +28,29 @@ impl Default for CertificateChecks {
     }
 }
 
+/// Login parameters for a single server, either IMAP or SMTP
+#[derive(Default, Debug, Clone)]
+pub struct ServerLoginParam {
+    pub server: String,
+    pub user: String,
+    pub password: String,
+    pub port: u16,
+    pub security: Socket,
+
+    /// TLS options: whether to allow invalid certificates and/or
+    /// invalid hostnames
+    pub certificate_checks: CertificateChecks,
+}
+
 #[derive(Default, Debug, Clone)]
 pub struct LoginParam {
     pub addr: String,
-    pub mail_server: String,
-    pub mail_user: String,
-    pub mail_pw: String,
-    pub mail_port: i32,
-    /// IMAP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub imap_certificate_checks: CertificateChecks,
-    pub send_server: String,
-    pub send_user: String,
-    pub send_pw: String,
-    pub send_port: i32,
-    /// SMTP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub smtp_certificate_checks: CertificateChecks,
+    pub imap: ServerLoginParam,
+    pub smtp: ServerLoginParam,
     pub server_flags: i32,
 }
 
 impl LoginParam {
-    /// Create a new `LoginParam` with default values.
-    pub fn new() -> Self {
-        Default::default()
-    }
-
     /// Read the login parameters from the database.
     pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
         let prefix = prefix.as_ref();
@@ -77,6 +79,13 @@ impl LoginParam {
         let key = format!("{}mail_pw", prefix);
         let mail_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}mail_security", prefix);
+        let mail_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}imap_certificate_checks", prefix);
         let imap_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -100,6 +109,13 @@ impl LoginParam {
         let key = format!("{}send_pw", prefix);
         let send_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}send_security", prefix);
+        let send_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}smtp_certificate_checks", prefix);
         let smtp_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -116,16 +132,22 @@ impl LoginParam {
 
         LoginParam {
             addr,
-            mail_server,
-            mail_user,
-            mail_pw,
-            mail_port,
-            imap_certificate_checks,
-            send_server,
-            send_user,
-            send_pw,
-            send_port,
-            smtp_certificate_checks,
+            imap: ServerLoginParam {
+                server: mail_server,
+                user: mail_user,
+                password: mail_pw,
+                port: mail_port as u16,
+                security: mail_security,
+                certificate_checks: imap_certificate_checks,
+            },
+            smtp: ServerLoginParam {
+                server: send_server,
+                user: send_user,
+                password: send_pw,
+                port: send_port as u16,
+                security: send_security,
+                certificate_checks: smtp_certificate_checks,
+            },
             server_flags,
         }
     }
@@ -143,41 +165,51 @@ impl LoginParam {
         sql.set_raw_config(context, key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_server))
+        sql.set_raw_config(context, key, Some(&self.imap.server))
             .await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(context, key, self.mail_port).await?;
+        sql.set_raw_config_int(context, key, self.imap.port as i32)
+            .await?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_user))
+        sql.set_raw_config(context, key, Some(&self.imap.user))
             .await?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_pw))
+        sql.set_raw_config(context, key, Some(&self.imap.password))
+            .await?;
+
+        let key = format!("{}mail_security", prefix);
+        sql.set_raw_config_int(context, key, self.imap.security as i32)
             .await?;
 
         let key = format!("{}imap_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.imap.certificate_checks as i32)
             .await?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_server))
+        sql.set_raw_config(context, key, Some(&self.smtp.server))
             .await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(context, key, self.send_port).await?;
+        sql.set_raw_config_int(context, key, self.smtp.port as i32)
+            .await?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_user))
+        sql.set_raw_config(context, key, Some(&self.smtp.user))
             .await?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_pw))
+        sql.set_raw_config(context, key, Some(&self.smtp.password))
+            .await?;
+
+        let key = format!("{}send_security", prefix);
+        sql.set_raw_config_int(context, key, self.smtp.security as i32)
             .await?;
 
         let key = format!("{}smtp_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.smtp.certificate_checks as i32)
             .await?;
 
         let key = format!("{}server_flags", prefix);
@@ -199,16 +231,24 @@ impl fmt::Display for LoginParam {
             f,
             "{} imap:{}:{}:{}:{}:cert_{} smtp:{}:{}:{}:{}:cert_{} {}",
             unset_empty(&self.addr),
-            unset_empty(&self.mail_user),
-            if !self.mail_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.mail_server),
-            self.mail_port,
-            self.imap_certificate_checks,
-            unset_empty(&self.send_user),
-            if !self.send_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.send_server),
-            self.send_port,
-            self.smtp_certificate_checks,
+            unset_empty(&self.imap.user),
+            if !self.imap.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.imap.server),
+            self.imap.port,
+            self.imap.certificate_checks,
+            unset_empty(&self.smtp.user),
+            if !self.smtp.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.smtp.server),
+            self.smtp.port,
+            self.smtp.certificate_checks,
             flags_readable,
         )
     }
@@ -237,30 +277,6 @@ fn get_readable_flags(flags: i32) -> String {
                 res += "AUTH_NORMAL ";
                 flag_added = true;
             }
-            if 1 << bit == 0x100 {
-                res += "IMAP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x200 {
-                res += "IMAP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x400 {
-                res += "IMAP_PLAIN ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x10000 {
-                res += "SMTP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x20000 {
-                res += "SMTP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x40000 {
-                res += "SMTP_PLAIN ";
-                flag_added = true;
-            }
             if flag_added {
                 res += &format!("{:#0x}", 1 << bit);
             }

src/message.rs

@@ -494,7 +494,7 @@ impl Message {
     pub fn get_text(&self) -> Option<String> {
         self.text
             .as_ref()
-            .map(|text| dc_truncate(text, 30000).to_string())
+            .map(|text| dc_truncate(text, DC_MAX_GET_TEXT_LEN).to_string())
     }
 
     pub fn get_filename(&self) -> Option<String> {
@@ -969,7 +969,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
-    let rawtxt = dc_truncate(rawtxt.trim(), 100_000);
+    let rawtxt = dc_truncate(rawtxt.trim(), DC_MAX_GET_INFO_LEN);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
     ret += &format!("Sent: {}", fts);
@@ -1096,30 +1096,70 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
 pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     let extension: &str = &path.extension()?.to_str()?.to_lowercase();
     let info = match extension {
+        // before using viewtype other than Viewtype::File,
+        // make sure, all target UIs support that type in the context of the used viewer/player.
+        // if in doubt, it is better to default to Viewtype::File that passes handing to an external app.
+        // (cmp. https://developer.android.com/guide/topics/media/media-formats )
         "3gp" => (Viewtype::Video, "video/3gpp"),
         "aac" => (Viewtype::Audio, "audio/aac"),
         "avi" => (Viewtype::Video, "video/x-msvideo"),
+        "doc" => (Viewtype::File, "application/msword"),
+        "docx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        ),
+        "epub" => (Viewtype::File, "application/epub+zip"),
         "flac" => (Viewtype::Audio, "audio/flac"),
         "gif" => (Viewtype::Gif, "image/gif"),
+        "html" => (Viewtype::File, "text/html"),
+        "htm" => (Viewtype::File, "text/html"),
+        "ico" => (Viewtype::File, "image/vnd.microsoft.icon"),
+        "jar" => (Viewtype::File, "application/java-archive"),
         "jpeg" => (Viewtype::Image, "image/jpeg"),
         "jpe" => (Viewtype::Image, "image/jpeg"),
         "jpg" => (Viewtype::Image, "image/jpeg"),
+        "json" => (Viewtype::File, "application/json"),
         "mov" => (Viewtype::Video, "video/quicktime"),
         "mp3" => (Viewtype::Audio, "audio/mpeg"),
         "mp4" => (Viewtype::Video, "video/mp4"),
+        "odp" => (
+            Viewtype::File,
+            "application/vnd.oasis.opendocument.presentation",
+        ),
+        "ods" => (
+            Viewtype::File,
+            "application/vnd.oasis.opendocument.spreadsheet",
+        ),
+        "odt" => (Viewtype::File, "application/vnd.oasis.opendocument.text"),
         "oga" => (Viewtype::Audio, "audio/ogg"),
         "ogg" => (Viewtype::Audio, "audio/ogg"),
-        "ogv" => (Viewtype::Video, "video/ogg"),
-        "opus" => (Viewtype::Audio, "audio/ogg"),
+        "ogv" => (Viewtype::File, "video/ogg"),
+        "opus" => (Viewtype::File, "audio/ogg"), // not supported eg. on Android 4
+        "otf" => (Viewtype::File, "font/otf"),
+        "pdf" => (Viewtype::File, "application/pdf"),
         "png" => (Viewtype::Image, "image/png"),
-        "spx" => (Viewtype::Audio, "audio/ogg"), // Ogg Speex Profile
-        "svg" => (Viewtype::Image, "image/svg+xml"),
+        "rar" => (Viewtype::File, "application/vnd.rar"),
+        "rtf" => (Viewtype::File, "application/rtf"),
+        "spx" => (Viewtype::File, "audio/ogg"), // Ogg Speex Profile
+        "svg" => (Viewtype::File, "image/svg+xml"),
         "tgs" => (Viewtype::Sticker, "application/x-tgsticker"),
+        "tiff" => (Viewtype::File, "image/tiff"),
+        "tif" => (Viewtype::File, "image/tiff"),
+        "ttf" => (Viewtype::File, "font/ttf"),
         "vcard" => (Viewtype::File, "text/vcard"),
         "vcf" => (Viewtype::File, "text/vcard"),
+        "wav" => (Viewtype::File, "audio/wav"),
+        "weba" => (Viewtype::File, "audio/webm"),
         "webm" => (Viewtype::Video, "video/webm"),
-        "webp" => (Viewtype::Image, "image/webp"),
+        "webp" => (Viewtype::Image, "image/webp"), // iOS via SDWebImage, Android since 4.0
         "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
+        "xhtml" => (Viewtype::File, "application/xhtml+xml"),
+        "xlsx" => (
+            Viewtype::File,
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        ),
+        "xml" => (Viewtype::File, "application/vnd.ms-excel"),
+        "zip" => (Viewtype::File, "application/zip"),
         _ => {
             return None;
         }
@@ -1757,16 +1797,6 @@ pub async fn update_server_uid(
     }
 }
 
-#[allow(dead_code)]
-pub async fn dc_empty_server(context: &Context, flags: u32) {
-    job::kill_action(context, Action::EmptyServer).await;
-    job::add(
-        context,
-        job::Job::new(Action::EmptyServer, flags, Params::new(), 0),
-    )
-    .await;
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1853,8 +1883,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &some_file, 50, &ctx).await,
             "Voice message" // file names are not added for voice messages
         );
 
@@ -1864,8 +1893,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &some_file, 50, &ctx).await,
             "Audio \u{2013} foo.bar" // file name is added for audio
         );
 
@@ -1881,8 +1909,7 @@ mod tests {
         );
 
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &mut some_file, 50, &ctx)
-                .await,
+            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &some_file, 50, &ctx).await,
             "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
         );
 
@@ -1890,7 +1917,7 @@ mod tests {
         asm_file.set(Param::File, "foo.bar");
         asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
         assert_eq!(
-            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &mut asm_file, 50, &ctx).await,
+            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &asm_file, 50, &ctx).await,
             "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
         );
     }
@@ -1967,7 +1994,7 @@ mod tests {
         let chatitems = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         for chatitem in chatitems {
             if let ChatItem::Message { msg_id } = chatitem {
-                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id.clone()).await {
+                if let Ok(msg) = Message::load_from_db(&t.ctx, msg_id).await {
                     if msg.get_viewtype() == Viewtype::Image {
                         has_image = true;
                         // just check that width/height are inside some reasonable ranges

src/mimefactory.rs

@@ -11,6 +11,7 @@ use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
+use crate::format_flowed::format_flowed;
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -236,22 +237,16 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     return true;
                 }
 
-                let force_plaintext = self
+                !self
                     .msg
                     .param
-                    .get_int(Param::ForcePlaintext)
-                    .unwrap_or_default();
-
-                if force_plaintext == 0 {
-                    return self
+                    .get_bool(Param::ForcePlaintext)
+                    .unwrap_or_default()
+                    && self
                         .msg
                         .param
-                        .get_int(Param::GuaranteeE2ee)
+                        .get_bool(Param::GuaranteeE2ee)
                         .unwrap_or_default()
-                        != 0;
-                }
-
-                false
             }
             Loaded::MDN { .. } => false,
         }
@@ -270,19 +265,30 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn should_force_plaintext(&self) -> i32 {
+    fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
                 if chat.typ == Chattype::VerifiedGroup {
-                    0
+                    false
                 } else {
                     self.msg
                         .param
-                        .get_int(Param::ForcePlaintext)
+                        .get_bool(Param::ForcePlaintext)
                         .unwrap_or_default()
                 }
             }
-            Loaded::MDN { .. } => ForcePlaintext::NoAutocryptHeader as i32,
+            Loaded::MDN { .. } => true,
+        }
+    }
+
+    fn should_skip_autocrypt(&self) -> bool {
+        match &self.loaded {
+            Loaded::Message { .. } => self
+                .msg
+                .param
+                .get_bool(Param::SkipAutocrypt)
+                .unwrap_or_default(),
+            Loaded::MDN { .. } => true,
         }
     }
 
@@ -475,6 +481,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let min_verified = self.min_verified();
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
+        let skip_autocrypt = self.should_skip_autocrypt();
         let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(self.context).await?;
@@ -498,7 +505,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => self.render_mdn().await?,
         };
 
-        if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
+        if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
             unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
@@ -509,7 +516,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let peerstates = self.peerstates_for_recipients().await?;
         let should_encrypt =
             encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
-        let is_encrypted = should_encrypt && force_plaintext == 0;
+        let is_encrypted = should_encrypt && !force_plaintext;
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
@@ -910,11 +917,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        let flowed_text = format_flowed(final_text);
+
         let footer = &self.selfstatus;
         let message_text = format!(
             "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            escape_message_footer_marks(final_text),
+            escape_message_footer_marks(&flowed_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -926,7 +935,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         // Message is sent as text/plain, with charset = utf-8
         let main_part = PartBuilder::new()
-            .content_type(&mime::TEXT_PLAIN_UTF_8)
+            .header((
+                "Content-Type".to_string(),
+                "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+            ))
             .body(message_text);
         let mut parts = Vec::new();
 

src/mimeparser.rs

@@ -17,6 +17,7 @@ use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::error::{bail, Result};
 use crate::events::EventType;
+use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
@@ -604,7 +605,7 @@ impl MimeMessage {
                 skip the rest.  (see
                 https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
-                if let Some(first) = mail.subparts.iter().next() {
+                if let Some(first) = mail.subparts.get(0) {
                     any_part_added = self.parse_mime_recursive(context, first).await?;
                 }
             }
@@ -641,7 +642,7 @@ impl MimeMessage {
                             }
                         }
                         Some(_) => {
-                            if let Some(first) = mail.subparts.iter().next() {
+                            if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self.parse_mime_recursive(context, first).await?;
                             }
                         }
@@ -715,6 +716,27 @@ impl MimeMessage {
                             simplify(out, self.has_chat_version())
                         };
 
+                        let is_format_flowed = if let Some(format) = mail.ctype.params.get("format")
+                        {
+                            format.as_str().to_ascii_lowercase() == "flowed"
+                        } else {
+                            false
+                        };
+
+                        let simplified_txt = if mime_type.type_() == mime::TEXT
+                            && mime_type.subtype() == mime::PLAIN
+                            && is_format_flowed
+                        {
+                            let delsp = if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                delsp.as_str().to_ascii_lowercase() == "yes"
+                            } else {
+                                false
+                            };
+                            unformat_flowed(&simplified_txt, delsp)
+                        } else {
+                            simplified_txt
+                        };
+
                         if !simplified_txt.is_empty() {
                             let mut part = Part::default();
                             part.typ = Viewtype::Text;
@@ -1022,6 +1044,28 @@ impl MimeMessage {
             message::handle_ndn(context, failure_report, error).await
         }
     }
+
+    /// Returns timestamp of the parent message.
+    ///
+    /// If there is no parent message or it is not found in the
+    /// database, returns None.
+    pub async fn get_parent_timestamp(&self, context: &Context) -> Result<Option<i64>> {
+        let parent_timestamp = if let Some(field) = self
+            .get(HeaderDef::InReplyTo)
+            .and_then(|msgid| parse_message_id(msgid).ok())
+        {
+            context
+                .sql
+                .query_get_value_result(
+                    "SELECT timestamp FROM msgs WHERE rfc724_mid=?",
+                    paramsv![field],
+                )
+                .await?
+        } else {
+            None
+        };
+        Ok(parent_timestamp)
+    }
 }
 
 async fn update_gossip_peerstates(
@@ -1274,6 +1318,8 @@ where
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use crate::test_utils::*;
 
@@ -1391,6 +1437,39 @@ mod tests {
         assert!(mimeparser.chat_disposition_notification_to.is_none());
     }
 
+    #[async_std::test]
+    async fn test_get_parent_timestamp() {
+        let context = TestContext::new().await;
+        let raw = b"From: foo@example.org\n\
+                    Content-Type: text/plain\n\
+                    Chat-Version: 1.0\n\
+                    In-Reply-To: <Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org>\n\
+                    \n\
+                    Some reply\n\
+                    ";
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(
+            mimeparser.get_parent_timestamp(&context.ctx).await.unwrap(),
+            None
+        );
+        let timestamp = 1570435529;
+        context
+            .ctx
+            .sql
+            .execute(
+                "INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)",
+                paramsv!["Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org", timestamp],
+            )
+            .await
+            .expect("Failed to write to the database");
+        assert_eq!(
+            mimeparser.get_parent_timestamp(&context.ctx).await.unwrap(),
+            Some(timestamp)
+        );
+    }
+
     #[async_std::test]
     async fn test_mimeparser_with_context() {
         let context = TestContext::new().await;
@@ -2029,4 +2108,13 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         let test = parse_message_ids("  < ").unwrap();
         assert!(test.is_empty());
     }
+
+    #[test]
+    fn test_mime_parse_format_flowed() {
+        let mime_type = "text/plain; charset=utf-8; Format=Flowed; DelSp=No"
+            .parse::<mime::Mime>()
+            .unwrap();
+        let format_param = mime_type.get_param("format").unwrap();
+        assert_eq!(format_param.as_str().to_ascii_lowercase(), "flowed");
+    }
 }

src/param.rs

@@ -40,10 +40,12 @@ pub enum Param {
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
     ErroneousE2ee = b'e',
 
-    /// For Messages: force unencrypted message, either `ForcePlaintext::AddAutocryptHeader` (1),
-    /// `ForcePlaintext::NoAutocryptHeader` (2) or 0.
+    /// For Messages: force unencrypted message, a value from `ForcePlaintext` enum.
     ForcePlaintext = b'u',
 
+    /// For Messages: do not include Autocrypt header.
+    SkipAutocrypt = b'o',
+
     /// For Messages
     WantsMdn = b'r',
 
@@ -95,6 +97,12 @@ pub enum Param {
     Recipients = b'R',
 
     /// For Groups
+    ///
+    /// An unpromoted group has not had any messages sent to it and thus only exists on the
+    /// creator's device.  Any changes made to an unpromoted group do not need to send
+    /// system messages to the group members to update them of the changes.  Once a message
+    /// has been sent to a group it is promoted and group changes require sending system
+    /// messages to all members.
     Unpromoted = b'U',
 
     /// For Groups and Contacts
@@ -122,14 +130,6 @@ pub enum Param {
     MsgId = b'I',
 }
 
-/// Possible values for `Param::ForcePlaintext`.
-#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive)]
-#[repr(u8)]
-pub enum ForcePlaintext {
-    AddAutocryptHeader = 1,
-    NoAutocryptHeader = 2,
-}
-
 /// An object for handling key=value parameter lists.
 ///
 /// The structure is serialized by calling `to_string()` on it.
@@ -469,8 +469,8 @@ mod tests {
         );
 
         // Blob in blobdir, expect blob.
-        let bar = t.ctx.get_blobdir().join("bar");
-        p.set(Param::File, bar.to_str().unwrap());
+        let bar_path = t.ctx.get_blobdir().join("bar");
+        p.set(Param::File, bar_path.to_str().unwrap());
         let blob = p
             .get_blob(Param::File, &t.ctx, false)
             .await

src/peerstate.rs

@@ -94,47 +94,42 @@ pub enum ToSave {
 }
 
 impl<'a> Peerstate<'a> {
-    pub fn new(context: &'a Context, addr: String) -> Self {
+    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
         Peerstate {
             context,
-            addr,
-            last_seen: 0,
-            last_seen_autocrypt: 0,
-            prefer_encrypt: Default::default(),
-            public_key: None,
-            public_key_fingerprint: None,
+            addr: header.addr.clone(),
+            last_seen: message_time,
+            last_seen_autocrypt: message_time,
+            prefer_encrypt: header.prefer_encrypt,
+            public_key: Some(header.public_key.clone()),
+            public_key_fingerprint: Some(header.public_key.fingerprint()),
             gossip_key: None,
             gossip_key_fingerprint: None,
             gossip_timestamp: 0,
             verified_key: None,
             verified_key_fingerprint: None,
-            to_save: None,
+            to_save: Some(ToSave::All),
             fingerprint_changed: false,
         }
     }
 
-    pub fn from_header(context: &'a Context, header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, header.addr.clone());
-
-        res.last_seen = message_time;
-        res.last_seen_autocrypt = message_time;
-        res.to_save = Some(ToSave::All);
-        res.prefer_encrypt = header.prefer_encrypt;
-        res.public_key = Some(header.public_key.clone());
-        res.recalc_fingerprint();
-
-        res
-    }
-
     pub fn from_gossip(context: &'a Context, gossip_header: &Aheader, message_time: i64) -> Self {
-        let mut res = Self::new(context, gossip_header.addr.clone());
-
-        res.gossip_timestamp = message_time;
-        res.to_save = Some(ToSave::All);
-        res.gossip_key = Some(gossip_header.public_key.clone());
-        res.recalc_fingerprint();
-
-        res
+        Peerstate {
+            context,
+            addr: gossip_header.addr.clone(),
+            last_seen: 0,
+            last_seen_autocrypt: 0,
+            prefer_encrypt: Default::default(),
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: Some(gossip_header.public_key.clone()),
+            gossip_key_fingerprint: Some(gossip_header.public_key.fingerprint()),
+            gossip_timestamp: message_time,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        }
     }
 
     pub async fn from_addr(context: &'a Context, addr: &str) -> Result<Option<Peerstate<'a>>> {
@@ -175,40 +170,44 @@ impl<'a> Peerstate<'a> {
                 public_key, gossip_timestamp, gossip_key, public_key_fingerprint,
                 gossip_key_fingerprint, verified_key, verified_key_fingerprint
                 */
-                let mut res = Self::new(context, row.get(0)?);
 
-                res.last_seen = row.get(1)?;
-                res.last_seen_autocrypt = row.get(2)?;
-                res.prefer_encrypt = EncryptPreference::from_i32(row.get(3)?).unwrap_or_default();
-                res.gossip_timestamp = row.get(5)?;
-
-                res.public_key_fingerprint = row
-                    .get::<_, Option<String>>(7)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()
-                    .unwrap_or_default();
-                res.gossip_key_fingerprint = row
-                    .get::<_, Option<String>>(8)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()
-                    .unwrap_or_default();
-                res.verified_key_fingerprint = row
-                    .get::<_, Option<String>>(10)?
-                    .map(|s| s.parse::<Fingerprint>())
-                    .transpose()
-                    .unwrap_or_default();
-                res.public_key = row
-                    .get(4)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
-                res.gossip_key = row
-                    .get(6)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
-                res.verified_key = row
-                    .get(9)
-                    .ok()
-                    .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok());
+                let res = Peerstate {
+                    context,
+                    addr: row.get(0)?,
+                    last_seen: row.get(1)?,
+                    last_seen_autocrypt: row.get(2)?,
+                    prefer_encrypt: EncryptPreference::from_i32(row.get(3)?).unwrap_or_default(),
+                    public_key: row
+                        .get(4)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    public_key_fingerprint: row
+                        .get::<_, Option<String>>(7)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    gossip_key: row
+                        .get(6)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    gossip_key_fingerprint: row
+                        .get::<_, Option<String>>(8)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    gossip_timestamp: row.get(5)?,
+                    verified_key: row
+                        .get(9)
+                        .ok()
+                        .and_then(|blob: Vec<u8>| SignedPublicKey::from_slice(&blob).ok()),
+                    verified_key_fingerprint: row
+                        .get::<_, Option<String>>(10)?
+                        .map(|s| s.parse::<Fingerprint>())
+                        .transpose()
+                        .unwrap_or_default(),
+                    to_save: None,
+                    fingerprint_changed: false,
+                };
 
                 Ok(res)
             })
@@ -291,7 +290,7 @@ impl<'a> Peerstate<'a> {
             return;
         }
 
-        if message_time > self.last_seen_autocrypt {
+        if message_time > self.last_seen {
             self.last_seen = message_time;
             self.last_seen_autocrypt = message_time;
             self.to_save = Some(ToSave::Timestamps);
@@ -490,7 +489,6 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
     use pretty_assertions::assert_eq;
-    use tempfile::TempDir;
 
     #[async_std::test]
     async fn test_peerstate_save_to_db() {
@@ -637,10 +635,44 @@ mod tests {
         assert_eq!(peerstate.verified_key_fingerprint, None);
     }
 
-    // TODO: don't copy this from stress.rs
-    #[allow(dead_code)]
-    struct TestContext {
-        ctx: Context,
-        dir: TempDir,
+    #[async_std::test]
+    async fn test_peerstate_degrade_reordering() {
+        let context = crate::test_utils::TestContext::new().await.ctx;
+        let addr = "example@example.org";
+        let pub_key = alice_keypair().public;
+        let header = Aheader::new(addr.to_string(), pub_key, EncryptPreference::Mutual);
+
+        let mut peerstate = Peerstate {
+            context: &context,
+            addr: addr.to_string(),
+            last_seen: 0,
+            last_seen_autocrypt: 0,
+            prefer_encrypt: EncryptPreference::NoPreference,
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: None,
+            gossip_timestamp: 0,
+            gossip_key_fingerprint: None,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: None,
+            fingerprint_changed: false,
+        };
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::NoPreference);
+
+        peerstate.apply_header(&header, 100);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
+
+        peerstate.degrade_encryption(300);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+        // This has message time 200, while encryption was degraded at timestamp 300.
+        // Because of reordering, header should not be applied.
+        peerstate.apply_header(&header, 200);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
+
+        // Same header will be applied in the future.
+        peerstate.apply_header(&header, 400);
+        assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
     }
 }

src/pgp.rs

@@ -439,9 +439,9 @@ mod tests {
             let bob = bob_keypair();
             TestKeys {
                 alice_secret: alice.secret.clone(),
-                alice_public: alice.public.clone(),
+                alice_public: alice.public,
                 bob_secret: bob.secret.clone(),
-                bob_public: bob.public.clone(),
+                bob_public: bob.public,
             }
         }
     }

src/provider/mod.rs

@@ -6,7 +6,7 @@ use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
-#[derive(Debug, Copy, Clone, PartialEq, ToPrimitive)]
+#[derive(Debug, Display, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Status {
     OK = 1,
@@ -14,21 +14,29 @@ pub enum Status {
     BROKEN = 3,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Display, PartialEq, Copy, Clone, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Protocol {
     SMTP = 1,
     IMAP = 2,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Display, PartialEq, Copy, Clone, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum Socket {
-    STARTTLS = 1,
-    SSL = 2,
+    Automatic = 0,
+    SSL = 1,
+    STARTTLS = 2,
+    Plain = 3,
 }
 
-#[derive(Debug, PartialEq)]
+impl Default for Socket {
+    fn default() -> Self {
+        Socket::Automatic
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
 #[repr(u8)]
 pub enum UsernamePattern {
     EMAIL = 1,
@@ -42,7 +50,7 @@ pub enum Oauth2Authorizer {
     Gmail = 2,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Server {
     pub protocol: Protocol,
     pub socket: Socket,
@@ -51,20 +59,6 @@ pub struct Server {
     pub username_pattern: UsernamePattern,
 }
 
-impl Server {
-    pub fn apply_username_pattern(&self, addr: String) -> String {
-        match self.username_pattern {
-            UsernamePattern::EMAIL => addr,
-            UsernamePattern::EMAILLOCALPART => {
-                if let Some(at) = addr.find('@') {
-                    return addr.split_at(at).0.to_string();
-                }
-                addr
-            }
-        }
-    }
-}
-
 #[derive(Debug)]
 pub struct ConfigDefault {
     pub key: Config,
@@ -83,25 +77,6 @@ pub struct Provider {
     pub oauth2_authorizer: Option<Oauth2Authorizer>,
 }
 
-impl Provider {
-    pub fn get_server(&self, protocol: Protocol) -> Option<&Server> {
-        for record in self.server.iter() {
-            if record.protocol == protocol {
-                return Some(record);
-            }
-        }
-        None
-    }
-
-    pub fn get_imap_server(&self) -> Option<&Server> {
-        self.get_server(Protocol::IMAP)
-    }
-
-    pub fn get_smtp_server(&self) -> Option<&Server> {
-        self.get_server(Protocol::SMTP)
-    }
-}
-
 pub fn get_provider_info(addr: &str) -> Option<&Provider> {
     let domain = match addr.parse::<EmailAddress>() {
         Ok(addr) => addr.domain,
@@ -118,6 +93,8 @@ pub fn get_provider_info(addr: &str) -> Option<&Provider> {
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
@@ -137,15 +114,16 @@ mod tests {
         let provider = get_provider_info("nauta.cu"); // this is no email address
         assert!(provider.is_none());
 
-        let provider = get_provider_info("user@nauta.cu").unwrap();
+        let addr = "user@nauta.cu";
+        let provider = get_provider_info(addr).unwrap();
         assert!(provider.status == Status::OK);
-        let server = provider.get_imap_server().unwrap();
+        let server = &provider.server[0];
         assert_eq!(server.protocol, Protocol::IMAP);
         assert_eq!(server.socket, Socket::STARTTLS);
         assert_eq!(server.hostname, "imap.nauta.cu");
         assert_eq!(server.port, 143);
         assert_eq!(server.username_pattern, UsernamePattern::EMAIL);
-        let server = provider.get_smtp_server().unwrap();
+        let server = &provider.server[1];
         assert_eq!(server.protocol, Protocol::SMTP);
         assert_eq!(server.socket, Socket::STARTTLS);
         assert_eq!(server.hostname, "smtp.nauta.cu");

src/securejoin.rs

@@ -1,7 +1,8 @@
 //! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
 
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
+use anyhow::{bail, Error};
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
@@ -11,16 +12,16 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
-use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::lot::LotState;
+use crate::lot::{Lot, LotState};
 use crate::message::Message;
 use crate::mimeparser::*;
 use crate::param::*;
 use crate::peerstate::*;
 use crate::qr::check_qr;
+use crate::sql;
 use crate::stock::StockMessage;
 use crate::token;
 
@@ -67,6 +68,41 @@ macro_rules! get_qr_attr {
     };
 }
 
+/// State for setup-contact/secure-join protocol joiner's side.
+///
+/// The setup-contact protocol needs to carry state for both the inviter (Alice) and the
+/// joiner/invitee (Bob).  For Alice this state is minimal and in the `tokens` table in the
+/// database.  For Bob this state is only carried live on the [Context] in this struct.
+#[derive(Debug, Default)]
+pub(crate) struct Bob {
+    /// The next message expected by the protocol.
+    expects: SecureJoinStep,
+    /// The QR-scanned information of the currently running protocol.
+    pub qr_scan: Option<Lot>,
+}
+
+/// The next message expected by [Bob] in the setup-contact/secure-join protocol.
+#[derive(Debug, PartialEq)]
+enum SecureJoinStep {
+    /// No setup-contact protocol running.
+    NotActive,
+    /// Expecting the auth-required message.
+    ///
+    /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
+    AuthRequired,
+    /// Expecting the contact-confirm message.
+    ///
+    /// This corresponds to the `vc-contact-confirm` or `vg-member-added` message of step
+    /// 6b.
+    ContactConfirm,
+}
+
+impl Default for SecureJoinStep {
+    fn default() -> Self {
+        Self::NotActive
+    }
+}
+
 pub async fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<String> {
     /*=======================================================
     ====             Alice - the inviter side            ====
@@ -152,78 +188,79 @@ async fn get_self_fingerprint(context: &Context) -> Option<Fingerprint> {
     }
 }
 
-async fn cleanup(
-    context: &Context,
-    contact_chat_id: ChatId,
-    ongoing_allocated: bool,
-    join_vg: bool,
-) -> ChatId {
+async fn cleanup(context: &Context, ongoing_allocated: bool) {
     let mut bob = context.bob.write().await;
-    bob.expects = 0;
-    let ret_chat_id: ChatId = if bob.status == DC_BOB_SUCCESS {
-        if join_vg {
-            chat::get_chat_id_by_grpid(
-                context,
-                bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap(),
-            )
-            .await
-            .unwrap_or((ChatId::new(0), false, Blocked::Not))
-            .0
-        } else {
-            contact_chat_id
-        }
-    } else {
-        ChatId::new(0)
-    };
+    bob.expects = SecureJoinStep::NotActive;
     bob.qr_scan = None;
-
     if ongoing_allocated {
         context.free_ongoing().await;
     }
-    ret_chat_id
 }
 
-/// Take a scanned QR-code and do the setup-contact/join-group handshake.
-/// See the ffi-documentation for more details.
-pub async fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
+#[derive(Debug, thiserror::Error)]
+pub enum JoinError {
+    #[error("Unknown QR-code")]
+    QrCode,
+    #[error("Aborted by user")]
+    Aborted,
+    #[error("Failed to send handshake message")]
+    SendMessage(#[from] SendMsgError),
+    // Note that this can currently only occur if there is a bug in the QR/Lot code as this
+    // is supposed to create a contact for us.
+    #[error("Unknown contact (this is a bug)")]
+    UnknownContact,
+    // Note that this can only occur if we failed to create the chat correctly.
+    #[error("No Chat found for group (this is a bug)")]
+    MissingChat(#[source] sql::Error),
+}
+
+/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake.
+///
+/// This is the start of the process for the joiner.  See the module and ffi documentation
+/// for more details.
+///
+/// When joining a group this will start an "ongoing" process and will block until the
+/// process is completed, the [ChatId] for the new group is not known any sooner.  When
+/// verifying a contact this returns immediately.
+pub async fn dc_join_securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     if context.alloc_ongoing().await.is_err() {
-        return cleanup(&context, ChatId::new(0), false, false).await;
+        cleanup(&context, false).await;
+        return Err(JoinError::Aborted);
     }
 
     securejoin(context, qr).await
 }
 
-async fn securejoin(context: &Context, qr: &str) -> ChatId {
+async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     /*========================================================
     ====             Bob - the joiner's side             =====
     ====   Step 2 in "Setup verified contact" protocol   =====
     ========================================================*/
 
-    let mut contact_chat_id = ChatId::new(0);
-    let mut join_vg: bool = false;
-
     info!(context, "Requesting secure-join ...",);
     ensure_secret_key_exists(context).await.ok();
     let qr_scan = check_qr(context, &qr).await;
     if qr_scan.state != LotState::QrAskVerifyContact && qr_scan.state != LotState::QrAskVerifyGroup
     {
         error!(context, "Unknown QR code.",);
-        return cleanup(&context, contact_chat_id, true, join_vg).await;
+        cleanup(&context, true).await;
+        return Err(JoinError::QrCode);
     }
-    contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
+    let contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id).await {
         Ok(chat_id) => chat_id,
         Err(_) => {
             error!(context, "Unknown contact.");
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::UnknownContact);
         }
     };
     if context.shall_stop_ongoing().await {
-        return cleanup(&context, contact_chat_id, true, join_vg).await;
+        cleanup(&context, true).await;
+        return Err(JoinError::Aborted);
     }
-    join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
+    let join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
     {
         let mut bob = context.bob.write().await;
-        bob.status = 0;
         bob.qr_scan = Some(qr_scan);
     }
     if fingerprint_equals_sender(
@@ -245,7 +282,7 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         // the scanned fingerprint matches Alice's key,
         // we can proceed to step 4b) directly and save two mails
         info!(context, "Taking protocol shortcut.");
-        context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+        context.bob.write().await.expects = SecureJoinStep::ContactConfirm;
         joiner_progress!(
             context,
             chat_id_2_contact_id(context, contact_chat_id).await,
@@ -273,10 +310,11 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         .await
         {
             error!(context, "failed to send handshake message: {}", err);
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::SendMessage(err));
         }
     } else {
-        context.bob.write().await.expects = DC_VC_AUTH_REQUIRED;
+        context.bob.write().await.expects = SecureJoinStep::AuthRequired;
 
         // Bob -> Alice
         if let Err(err) = send_handshake_msg(
@@ -290,7 +328,8 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         .await
         {
             error!(context, "failed to send handshake message: {}", err);
-            return cleanup(&context, contact_chat_id, true, join_vg).await;
+            cleanup(&context, true).await;
+            return Err(JoinError::SendMessage(err));
         }
     }
 
@@ -299,15 +338,45 @@ async fn securejoin(context: &Context, qr: &str) -> ChatId {
         while !context.shall_stop_ongoing().await {
             async_std::task::sleep(Duration::from_millis(50)).await;
         }
-        cleanup(&context, contact_chat_id, true, join_vg).await
+
+        // handle_securejoin_handshake() calls Context::stop_ongoing before the group chat
+        // is created (it is created after handle_securejoin_handshake() returns by
+        // dc_receive_imf()).  As a hack we just wait a bit for it to appear.
+        let start = Instant::now();
+        let chatid = loop {
+            {
+                let bob = context.bob.read().await;
+                let grpid = bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap();
+                match chat::get_chat_id_by_grpid(context, grpid).await {
+                    Ok((chatid, _is_verified, _blocked)) => break chatid,
+                    Err(err) => {
+                        if start.elapsed() > Duration::from_secs(7) {
+                            return Err(JoinError::MissingChat(err));
+                        }
+                    }
+                }
+            }
+            async_std::task::sleep(Duration::from_millis(50)).await;
+        };
+
+        cleanup(&context, true).await;
+        Ok(chatid)
     } else {
         // for a one-to-one-chat, the chat is already known, return the chat-id,
         // the verification runs in background
         context.free_ongoing().await;
-        contact_chat_id
+        Ok(contact_chat_id)
     }
 }
 
+/// Error for [send_handshake_msg].
+///
+/// Wrapping the [anyhow::Error] means we can "impl From" more easily on errors from this
+/// function.
+#[derive(Debug, thiserror::Error)]
+#[error("Failed sending handshake message")]
+pub struct SendMsgError(#[from] anyhow::Error);
+
 async fn send_handshake_msg(
     context: &Context,
     contact_chat_id: ChatId,
@@ -315,7 +384,7 @@ async fn send_handshake_msg(
     param2: impl AsRef<str>,
     fingerprint: Option<Fingerprint>,
     grpid: impl AsRef<str>,
-) -> Result<(), HandshakeError> {
+) -> Result<(), SendMsgError> {
     let mut msg = Message::default();
     msg.viewtype = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
@@ -336,18 +405,12 @@ async fn send_handshake_msg(
         msg.param.set(Param::Arg4, grpid.as_ref());
     }
     if step == "vg-request" || step == "vc-request" {
-        msg.param.set_int(
-            Param::ForcePlaintext,
-            ForcePlaintext::AddAutocryptHeader as i32,
-        );
+        msg.param.set_int(Param::ForcePlaintext, 1);
     } else {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
 
-    chat::send_msg(context, contact_chat_id, &mut msg)
-        .await
-        .map_err(HandshakeError::MsgSendFailed)?;
-
+    chat::send_msg(context, contact_chat_id, &mut msg).await?;
     Ok(())
 }
 
@@ -407,7 +470,7 @@ pub(crate) enum HandshakeError {
     #[error("No configured self address found")]
     NoSelfAddr,
     #[error("Failed to send message")]
-    MsgSendFailed(#[source] Error),
+    MsgSendFailed(#[from] SendMsgError),
     #[error("Failed to parse fingerprint")]
     BadFingerprint(#[from] crate::key::FingerprintError),
 }
@@ -521,7 +584,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 let bob = context.bob.read().await;
                 let scan = bob.qr_scan.as_ref();
                 scan.is_none()
-                    || bob.expects != DC_VC_AUTH_REQUIRED
+                    || bob.expects != SecureJoinStep::AuthRequired
                     || join_vg && scan.unwrap().state != LotState::QrAskVerifyGroup
             };
 
@@ -545,7 +608,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     },
                 )
                 .await;
-                context.bob.write().await.status = 0; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
@@ -558,14 +620,13 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Fingerprint mismatch on joiner-side.",
                 )
                 .await;
-                context.bob.write().await.status = 0; // secure-join failed
                 context.stop_ongoing().await;
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
             let own_fingerprint = get_self_fingerprint(context).await.unwrap();
             joiner_progress!(context, contact_id, 400);
-            context.bob.write().await.expects = DC_VC_CONTACT_CONFIRM;
+            context.bob.write().await.expects = SecureJoinStep::ContactConfirm;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -709,7 +770,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 HandshakeMessage::Ignore
             };
 
-            if context.bob.read().await.expects != DC_VC_CONTACT_CONFIRM {
+            if context.bob.read().await.expects != SecureJoinStep::ContactConfirm {
                 info!(context, "Message belongs to a different handshake.",);
                 return Ok(abort_retval);
             }
@@ -756,7 +817,6 @@ pub(crate) async fn handle_securejoin_handshake(
                     "Contact confirm message not encrypted.",
                 )
                 .await;
-                context.bob.write().await.status = 0;
                 return Ok(abort_retval);
             }
 
@@ -788,7 +848,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 return Ok(abort_retval);
             }
             secure_connection_established(context, contact_chat_id).await;
-            context.bob.write().await.expects = 0;
+            context.bob.write().await.expects = SecureJoinStep::NotActive;
 
             // Bob -> Alice
             send_handshake_msg(
@@ -805,7 +865,6 @@ pub(crate) async fn handle_securejoin_handshake(
             )
             .await?;
 
-            context.bob.write().await.status = 1;
             context.stop_ongoing().await;
             Ok(if join_vg {
                 HandshakeMessage::Propagate
@@ -825,6 +884,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     return Ok(HandshakeMessage::Ignore);
                 }
                 if join_vg {
+                    // Responsible for showing "$Bob securely joined $group" message
                     inviter_progress!(context, contact_id, 800);
                     inviter_progress!(context, contact_id, 1000);
                     let field_grpid = mime_message
@@ -1036,3 +1096,337 @@ fn encrypted_and_signed(
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::chat;
+    use crate::peerstate::Peerstate;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_setup_contact() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Generate QR-code, ChatId(0) indicates setup-contact
+        let qr = dc_get_securejoin_qr(&alice.ctx, ChatId::new(0))
+            .await
+            .unwrap();
+
+        // Bob scans QR-code, sends vc-request
+        let bob_chatid = dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        let sent = bob.pop_sent_msg().await;
+        assert_eq!(sent.id(), bob_chatid);
+        assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
+        let msg = alice.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
+
+        // Alice receives vc-request, sends vc-auth-required
+        alice.recv_msg(&sent).await;
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-auth-required");
+
+        // Bob receives vc-auth-required, sends vc-request-with-auth
+        bob.recv_msg(&sent).await;
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown).await;
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vc-request-with-auth, sends vc-contact-confirm
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm"
+        );
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vc-contact-confirm, sends vc-contact-confirm-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm-received"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_setup_contact_bad_qr() {
+        let bob = TestContext::new_bob().await;
+        let ret = dc_join_securejoin(&bob.ctx, "not a qr code").await;
+        assert!(matches!(ret, Err(JoinError::QrCode)));
+    }
+
+    #[async_std::test]
+    async fn test_setup_contact_bob_knows_alice() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Ensure Bob knows Alice_FP
+        let alice_pubkey = SignedPublicKey::load_self(&alice.ctx).await.unwrap();
+        let peerstate = Peerstate {
+            context: &bob.ctx,
+            addr: "alice@example.com".into(),
+            last_seen: 10,
+            last_seen_autocrypt: 10,
+            prefer_encrypt: EncryptPreference::Mutual,
+            public_key: Some(alice_pubkey.clone()),
+            public_key_fingerprint: Some(alice_pubkey.fingerprint()),
+            gossip_key: Some(alice_pubkey.clone()),
+            gossip_timestamp: 10,
+            gossip_key_fingerprint: Some(alice_pubkey.fingerprint()),
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        peerstate.save_to_db(&bob.ctx.sql, true).await.unwrap();
+
+        // Generate QR-code, ChatId(0) indicates setup-contact
+        let qr = dc_get_securejoin_qr(&alice.ctx, ChatId::new(0))
+            .await
+            .unwrap();
+
+        // Bob scans QR-code, sends vc-request-with-auth, skipping vc-request
+        dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await
+        .unwrap();
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vc-request-with-auth, sends vc-contact-confirm
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm"
+        );
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vc-contact-confirm, sends vc-contact-confirm-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vc-contact-confirm-received"
+        );
+    }
+
+    #[async_std::test]
+    async fn test_secure_join() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chatid = chat::create_group_chat(&alice.ctx, VerifiedStatus::Verified, "the chat")
+            .await
+            .unwrap();
+
+        // Generate QR-code, secure-join implied by chatid
+        let qr = dc_get_securejoin_qr(&alice.ctx, chatid).await.unwrap();
+
+        // Bob scans QR-code, sends vg-request; blocks on ongoing process
+        let joiner = {
+            let qr = qr.clone();
+            let ctx = bob.ctx.clone();
+            async_std::task::spawn(async move { dc_join_securejoin(&ctx, &qr).await.unwrap() })
+        };
+
+        let sent = bob.pop_sent_msg().await;
+        assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
+        let msg = alice.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
+
+        // Alice receives vg-request, sends vg-auth-required
+        alice.recv_msg(&sent).await;
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-auth-required");
+
+        // Bob receives vg-auth-required, sends vg-request-with-auth
+        bob.recv_msg(&sent).await;
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vg-request-with-auth"
+        );
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
+        let bob_fp = SignedPublicKey::load_self(&bob.ctx)
+            .await
+            .unwrap()
+            .fingerprint();
+        assert_eq!(
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
+            bob_fp.hex()
+        );
+
+        // Alice should not yet have Bob verified
+        let contact_bob_id =
+            Contact::lookup_id_by_addr(&alice.ctx, "bob@example.net", Origin::Unknown).await;
+        let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Alice receives vg-request-with-auth, sends vg-member-added
+        alice.recv_msg(&sent).await;
+        assert_eq!(
+            contact_bob.is_verified(&alice.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = alice.pop_sent_msg().await;
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-member-added");
+
+        // Bob should not yet have Alice verified
+        let contact_alice_id =
+            Contact::lookup_id_by_addr(&bob.ctx, "alice@example.com", Origin::Unknown).await;
+        let contact_alice = Contact::load_from_db(&bob.ctx, contact_alice_id)
+            .await
+            .unwrap();
+        assert_eq!(
+            contact_bob.is_verified(&bob.ctx).await,
+            VerifiedStatus::Unverified
+        );
+
+        // Bob receives vg-member-added, sends vg-member-added-received
+        bob.recv_msg(&sent).await;
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await,
+            VerifiedStatus::BidirectVerified
+        );
+
+        let sent = bob.pop_sent_msg().await;
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        assert_eq!(
+            msg.get(HeaderDef::SecureJoin).unwrap(),
+            "vg-member-added-received"
+        );
+
+        let bob_chatid = joiner.await;
+        let bob_chat = Chat::load_from_db(&bob.ctx, bob_chatid).await.unwrap();
+        assert!(bob_chat.is_verified());
+    }
+}

src/smtp/mod.rs

@@ -10,9 +10,9 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::EventType;
-use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
 use crate::oauth2::*;
-use crate::provider::get_provider_info;
+use crate::provider::{get_provider_info, Socket};
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -94,31 +94,65 @@ impl Smtp {
             .unwrap_or_default()
     }
 
+    /// Connect using configured parameters.
+    pub async fn connect_configured(&mut self, context: &Context) -> Result<()> {
+        if self.is_connected().await {
+            return Ok(());
+        }
+
+        let lp = LoginParam::from_database(context, "configured_").await;
+        let res = self
+            .connect(
+                context,
+                &lp.smtp,
+                &lp.addr,
+                lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+            )
+            .await;
+        if let Err(ref err) = res {
+            let message = context
+                .stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                    err.to_string(),
+                )
+                .await;
+
+            context.emit_event(EventType::ErrorNetwork(message));
+        };
+        res
+    }
+
     /// Connect using the provided login params.
-    pub async fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
+    pub async fn connect(
+        &mut self,
+        context: &Context,
+        lp: &ServerLoginParam,
+        addr: &str,
+        oauth2: bool,
+    ) -> Result<()> {
         if self.is_connected().await {
             warn!(context, "SMTP already connected.");
             return Ok(());
         }
 
-        if lp.send_server.is_empty() || lp.send_port == 0 {
-            context.emit_event(EventType::ErrorNetwork("SMTP bad parameters.".into()));
+        if lp.server.is_empty() || lp.port == 0 {
             return Err(Error::BadParameters);
         }
 
         let from =
-            EmailAddress::new(lp.addr.clone()).map_err(|err| Error::InvalidLoginAddress {
-                address: lp.addr.clone(),
+            EmailAddress::new(addr.to_string()).map_err(|err| Error::InvalidLoginAddress {
+                address: addr.to_string(),
                 error: err,
             })?;
 
         self.from = Some(from);
 
-        let domain = &lp.send_server;
-        let port = lp.send_port as u16;
+        let domain = &lp.server;
+        let port = lp.port;
 
-        let provider = get_provider_info(&lp.addr);
-        let strict_tls = match lp.smtp_certificate_checks {
+        let provider = get_provider_info(addr);
+        let strict_tls = match lp.certificate_checks {
             CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
             CertificateChecks::Strict => true,
             CertificateChecks::AcceptInvalidCertificates
@@ -127,17 +161,16 @@ impl Smtp {
         let tls_config = dc_build_tls(strict_tls);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
-        let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
+        let (creds, mechanism) = if oauth2 {
             // oauth2
-            let addr = &lp.addr;
-            let send_pw = &lp.send_pw;
+            let send_pw = &lp.password;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false).await;
             if access_token.is_none() {
                 return Err(Error::Oauth2Error {
                     address: addr.to_string(),
                 });
             }
-            let user = &lp.send_user;
+            let user = &lp.user;
             (
                 smtp::authentication::Credentials::new(
                     user.to_string(),
@@ -147,8 +180,8 @@ impl Smtp {
             )
         } else {
             // plain
-            let user = lp.send_user.clone();
-            let pw = lp.send_pw.clone();
+            let user = lp.user.clone();
+            let pw = lp.password.clone();
             (
                 smtp::authentication::Credentials::new(user, pw),
                 vec![
@@ -158,12 +191,9 @@ impl Smtp {
             )
         };
 
-        let security = if 0
-            != lp.server_flags & (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_PLAIN) as i32
-        {
-            smtp::ClientSecurity::Opportunistic(tls_parameters)
-        } else {
-            smtp::ClientSecurity::Wrapper(tls_parameters)
+        let security = match lp.security {
+            Socket::STARTTLS | Socket::Plain => smtp::ClientSecurity::Opportunistic(tls_parameters),
+            _ => smtp::ClientSecurity::Wrapper(tls_parameters),
         };
 
         let client = smtp::SmtpClient::with_security((domain.as_str(), port), security)
@@ -179,15 +209,6 @@ impl Smtp {
 
         let mut trans = client.into_transport();
         if let Err(err) = trans.connect().await {
-            let message = context
-                .stock_string_repl_str2(
-                    StockMessage::ServerResponse,
-                    format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
-                )
-                .await;
-
-            emit_event!(context, EventType::ErrorNetwork(message));
             return Err(Error::ConnectionFailure(err));
         }
 
@@ -196,7 +217,7 @@ impl Smtp {
 
         context.emit_event(EventType::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
-            lp.send_user,
+            lp.user,
         )));
 
         Ok(())

src/smtp/send.rs

@@ -5,6 +5,7 @@ use async_smtp::*;
 
 use crate::context::Context;
 use crate::events::EventType;
+use itertools::Itertools;
 use std::time::Duration;
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -33,11 +34,7 @@ impl Smtp {
     ) -> Result<()> {
         let message_len_bytes = message.len();
 
-        let recipients_display = recipients
-            .iter()
-            .map(|x| format!("{}", x))
-            .collect::<Vec<String>>()
-            .join(",");
+        let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
 
         let envelope =
             Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;

src/sql.rs

@@ -1288,6 +1288,43 @@ async fn open(
             update_icons = true;
             sql.set_raw_config_int(context, "dbversion", 66).await?;
         }
+        if dbversion < 67 {
+            info!(context, "[migration] v67");
+            for prefix in &["", "configured_"] {
+                if let Some(server_flags) = sql
+                    .get_raw_config_int(context, format!("{}server_flags", prefix))
+                    .await
+                {
+                    let imap_socket_flags = server_flags & 0x700;
+                    let key = format!("{}mail_security", prefix);
+                    match imap_socket_flags {
+                        0x100 => sql.set_raw_config_int(context, key, 2).await?, // STARTTLS
+                        0x200 => sql.set_raw_config_int(context, key, 1).await?, // SSL/TLS
+                        0x400 => sql.set_raw_config_int(context, key, 3).await?, // Plain
+                        _ => sql.set_raw_config_int(context, key, 0).await?,
+                    }
+                    let smtp_socket_flags = server_flags & 0x70000;
+                    let key = format!("{}send_security", prefix);
+                    match smtp_socket_flags {
+                        0x10000 => sql.set_raw_config_int(context, key, 2).await?, // STARTTLS
+                        0x20000 => sql.set_raw_config_int(context, key, 1).await?, // SSL/TLS
+                        0x40000 => sql.set_raw_config_int(context, key, 3).await?, // Plain
+                        _ => sql.set_raw_config_int(context, key, 0).await?,
+                    }
+                }
+            }
+            sql.set_raw_config_int(context, "dbversion", 67).await?;
+        }
+        if dbversion < 68 {
+            info!(context, "[migration] v68");
+            // the index is used to speed up get_fresh_msg_cnt(), see comment there for more details
+            sql.execute(
+                "CREATE INDEX IF NOT EXISTS msgs_index7 ON msgs (state, hidden, chat_id);",
+                paramsv![],
+            )
+            .await?;
+            sql.set_raw_config_int(context, "dbversion", 68).await?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)

src/stock.rs

@@ -216,6 +216,9 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "You are invited to a video chat, click %1$s to join."))]
     VideochatInviteMsgBody = 83,
+
+    #[strum(props(fallback = "Configuration failed. Error: “%1$s”"))]
+    ConfigurationFailed = 84,
 }
 
 /*

src/test_utils.rs

@@ -2,20 +2,33 @@
 //!
 //! This module is only compiled for test runs.
 
+use std::str::FromStr;
+use std::time::{Duration, Instant};
+
+use async_std::path::PathBuf;
+use async_std::sync::RwLock;
 use tempfile::{tempdir, TempDir};
 
+use crate::chat::ChatId;
 use crate::config::Config;
 use crate::context::Context;
+use crate::dc_receive_imf::dc_receive_imf;
 use crate::dc_tools::EmailAddress;
+use crate::job::Action;
 use crate::key::{self, DcKey};
+use crate::mimeparser::MimeMessage;
+use crate::param::{Param, Params};
 
 /// A Context and temporary directory.
 ///
 /// The temporary directory can be used to store the SQLite database,
 /// see e.g. [test_context] which does this.
+#[derive(Debug)]
 pub(crate) struct TestContext {
     pub ctx: Context,
     pub dir: TempDir,
+    /// Counter for fake IMAP UIDs in [recv_msg], for private use in that function only.
+    recv_idx: RwLock<u32>,
 }
 
 impl TestContext {
@@ -35,7 +48,11 @@ impl TestContext {
         let ctx = Context::new("FakeOS".into(), dbfile.into(), id)
             .await
             .unwrap();
-        Self { ctx, dir }
+        Self {
+            ctx,
+            dir,
+            recv_idx: RwLock::new(0),
+        }
     }
 
     /// Create a new configured [TestContext].
@@ -48,6 +65,19 @@ impl TestContext {
         t
     }
 
+    /// Create a new configured [TestContext].
+    ///
+    /// This is a shortcut which configures bob@example.net with a fixed key.
+    pub async fn new_bob() -> Self {
+        let t = Self::new().await;
+        let keypair = bob_keypair();
+        t.configure_addr(&keypair.addr.to_string()).await;
+        key::store_self_keypair(&t.ctx, &keypair, key::KeyPairUse::Default)
+            .await
+            .expect("Failed to save Bob's key");
+        t
+    }
+
     /// Configure with alice@example.com.
     ///
     /// The context will be fake-configured as the alice user, with a pre-generated secret
@@ -76,6 +106,122 @@ impl TestContext {
             .await
             .unwrap();
     }
+
+    /// Retrieve a sent message from the jobs table.
+    ///
+    /// This retrieves and removes a message which has been scheduled to send from the jobs
+    /// table.  Messages are returned in the order they have been sent.
+    ///
+    /// Panics if there is no message or on any error.
+    pub async fn pop_sent_msg(&self) -> SentMessage {
+        let start = Instant::now();
+        let (rowid, foreign_id, raw_params) = loop {
+            let row = self
+                .ctx
+                .sql
+                .query_row(
+                    r#"
+                    SELECT id, foreign_id, param
+                      FROM jobs
+                     WHERE action=?
+                  ORDER BY desired_timestamp;
+                "#,
+                    paramsv![Action::SendMsgToSmtp],
+                    |row| {
+                        let id: i64 = row.get(0)?;
+                        let foreign_id: i64 = row.get(1)?;
+                        let param: String = row.get(2)?;
+                        Ok((id, foreign_id, param))
+                    },
+                )
+                .await;
+            if let Ok(row) = row {
+                break row;
+            }
+            if start.elapsed() < Duration::from_secs(3) {
+                async_std::task::sleep(Duration::from_millis(100)).await;
+            } else {
+                panic!("no sent message found in jobs table");
+            }
+        };
+        let id = ChatId::new(foreign_id as u32);
+        let params = Params::from_str(&raw_params).unwrap();
+        let blob_path = params
+            .get_blob(Param::File, &self.ctx, false)
+            .await
+            .expect("failed to parse blob from param")
+            .expect("no Param::File found in Params")
+            .to_abs_path();
+        self.ctx
+            .sql
+            .execute("DELETE FROM jobs WHERE id=?;", paramsv![rowid])
+            .await
+            .expect("failed to remove job");
+        SentMessage {
+            id,
+            params,
+            blob_path,
+        }
+    }
+
+    /// Parse a message.
+    ///
+    /// Parsing a message does not run the entire receive pipeline, but is not without
+    /// side-effects either.  E.g. if the message includes autocrypt headers the relevant
+    /// peerstates will be updated.  Later receiving the message using [recv_msg] is
+    /// unlikely to be affected as the peerstate would be processed again in exactly the
+    /// same way.
+    pub async fn parse_msg(&self, msg: &SentMessage) -> MimeMessage {
+        MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes())
+            .await
+            .unwrap()
+    }
+
+    /// Receive a message.
+    ///
+    /// Receives a message using the `dc_receive_imf()` pipeline.
+    pub async fn recv_msg(&self, msg: &SentMessage) {
+        let mut idx = self.recv_idx.write().await;
+        *idx += 1;
+        dc_receive_imf(&self.ctx, msg.payload().as_bytes(), "INBOX", *idx, false)
+            .await
+            .unwrap();
+    }
+}
+
+/// A raw message as it was scheduled to be sent.
+///
+/// This is a raw message, probably in the shape DC was planning to send it but not having
+/// passed through a SMTP-IMAP pipeline.
+#[derive(Debug, Clone)]
+pub struct SentMessage {
+    id: ChatId,
+    params: Params,
+    blob_path: PathBuf,
+}
+
+impl SentMessage {
+    /// The ChatId the message belonged to.
+    pub fn id(&self) -> ChatId {
+        self.id
+    }
+
+    /// A recipient the message was destined for.
+    ///
+    /// If there are multiple recipients this is just a random one, so is not very useful.
+    pub fn recipient(&self) -> EmailAddress {
+        let raw = self
+            .params
+            .get(Param::Recipients)
+            .expect("no recipients in params");
+        let rcpt = raw.split(' ').next().expect("no recipient found");
+        rcpt.parse().expect("failed to parse email address")
+    }
+
+    /// The raw message payload.
+    pub fn payload(&self) -> String {
+        std::fs::read_to_string(&self.blob_path).unwrap()
+    }
 }
 
 /// Load a pre-generated keypair for alice@example.com from disk.

test-data/autoconfig/lakenet.ch.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<clientConfig version="1.1">
+  <emailProvider id="lakenet.ch">
+    <domain>%EMAILDOMAIN%</domain>
+    <displayName>%EMAILDOMAIN% Mail</displayName>
+    <displayShortName>%EMAILDOMAIN%</displayShortName>
+    <incomingServer type="imap">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="imap">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>143</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="pop3">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="pop3">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>110</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname>mail.lakenet.ch</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url="https://www.lakenet.ch/">
+      <descr lang="it">Impostazioni per le e-mail LakeNet</descr>
+      <descr lang="fr">Reglages pour le courriel e-mail LakeNet</descr>
+      <descr lang="en">Settings for LakeNet's e-mail accounts</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url="https://lakenet.ch/webmail/" />
+    <loginPageInfo url="https://lakenet.ch/webmail/">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id="rcmloginuser" name="_user"/>
+      <passwordField id="rcmloginpwd" name="_pass"/>
+      <loginButton id="rcmloginsubmit"/>
+    </loginPageInfo>
+  </webMail>
+  <clientConfigUpdate url="https://lakenet.ch/.well-known/autoconfig/mail/config-v1.1.xml" />
+</clientConfig>

test-data/autoconfig/outlook.com.xml

@@ -0,0 +1,71 @@
+<clientConfig version="1.1">
+  <!-- Retrieved from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11 -->
+  <emailProvider id="outlook.com">
+    <domain>hotmail.com</domain>
+    <domain>hotmail.co.uk</domain>
+    <domain>hotmail.co.jp</domain>
+    <domain>hotmail.com.br</domain>
+    <domain>hotmail.de</domain>
+    <domain>hotmail.fr</domain>
+    <domain>hotmail.it</domain>
+    <domain>hotmail.es</domain>
+    <domain>live.com</domain>
+    <domain>live.co.uk</domain>
+    <domain>live.co.jp</domain>
+    <domain>live.de</domain>
+    <domain>live.fr</domain>
+    <domain>live.it</domain>
+    <domain>live.jp</domain>
+    <domain>msn.com</domain>
+    <domain>outlook.com</domain>
+    <displayName>Outlook.com (Microsoft)</displayName>
+    <displayShortName>Outlook</displayShortName>
+    <incomingServer type="exchange">
+      <hostname>outlook.office365.com</hostname>
+      <port>443</port>
+      <username>%EMAILADDRESS%</username>
+      <socketType>SSL</socketType>
+      <authentication>OAuth2</authentication>
+      <owaURL>https://outlook.office365.com/owa/</owaURL>
+      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
+      <useGlobalPreferredServer>true</useGlobalPreferredServer>
+    </incomingServer>
+    <incomingServer type="imap">
+      <hostname>outlook.office365.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type="pop3">
+      <hostname>outlook.office365.com</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <pop3>
+        <leaveMessagesOnServer>true</leaveMessagesOnServer>
+        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
+      </pop3>
+    </incomingServer>
+    <outgoingServer type="smtp">
+      <hostname>smtp.office365.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url="http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app">
+      <descr lang="en">Set up an email app with Outlook.com</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url="https://www.outlook.com/"/>
+    <loginPageInfo url="https://www.outlook.com/">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id="i0116" name="login"/>
+      <passwordField id="i0118" name="passwd"/>
+      <loginButton id="idSIButton9" name="SI"/>
+    </loginPageInfo>
+  </webMail>
+</clientConfig>";

test-data/message/wrong-html.eml

@@ -0,0 +1,114 @@
+Return-Path: <camping@otherdomain.eu>
+X-Original-To: alice@example.com
+Delivered-To: m045a7e8@dd37930.kasserver.com
+Received: from mout.kundenserver.de (mout.kundenserver.de [212.227.126.131])
+	by dd37930.kasserver.com (Postfix) with ESMTPS id 271F34B4258C
+	for <holger@somedomain.de>; Thu,  6 Aug 2020 18:40:32 +0200 (CEST)
+Received: from oxbsltgw18.schlund.de ([172.19.249.35]) by
+ mrelayeu.kundenserver.de (mreue009 [213.165.67.103]) with ESMTPSA (Nemesis)
+ id 1MpDRv-1kW93Y0lGZ-00qjvh for <holger@somedomain.de>; Thu, 06 Aug 2020
+ 18:40:31 +0200
+Date: Thu, 6 Aug 2020 18:40:30 +0200 (CEST)
+From: Camping <camping@otherdomain.eu>
+Reply-To: Camping <camping@otherdomain.eu>
+To: Alice <alice@example.com>
+Message-ID: <512278196.1287440.1596732031020@email.ionos.fr>
+Subject: Re: subj?
+MIME-Version: 1.0
+Content-Type: multipart/related;
+	boundary="----=_Part_1287438_2124736777.1596732031007"
+X-Mailer: Open-Xchange Mailer v7.10.1-Rev32
+X-Originating-Client: open-xchange-appsuite
+
+------=_Part_1287438_2124736777.1596732031007
+MIME-Version: 1.0
+Content-Type: text/html; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+
+<!doctype html>
+<html>
+ <head>=20
+  <meta charset=3D"UTF-8">=20
+ </head>
+ <body>
+  <div class=3D"default-style">
+   Guten Abend,
+   <br>
+  </div>
+  <div class=3D"default-style">
+   <br>
+  </div>
+  <div class=3D"default-style">
+   Lots of text
+   <br>
+  </div>
+  <div class=3D"default-style">
+   text with Umlaut =C3=A4...
+   <br>
+  </div>
+  <div class=3D"default-style">
+   MfG
+   <br>
+  </div>
+  <div class=3D"default-style">
+   <br>
+  </div>
+  <div class=3D"io-ox-signature">
+   <p>--------------------------------------<br></p>
+   <p style=3D"text-align: left;"><a href=3D"https://example.com=
+/">Camping </a><br></p>
+   <p>someaddress<br></p>
+   <p>sometown</p>
+   <p><img alt=3D"" class=3D"aspect-ratio" style=3D"width: 505px; height: 1=
+68px; max-width: 100%;" id=3D"d5cd260f-1b1f-4bfa-81d7-c0a74923b979" src=3D"=
+cid:d5cd260f-1b1f-4bfa-81d7-c0a74923b979" width=3D"505" height=3D"168"><br>=
+</p>
+  </div>
+  <blockquote type=3D"cite">
+   <div>
+    Le 5 ao=C3=BBt 2020 =C3=A0 10:46, holger &lt;
+    <a href=3D"mailto:holger@somedomain.de">holger@somedomain.de</a>&gt; a =
+=C3=A9crit&nbsp;:
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    Bonjour,
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    <br>
+   </div>
+   <div>
+    --
+   </div>
+   <div>
+    Sent with my Delta Chat Messenger:=20
+    <a href=3D"https://delta.chat" rel=3D"noopener" target=3D"_blank">https=
+://delta.chat</a>
+    <br>
+   </div>
+  </blockquote>=20
+ </body>
+</html>
+------=_Part_1287438_2124736777.1596732031007
+Content-Type: image/png
+Content-Transfer-Encoding: base64
+Content-ID: <d5cd260f-1b1f-4bfa-81d7-c0a74923b979>
+Content-Disposition: inline
+
+iVBORw0KGgoAAAANSUhEUgAAAlgAAADICAYAAAA0n5+2AAAgAElEQVR4nCzaZ3sjhIGo7fyMPSch
+MN29S7YsW71avfcuWZJlS3K33HuvY08fpjEVhoGhDh0CSSAESEhCgDRSKNns5mx2k8AM5Dznw/v+
+[scrubbed]
+/kNZ/B9Uub32fzngSwAAAABJRU5ErkJggg==
+------=_Part_1287438_2124736777.1596732031007--
+

tests/stress.rs

@@ -2,7 +2,7 @@
 
 use deltachat::config;
 use deltachat::context::*;
-use tempfile::{tempdir, TempDir};
+use tempfile::tempdir;
 
 /* some data used for testing
  ******************************************************************************/
@@ -92,26 +92,19 @@ async fn stress_functions(context: &Context) {
     // free(qr.cast());
 }
 
-#[allow(dead_code)]
-struct TestContext {
-    ctx: Context,
-    dir: TempDir,
-}
-
-async fn create_test_context() -> TestContext {
+async fn create_test_context() -> Context {
     use rand::Rng;
 
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = rand::thread_rng().gen();
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), id)
+    Context::new("FakeOs".into(), dbfile.into(), id)
         .await
-        .unwrap();
-    TestContext { ctx, dir }
+        .unwrap()
 }
 
 #[async_std::test]
 async fn test_stress_tests() {
     let context = create_test_context().await;
-    stress_functions(&context.ctx).await;
+    stress_functions(&context).await;
 }