sipmlify, no randomness anymore

values are either 'delete' or 'wal';
having this info in debug reports is useful for debugging anyway,
if we switch to WAL or not.

.circleci/config.yml

@@ -7,6 +7,10 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
 
 restore-workspace: &restore-workspace
   attach_workspace:

CHANGELOG.md

@@ -1,5 +1,44 @@
 # Changelog 
 
+## 1.33.0
+
+- let `dc_set_muted()` also mute one-to-one chats #1470
+
+- fix a bug that led to load and traffic if the server does not use sent-folder
+  #1472
+
+
+## 1.32.0
+
+- fix endless loop when trying to download messages with bad RFC Message-ID,
+  also be more reliable on similar errors #1463 #1466 #1462
+
+- fix bug with comma in contact request #1438
+
+- do not refer to hidden messages on replies #1459
+
+- improve error handling #1468 #1465 #1464
+
+
+## 1.31.0
+
+- always describe the context of the displayed error #1451
+
+- do not emit `DC_EVENT_ERROR` when message sending fails;
+  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
+
+- new config-option `media_quality` #1449
+
+- try over if writing message to database fails #1447
+
+
+## 1.30.0
+
+- expunge deleted messages #1440
+
+- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
+
+
 ## 1.29.0
 
 - new config options `delete_device_after` and `delete_server_after`,

Cargo.lock

@@ -629,7 +629,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.29.0"
+version = "1.33.0"
 dependencies = [
  "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -695,10 +695,10 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.29.0"
+version = "1.33.0"
 dependencies = [
  "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
- "deltachat 1.29.0",
+ "deltachat 1.33.0",
  "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.29.0"
+version = "1.33.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"

ci_scripts/ci_upload.sh

@@ -37,7 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip
+pip3 install -U pip setuptools
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        assert projectnames == ["deltachat"]
+        clear_index = not projectnames
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,8 +60,11 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                url = baseurl + username + "/" + indexname
-                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                clear_index = True
+                break
+        if clear_index:
+            url = baseurl + username + "/" + indexname
+            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.29.0"
+version = "1.33.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -378,10 +378,21 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    >=1=seconds, after which messages are deleted automatically from the device.
  *                    Messages in the "saved messages" chat (see dc_chat_is_self_talk()) are skipped.
  *                    Messages are deleted whether they were seen or not, the UI should clearly point that out.
+ *                    See also dc_estimate_deletion_cnt().
  * - `delete_server_after` = 0=do not delete messages from server automatically (default),
- *                    >=1=seconds, after which messages are deleted automatically from the server.
+ *                    1=delete messages directly after receiving from server, mvbox is skipped.
+ *                    >1=seconds, after which messages are deleted automatically from the server, mvbox is used as defined.
  *                    "Saved messages" are deleted from the server as well as
  *                    emails matching the `show_emails` settings above, the UI should clearly point that out.
+ *                    See also dc_estimate_deletion_cnt().
+ * - `media_quality` = DC_MEDIA_QUALITY_BALANCED (0) =
+ *                    good outgoing images/videos/voice quality at reasonable sizes (default)
+ *                    DC_MEDIA_QUALITY_WORSE (1)
+ *                    allow worse images/videos/voice quality to gain smaller sizes,
+ *                    suitable for providers or areas known to have a bad connection.
+ *                    In contrast to other options, the implementation of this option is currently up to the UIs;
+ *                    this may change in future, however,
+ *                    having the option in the core allows provider-specific-defaults already today.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -949,8 +960,10 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *       if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
  *       the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
  *       chats
- *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist,
+ *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
+ *       and hides the "Device chat" and the deaddrop.
  *       typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+ *       to also hide the archive link.
  *     - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
  *       to the list (may be used eg. for selecting chats on forwarding, the flag is
  *       not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -1310,6 +1323,7 @@ int             dc_get_fresh_msg_cnt         (dc_context_t* context, uint32_t ch
  * by the dc_set_config()-options `delete_device_after` or `delete_server_after`.
  * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages.
  *
+ * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param from_server 1=Estimate deletion count for server, 0=Estimate deletion count for device
  * @param seconds Count messages older than the given number of seconds.
@@ -1626,7 +1640,7 @@ int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t ch
 /**
  * Set mute duration of a chat.
  *
- * This value can be checked by the ui upon receiving a new message to decide whether it should trigger an notification.
+ * The ui can then call dc_chat_is_muted() when receiving a new message to decide whether it should trigger an notification.
  *
  * Sends out #DC_EVENT_CHAT_MODIFIED.
  *
@@ -2964,7 +2978,7 @@ int             dc_chat_is_sending_locations (const dc_chat_t* chat);
 
 
 /**
- * Check whether the chat is currently muted
+ * Check whether the chat is currently muted (can be changed by dc_set_chat_mute_duration()).
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -4492,6 +4506,14 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_SHOW_EMAILS_ACCEPTED_CONTACTS 1
 #define DC_SHOW_EMAILS_ALL               2
 
+
+/*
+ * Values for dc_get|set_config("media_quality")
+ */
+#define DC_MEDIA_QUALITY_BALANCED 0
+#define DC_MEDIA_QUALITY_WORSE    1
+
+
 /*
  * Values for dc_get|set_config("key_gen_type")
  */

python/examples/test_examples.py

@@ -51,7 +51,7 @@ def test_group_tracking_plugin(acfactory, lp):
 
     botproc.fnmatch_lines("""
         *ac_chat_modified*bot test group*
-    """.format(ac1.get_config("addr")))
+    """)
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))

python/src/deltachat/account.py

@@ -30,7 +30,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None):
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -39,6 +39,7 @@ class Account(object):
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
+        self._logging = logging
         self.add_account_plugin(self)
 
         self._dc_context = ffi.gc(
@@ -63,6 +64,14 @@ class Account(object):
         atexit.register(self.shutdown)
         hook.dc_account_init(account=self)
 
+    def disable_logging(self):
+        """ disable logging. """
+        self._logging = False
+
+    def enable_logging(self):
+        """ re-enable logging. """
+        self._logging = True
+
     @hookspec.account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
         for name, kwargs in self._map_ffi_event(ffi_event):
@@ -73,7 +82,8 @@ class Account(object):
     #    self.shutdown()
 
     def ac_log_line(self, msg):
-        self._pm.hook.ac_log_line(message=msg)
+        if self._logging:
+            self._pm.hook.ac_log_line(message=msg)
 
     def _check_config_key(self, name):
         if name not in self._configkeys:

python/src/deltachat/testplugin.py

@@ -5,9 +5,13 @@ import subprocess
 import queue
 import threading
 import fnmatch
+import time
+import weakref
+import tempfile
+
 import pytest
 import requests
-import time
+
 from . import Account, const
 from .tracker import ConfigureTracker
 from .capi import lib
@@ -15,7 +19,7 @@ from .eventlogger import FFIEventLogger, FFIEventTracker
 from _pytest.monkeypatch import MonkeyPatch
 from _pytest._code import Source
 
-import tempfile
+import deltachat
 
 
 def pytest_addoption(parser):
@@ -40,11 +44,55 @@ def pytest_configure(config):
         if cfg:
             config.option.liveconfig = cfg
 
+    # Make sure we don't get garbled output because threads keep running
+    # collect all ever created accounts in a weakref-set (so we don't
+    # keep objects unneccessarily alive) and enable/disable logging
+    # for each pytest test phase # (setup/call/teardown).
+    # Additionally make the acfactory use a logging/no-logging default.
 
-def pytest_runtest_setup(item):
-    if (list(item.iter_markers(name="ignored"))
-            and not item.config.getoption("ignored")):
-        pytest.skip("Ignored tests not requested, use --ignored")
+    class LoggingAspect:
+        def __init__(self):
+            self._accounts = weakref.WeakSet()
+
+        @deltachat.global_hookimpl
+        def dc_account_init(self, account):
+            self._accounts.add(account)
+
+        def disable_logging(self, item):
+            for acc in self._accounts:
+                acc.disable_logging()
+            acfactory = item.funcargs.get("acfactory")
+            if acfactory:
+                acfactory.set_logging_default(False)
+
+        def enable_logging(self, item):
+            for acc in self._accounts:
+                acc.enable_logging()
+            acfactory = item.funcargs.get("acfactory")
+            if acfactory:
+                acfactory.set_logging_default(True)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_runtest_setup(self, item):
+            self.enable_logging(item)
+            yield
+            self.disable_logging(item)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_pyfunc_call(self, pyfuncitem):
+            self.enable_logging(pyfuncitem)
+            yield
+            self.disable_logging(pyfuncitem)
+
+        @pytest.hookimpl(hookwrapper=True)
+        def pytest_runtest_teardown(self, item):
+            self.enable_logging(item)
+            yield
+            self.disable_logging(item)
+
+    la = LoggingAspect()
+    config.pluginmanager.register(la)
+    deltachat.register_global_plugin(la)
 
 
 def pytest_report_header(config, startdir):
@@ -164,24 +212,35 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
+            self._accounts = []
             self.init_time = time.time()
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
+            self.set_logging_default(False)
 
         def finalize(self):
             while self._finalizers:
                 fin = self._finalizers.pop()
                 fin()
 
+            while self._accounts:
+                acc = self._accounts.pop()
+                acc.shutdown()
+                acc.disable_logging()
+
         def make_account(self, path, logid, quiet=False):
-            ac = Account(path)
+            ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
             ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            ac.addr = ac.get_self_contact().addr
             if not quiet:
                 ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
-            self._finalizers.append(ac.shutdown)
+            self._accounts.append(ac)
             return ac
 
+        def set_logging_default(self, logging):
+            self._logging = bool(logging)
+
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
@@ -262,6 +321,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac2._configtracker.wait_finish()
             return ac1, ac2
 
+        def get_many_online_accounts(self, num, move=True, quiet=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
+                        for i in range(num)]
+            for acc in accounts:
+                acc._configtracker.wait_finish()
+            return accounts
+
         def clone_online_account(self, account, pre_generated_key=True):
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)

python/tests/stress_test_db.py

@@ -0,0 +1,105 @@
+import time
+import os
+from queue import Queue
+
+import deltachat
+
+
+def test_db_busy_error(acfactory, tmpdir):
+    starttime = time.time()
+
+    def log(string):
+        print("%3.2f %s" % (time.time() - starttime, string))
+
+    # make a number of accounts
+    accounts = acfactory.get_many_online_accounts(5, quiet=False)
+    log("created %s accounts" % len(accounts))
+
+    # put a bigfile into each account
+    for acc in accounts:
+        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
+        with open(acc.bigfile, "wb") as f:
+            f.write(b"01234567890"*1000_000)
+    log("created %s bigfiles" % len(accounts))
+
+    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
+    chat = accounts[0].create_group_chat("stress-group")
+    for addr in contact_addrs[1:]:
+        chat.add_contact(chat.account.create_contact(addr))
+
+    # setup auto-responder bots which report back failures/actions
+    report_queue = Queue()
+
+    def report_func(replier, report_type, *report_args):
+        report_queue.put((replier, report_type, report_args))
+
+    # each replier receives all events and sends report events to receive_queue
+    repliers = []
+    for acc in accounts:
+        replier = AutoReplier(acc, num_send=1000, num_bigfiles=0, report_func=report_func)
+        acc.add_account_plugin(replier)
+        repliers.append(replier)
+
+    # kick off message sending
+    # after which repliers will reply to each other
+    chat.send_text("hello")
+
+    alive_count = len(accounts)
+    while alive_count > 0:
+        replier, report_type, report_args = report_queue.get(10)
+        addr = replier.account.get_self_contact().addr
+        assert addr
+        if report_type == ReportType.exit:
+            alive_count -= 1
+            log("{} EXIT -- remaining: {}".format(addr, alive_count))
+            replier.account.shutdown(wait=True)
+        elif report_type == ReportType.message_sent:
+            log("{} sent message: {}".format(addr, report_args[0].text))
+        elif report_type == ReportType.message_incoming:
+            log("{} incoming message: {}".format(addr, report_args[0].text))
+        elif report_type == ReportType.ffi_error:
+            log("{} ERROR: {}".format(addr, report_args[0]))
+            replier.account.shutdown(wait=True)
+            alive_count -= 1
+
+
+class ReportType:
+    exit = "exit"
+    message_sent = "message-sent"
+    ffi_error = "ffi-error"
+    message_incoming = "message-incoming"
+
+
+class AutoReplier:
+    def __init__(self, account, report_func, num_send, num_bigfiles):
+        self.account = account
+        self.report_func = report_func
+        self.num_send = num_send
+        self.num_bigfiles = num_bigfiles
+        self.current_sent = 0
+
+    @deltachat.account_hookimpl
+    def ac_incoming_message(self, message):
+        if self.current_sent >= self.num_send:
+            return
+        message.accept_sender_contact()
+        message.mark_seen()
+        self.report_func(self, ReportType.message_incoming, message)
+
+        self.current_sent += 1
+        # we are still alive, let's send a reply
+        if self.num_bigfiles and self.current_sent % self.num_bigfiles == 0:
+            message.chat.send_text("send big file as reply to: {}".format(message.text))
+            msg = message.chat.send_file(self.account.bigfile)
+        else:
+            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
+            assert msg.text
+        self.report_func(self, ReportType.message_sent, msg)
+        if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
+            return
+
+    @deltachat.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        if ffi_event.name == "DC_EVENT_ERROR":
+            self.report_func(self, ReportType.ffi_error, ffi_event)

src/chat.rs

@@ -385,7 +385,7 @@ impl ChatId {
         let sql = &context.sql;
         let query = format!(
             "SELECT {} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) AND NOT hidden \
              ORDER BY timestamp DESC, id DESC \
              LIMIT 1;",
             fields
@@ -1612,8 +1612,7 @@ pub fn delete_device_expired_messages(context: &Context) -> Result<bool, Error>
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
-             AND chat_id != ? \
-             AND NOT hidden",
+             AND chat_id != ?",
             params![
                 DC_CHAT_ID_TRASH,
                 threshold_timestamp,
@@ -2072,18 +2071,17 @@ impl rusqlite::types::FromSql for MuteDuration {
 
 pub fn set_muted(context: &Context, chat_id: ChatId, duration: MuteDuration) -> Result<(), Error> {
     ensure!(!chat_id.is_special(), "Invalid chat ID");
-    if real_group_exists(context, chat_id)
-        && sql::execute(
-            context,
-            &context.sql,
-            "UPDATE chats SET muted_until=? WHERE id=?;",
-            params![duration, chat_id],
-        )
-        .is_ok()
+    if sql::execute(
+        context,
+        &context.sql,
+        "UPDATE chats SET muted_until=? WHERE id=?;",
+        params![duration, chat_id],
+    )
+    .is_ok()
     {
         context.call_cb(Event::ChatModified(chat_id));
     } else {
-        bail!("Failed to set name");
+        bail!("Failed to set mute duration, chat might not exist -");
     }
     Ok(())
 }

src/chatlist.rs

@@ -92,6 +92,11 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
+        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
+        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
+        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
+        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
+
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
         if let Err(err) = delete_device_expired_messages(context) {
@@ -111,7 +116,7 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+        let skip_id = if flag_for_forwarding {
             chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
                 .unwrap_or_default()
                 .0
@@ -155,7 +160,7 @@ impl Chatlist {
                 process_row,
                 process_rows,
             )?
-        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
+        } else if flag_archived_only {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
             // however, then the number of archived chats do not match, which might be even more irritating.
@@ -211,7 +216,7 @@ impl Chatlist {
             )?
         } else {
             //  show normal chatlist
-            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+            let sort_id_up = if flag_for_forwarding {
                 chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
                     .unwrap_or_default()
                     .0
@@ -237,9 +242,9 @@ impl Chatlist {
                 process_row,
                 process_rows,
             )?;
-            if 0 == listflags & DC_GCL_NO_SPECIALS {
+            if !flag_no_specials {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
+                    if !flag_for_forwarding {
                         ids.insert(
                             0,
                             (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
@@ -252,7 +257,7 @@ impl Chatlist {
         };
 
         if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
-            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
+            if ids.is_empty() && flag_add_alldone_hint {
                 ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
             ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));

src/config.rs

@@ -65,6 +65,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
+    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
+    MediaQuality,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -248,9 +251,11 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants;
     use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
     use image::GenericImageView;
+    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -346,4 +351,21 @@ mod tests {
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
+
+    #[test]
+    fn test_media_quality_config_option() {
+        let t = dummy_context();
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
+        assert_eq!(media_quality, 0);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Balanced);
+
+        t.ctx.set_config(Config::MediaQuality, Some("1")).unwrap();
+
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
+        assert_eq!(media_quality, 1);
+        assert_eq!(constants::MediaQuality::Worse as i32, 1);
+        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
+        assert_eq!(media_quality, constants::MediaQuality::Worse);
+    }
 }

src/configure/mod.rs

@@ -69,6 +69,7 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     let mut smtp_connected_here = false;
 
     let mut param_autoconfig: Option<LoginParam> = None;
+    let was_configured_before = context.is_configured();
 
     context
         .inbox_thread
@@ -441,6 +442,15 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     }
 
     if let Some(provider) = provider::get_provider_info(&param.addr) {
+        if !was_configured_before {
+            if let Some(config_defaults) = &provider.config_defaults {
+                for def in config_defaults.iter() {
+                    info!(context, "apply config_defaults {}={}", def.key, def.value);
+                    context.set_config(def.key, Some(def.value));
+                }
+            }
+        }
+
         if !provider.after_login_hint.is_empty() {
             let mut msg = Message::new(Viewtype::Text);
             msg.text = Some(provider.after_login_hint.to_string());

src/constants.rs

@@ -57,6 +57,19 @@ impl Default for ShowEmails {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum MediaQuality {
+    Balanced = 0,
+    Worse = 1,
+}
+
+impl Default for MediaQuality {
+    fn default() -> Self {
+        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
+    }
+}
+
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum KeyGenType {

src/context.rs

@@ -240,7 +240,10 @@ impl Context {
             .sql
             .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-
+        let journal_mode = self
+            .sql
+            .query_get_value(self, "PRAGMA journal_mode;", rusqlite::NO_PARAMS)
+            .unwrap_or_else(|| "unknown".to_string());
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
         let bcc_self = self.get_config_int(Config::BccSelf);
@@ -285,6 +288,7 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
+        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(

src/dc_receive_imf.rs

@@ -3,6 +3,8 @@ use sha2::{Digest, Sha256};
 
 use num_traits::FromPrimitive;
 
+use mailparse::SingleInfo;
+
 use crate::chat::{self, Chat, ChatId};
 use crate::config::Config;
 use crate::constants::*;
@@ -32,6 +34,10 @@ enum CreateEvent {
 }
 
 /// Receive a message and add it to the database.
+///
+/// Returns an error on recoverable errors, e.g. database errors. In this case,
+/// message parsing should be retried later. If message itself is wrong, logs
+/// the error and returns success.
 pub fn dc_receive_imf(
     context: &Context,
     imf_raw: &[u8],
@@ -55,10 +61,19 @@ pub fn dc_receive_imf(
         println!("{}", String::from_utf8_lossy(imf_raw));
     }
 
-    let mut mime_parser = MimeMessage::from_bytes(context, imf_raw)?;
+    let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw) {
+        Err(err) => {
+            warn!(context, "dc_receive_imf: can't parse MIME: {}", err);
+            return Ok(());
+        }
+        Ok(mime_parser) => mime_parser,
+    };
 
     // we can not add even an empty record if we have no info whatsoever
-    ensure!(mime_parser.has_headers(), "No Headers Found");
+    if !mime_parser.has_headers() {
+        warn!(context, "dc_receive_imf: no headers found");
+        return Ok(());
+    }
 
     // the function returns the number of created messages in the database
     let mut chat_id = ChatId::new(0);
@@ -97,29 +112,23 @@ pub fn dc_receive_imf(
     // we do not check Return-Path any more as this is unreliable, see
     // https://github.com/deltachat/deltachat-core/issues/150)
     let (from_id, from_id_blocked, incoming_origin) =
-        if let Some(field_from) = mime_parser.get(HeaderDef::From_) {
-            from_field_to_contact_id(context, field_from)?
-        } else {
-            (0, false, Origin::Unknown)
-        };
+        from_field_to_contact_id(context, &mime_parser.from)?;
+
     let incoming = from_id != DC_CONTACT_ID_SELF;
 
     let mut to_ids = ContactIds::new();
-    for header_def in &[HeaderDef::To, HeaderDef::Cc] {
-        if let Some(field) = mime_parser.get(header_def.clone()) {
-            to_ids.extend(&dc_add_or_lookup_contacts_by_address_list(
-                context,
-                &field,
-                if !incoming {
-                    Origin::OutgoingTo
-                } else if incoming_origin.is_known() {
-                    Origin::IncomingTo
-                } else {
-                    Origin::IncomingUnknownTo
-                },
-            )?);
-        }
-    }
+
+    to_ids.extend(&dc_add_or_lookup_contacts_by_address_list(
+        context,
+        &mime_parser.recipients,
+        if !incoming {
+            Origin::OutgoingTo
+        } else if incoming_origin.is_known() {
+            Origin::IncomingTo
+        } else {
+            Origin::IncomingUnknownTo
+        },
+    )?);
 
     // Add parts
 
@@ -229,11 +238,11 @@ pub fn dc_receive_imf(
 /// Also returns whether it is blocked or not and its origin.
 pub fn from_field_to_contact_id(
     context: &Context,
-    field_from: &str,
+    from_address_list: &[SingleInfo],
 ) -> Result<(u32, bool, Origin)> {
     let from_ids = dc_add_or_lookup_contacts_by_address_list(
         context,
-        &field_from,
+        from_address_list,
         Origin::IncomingUnknownFrom,
     )?;
 
@@ -243,7 +252,7 @@ pub fn from_field_to_contact_id(
         if from_ids.len() > 1 {
             warn!(
                 context,
-                "mail has more than one From address, only using first: {:?}", field_from
+                "mail has more than one From address, only using first: {:?}", from_address_list
             );
         }
         let from_id = from_ids.get_index(0).cloned().unwrap_or_default();
@@ -256,7 +265,10 @@ pub fn from_field_to_contact_id(
         }
         Ok((from_id, from_id_blocked, incoming_origin))
     } else {
-        warn!(context, "mail has an empty From header: {:?}", field_from);
+        warn!(
+            context,
+            "mail has an empty From header: {:?}", from_address_list
+        );
         // if there is no from given, from_id stays 0 which is just fine. These messages
         // are very rare, however, we have to add them to the database (they go to the
         // "deaddrop" chat) to avoid a re-download from the server. See also [**]
@@ -305,7 +317,8 @@ fn add_parts(
             message::update_server_uid(context, &rfc724_mid, server_folder.as_ref(), server_uid);
         }
 
-        bail!("Message already in DB");
+        warn!(context, "Message already in DB");
+        return Ok(());
     }
 
     let mut msgrmsg = if mime_parser.has_chat_version() {
@@ -369,7 +382,8 @@ fn add_parts(
                     *hidden = true;
                     context.bob.write().unwrap().status = 0; // secure-join failed
                     context.stop_ongoing();
-                    error!(context, "Error in Secure-Join message handling: {}", err);
+                    warn!(context, "Error in Secure-Join message handling: {}", err);
+                    return Ok(());
                 }
             }
         }
@@ -496,7 +510,8 @@ fn add_parts(
                 }
                 Err(err) => {
                     *hidden = true;
-                    error!(context, "Error in Secure-Join watching: {}", err);
+                    warn!(context, "Error in Secure-Join watching: {}", err);
+                    return Ok(());
                 }
             }
         }
@@ -1505,7 +1520,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
         return false;
     }
 
-    if let Ok(ids) = mailparse::msgidparse(mid_list) {
+    if let Ok(ids) = parse_message_ids(mid_list) {
         for id in ids.iter() {
             if is_known_rfc724_mid(context, id) {
                 return true;
@@ -1553,7 +1568,7 @@ fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) -
 }
 
 pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
-    if let Ok(ids) = mailparse::msgidparse(mid_list) {
+    if let Ok(ids) = parse_message_ids(mid_list) {
         for id in ids.iter() {
             if is_msgrmsg_rfc724_mid(context, id) {
                 return true;
@@ -1577,38 +1592,17 @@ fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
 
 fn dc_add_or_lookup_contacts_by_address_list(
     context: &Context,
-    addr_list_raw: &str,
+    address_list: &[SingleInfo],
     origin: Origin,
 ) -> Result<ContactIds> {
-    let addrs = match mailparse::addrparse(addr_list_raw) {
-        Ok(addrs) => addrs,
-        Err(err) => {
-            bail!("could not parse {:?}: {:?}", addr_list_raw, err);
-        }
-    };
-
     let mut contact_ids = ContactIds::new();
-    for addr in addrs.iter() {
-        match addr {
-            mailparse::MailAddr::Single(info) => {
-                contact_ids.insert(add_or_lookup_contact_by_addr(
-                    context,
-                    &info.display_name,
-                    &info.addr,
-                    origin,
-                )?);
-            }
-            mailparse::MailAddr::Group(infos) => {
-                for info in &infos.addrs {
-                    contact_ids.insert(add_or_lookup_contact_by_addr(
-                        context,
-                        &info.display_name,
-                        &info.addr,
-                        origin,
-                    )?);
-                }
-            }
-        }
+    for info in address_list.iter() {
+        contact_ids.insert(add_or_lookup_contact_by_addr(
+            context,
+            &info.display_name,
+            &info.addr,
+            origin,
+        )?);
     }
 
     Ok(contact_ids)
@@ -2001,4 +1995,151 @@ mod tests {
         let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap();
         assert!(one2one.get_visibility() == ChatVisibility::Archived);
     }
+
+    #[test]
+    fn test_no_from() {
+        // if there is no from given, from_id stays 0 which is just fine. These messages
+        // are very rare, however, we have to add them to the database (they go to the
+        // "deaddrop" chat) to avoid a re-download from the server. See also [**]
+
+        let t = configured_offline_context();
+        let context = &t.ctx;
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert!(chats.get_msg_id(0).is_err());
+
+        dc_receive_imf(
+            context,
+            b"To: bob@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <3924@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        // Check that the message was added to the database:
+        assert!(chats.get_msg_id(0).is_ok());
+    }
+
+    #[test]
+    fn test_escaped_from() {
+        let t = configured_offline_context();
+        let contact_id = Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap();
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id).unwrap();
+        dc_receive_imf(
+            &t.ctx,
+            b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
+                 To: alice@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        ).unwrap();
+        assert_eq!(
+            Contact::load_from_db(&t.ctx, contact_id)
+                .unwrap()
+                .get_authname(),
+            "Фамилия Имя", // The name was "Имя, Фамилия" and ("lastname, firstname") and should be swapped to "firstname, lastname"
+        );
+        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None);
+        assert_eq!(msgs.len(), 1);
+        let msg_id = msgs.first().unwrap();
+        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap();
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert_eq!(msg.text.unwrap(), "hello");
+        assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
+    }
+
+    #[test]
+    fn test_escaped_recipients() {
+        let t = configured_offline_context();
+        Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap();
+
+        let carl_contact_id =
+            Contact::add_or_lookup(&t.ctx, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom)
+                .unwrap()
+                .0;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: Foobar <foobar@example.com>\n\
+                 To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.org\n\
+                 Cc: =?utf-8?q?=3Ch2=3E?= <carl@host.tld>\n\
+                 Subject: foo\n\
+                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: <foobar@example.com>\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .unwrap();
+        assert_eq!(
+            Contact::load_from_db(&t.ctx, carl_contact_id)
+                .unwrap()
+                .get_name(),
+            "h2"
+        );
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let msg = Message::load_from_db(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap();
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert_eq!(msg.text.unwrap(), "hello");
+        assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1);
+    }
+
+    #[test]
+    fn test_cc_to_contact() {
+        let t = configured_offline_context();
+        Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap();
+
+        let carl_contact_id = Contact::add_or_lookup(
+            &t.ctx,
+            "garabage",
+            "carl@host.tld",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap()
+        .0;
+
+        dc_receive_imf(
+            &t.ctx,
+            b"From: Foobar <foobar@example.com>\n\
+                 To: alice@example.org\n\
+                 Cc: Carl <carl@host.tld>\n\
+                 Subject: foo\n\
+                 Message-ID: <asdklfjjaweofi@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Disposition-Notification-To: <foobar@example.com>\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .unwrap();
+        assert_eq!(
+            Contact::load_from_db(&t.ctx, carl_contact_id)
+                .unwrap()
+                .get_name(),
+            "Carl"
+        );
+    }
 }

src/e2ee.rs

@@ -9,7 +9,8 @@ use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
 use crate::error::*;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::headerdef::HeaderDef;
+use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
@@ -122,8 +123,8 @@ pub fn try_decrypt(
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
         .headers
-        .get_header_value(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
+        .get_header(HeaderDef::From_)
+        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();

src/headerdef.rs

@@ -53,12 +53,16 @@ impl HeaderDef {
 
 pub trait HeaderDefMap {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
 }
 
 impl HeaderDefMap for [MailHeader<'_>] {
     fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
         self.get_first_value(headerdef.get_headername())
     }
+    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
+        self.get_first_header(headerdef.get_headername())
+    }
 }
 
 #[cfg(test)]

src/imap/idle.rs

@@ -59,7 +59,7 @@ impl Imap {
         task::block_on(async move { self.config.read().await.can_idle })
     }
 
-    pub fn idle(&self, context: &Context, watch_folder: String) -> Result<()> {
+    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<()> {
         task::block_on(async move {
             if !self.can_idle() {
                 return Err(Error::IdleAbilityMissing);
@@ -67,7 +67,7 @@ impl Imap {
 
             self.setup_handle_if_needed(context).await?;
 
-            self.select_folder(context, watch_folder).await?;
+            self.select_folder(context, watch_folder.clone()).await?;
 
             let session = self.session.lock().await.take();
             let timeout = Duration::from_secs(23 * 60);

src/imap/mod.rs

@@ -25,6 +25,7 @@ use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{job_add, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::message::{self, update_server_uid};
+use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
@@ -134,7 +135,7 @@ impl async_imap::Authenticator for OAuth2 {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 enum FolderMeaning {
     Unknown,
     SentObjects,
@@ -268,7 +269,7 @@ impl Imap {
                     let imap_port = config.imap_port;
                     context.stock_string_repl_str2(
                         StockMessage::ServerResponse,
-                        format!("{}:{}", imap_server, imap_port),
+                        format!("IMAP {}:{}", imap_server, imap_port),
                         err.to_string(),
                     )
                 };
@@ -476,7 +477,7 @@ impl Imap {
         folder: &str,
     ) -> Result<(u32, u32)> {
         task::block_on(async move {
-            self.select_folder(context, folder).await?;
+            self.select_folder(context, Some(folder)).await?;
 
             // compare last seen UIDVALIDITY against the current one
             let (uid_validity, last_seen_uid) = self.get_config_last_seen_uid(context, &folder);
@@ -604,7 +605,12 @@ impl Imap {
 
             let headers = get_fetch_headers(fetch)?;
             let message_id = prefetch_get_message_id(&headers).unwrap_or_default();
-            if precheck_imf(context, &message_id, folder.as_ref(), cur_uid)? {
+            if let Ok(true) =
+                precheck_imf(context, &message_id, folder.as_ref(), cur_uid).map_err(|err| {
+                    warn!(context, "precheck_imf error: {}", err);
+                    err
+                })
+            {
                 // we know the message-id already or don't want the message otherwise.
                 info!(
                     context,
@@ -613,6 +619,9 @@ impl Imap {
                     folder.as_ref(),
                 );
             } else {
+                // we do not know the message-id
+                // or the message-id is missing (in this case, we create one in the further process)
+                // or some other error happened
                 let show = prefetch_should_download(context, &headers, show_emails)
                     .map_err(|err| {
                         warn!(context, "prefetch_should_download error: {}", err);
@@ -733,13 +742,7 @@ impl Imap {
                 if let Err(err) =
                     dc_receive_imf(context, &body, folder.as_ref(), server_uid, is_seen)
                 {
-                    warn!(
-                        context,
-                        "dc_receive_imf failed for imap-message {}/{}: {:?}",
-                        folder.as_ref(),
-                        server_uid,
-                        err
-                    );
+                    return Err(Error::Other(format!("dc_receive_imf error: {}", err)));
                 }
             }
         } else {
@@ -912,7 +915,7 @@ impl Imap {
                     return Some(ImapActionResult::RetryLater);
                 }
             }
-            match self.select_folder(context, &folder).await {
+            match self.select_folder(context, Some(&folder)).await {
                 Ok(()) => None,
                 Err(select_folder::Error::ConnectionLost) => {
                     warn!(context, "Lost imap connection");
@@ -1060,13 +1063,15 @@ impl Imap {
                     }
                 };
 
-                let sentbox_folder =
-                    folders
-                        .iter()
-                        .find(|folder| match get_folder_meaning(folder) {
-                            FolderMeaning::SentObjects => true,
-                            _ => false,
-                        });
+                let sentbox_folder = folders
+                    .iter()
+                    .find(|folder| get_folder_meaning(folder) == FolderMeaning::SentObjects)
+                    .or_else(|| {
+                        info!(context, "can't find sentbox by attributes, checking names");
+                        folders.iter().find(|folder| {
+                            get_folder_meaning_by_name(folder) == FolderMeaning::SentObjects
+                        })
+                    });
                 info!(context, "sentbox folder is {:?}", sentbox_folder);
 
                 let mut delimiter = ".";
@@ -1183,7 +1188,7 @@ impl Imap {
                 error!(context, "could not setup imap connection: {}", err);
                 return;
             }
-            if let Err(err) = self.select_folder(context, &folder).await {
+            if let Err(err) = self.select_folder(context, Some(&folder)).await {
                 error!(
                     context,
                     "Could not select {} for expunging: {}", folder, err
@@ -1201,7 +1206,7 @@ impl Imap {
 
             // we now trigger expunge to actually delete messages
             self.config.write().await.selected_folder_needs_expunge = true;
-            match self.select_folder(context, &folder).await {
+            match self.select_folder::<String>(context, None).await {
                 Ok(()) => {
                     emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
                 }
@@ -1232,7 +1237,7 @@ impl Imap {
 // CAVE: if possible, take care not to add a name here that is "sent" in one language
 // but sth. different in others - a hard job.
 fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
-    let sent_names = vec!["sent", "sent objects", "gesendet"];
+    let sent_names = vec!["sent", "sentmail", "sent objects", "gesendet"];
     let lower = folder_name.name().to_lowercase();
 
     if sent_names.into_iter().any(|s| s == lower) {
@@ -1243,27 +1248,18 @@ fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
 }
 
 fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
-    if folder_name.attributes().is_empty() {
-        return FolderMeaning::Unknown;
-    }
-
-    let mut res = FolderMeaning::Unknown;
     let special_names = vec!["\\Spam", "\\Trash", "\\Drafts", "\\Junk"];
 
     for attr in folder_name.attributes() {
         if let NameAttribute::Custom(ref label) = attr {
             if special_names.iter().any(|s| *s == label) {
-                res = FolderMeaning::Other;
+                return FolderMeaning::Other;
             } else if label == "\\Sent" {
-                res = FolderMeaning::SentObjects
+                return FolderMeaning::SentObjects;
             }
         }
     }
-
-    match res {
-        FolderMeaning::Unknown => get_folder_meaning_by_name(folder_name),
-        _ => res,
-    }
+    FolderMeaning::Unknown
 }
 
 fn precheck_imf(
@@ -1379,11 +1375,8 @@ fn prefetch_should_download(
         .get_header_value(HeaderDef::AutocryptSetupMessage)
         .is_some();
 
-    let from_field = headers
-        .get_header_value(HeaderDef::From_)
-        .unwrap_or_default();
-
-    let (_contact_id, blocked_contact, origin) = from_field_to_contact_id(context, &from_field)?;
+    let (_contact_id, blocked_contact, origin) =
+        from_field_to_contact_id(context, &mimeparser::get_from(headers))?;
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message

src/imap/select_folder.rs

@@ -56,7 +56,7 @@ impl Imap {
     pub(super) async fn select_folder<S: AsRef<str>>(
         &self,
         context: &Context,
-        folder: S,
+        folder: Option<S>,
     ) -> Result<()> {
         if self.session.lock().await.is_none() {
             let mut cfg = self.config.write().await;
@@ -71,41 +71,46 @@ impl Imap {
             self.close_folder(context).await?;
         }
 
-        if self.config.read().await.selected_folder.as_deref() == Some(folder.as_ref()) {
+        let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref());
+        if self.config.read().await.selected_folder.as_deref() == folder_str {
             return Ok(());
         }
 
         // select new folder
-        if let Some(ref mut session) = &mut *self.session.lock().await {
-            let res = session.select(&folder).await;
+        if let Some(ref folder) = folder {
+            if let Some(ref mut session) = &mut *self.session.lock().await {
+                let res = session.select(folder).await;
 
-            // https://tools.ietf.org/html/rfc3501#section-6.3.1
-            // says that if the server reports select failure we are in
-            // authenticated (not-select) state.
+                // https://tools.ietf.org/html/rfc3501#section-6.3.1
+                // says that if the server reports select failure we are in
+                // authenticated (not-select) state.
 
-            match res {
-                Ok(mailbox) => {
-                    let mut config = self.config.write().await;
-                    config.selected_folder = Some(folder.as_ref().to_string());
-                    config.selected_mailbox = Some(mailbox);
-                    Ok(())
-                }
-                Err(async_imap::error::Error::ConnectionLost) => {
-                    self.trigger_reconnect();
-                    self.config.write().await.selected_folder = None;
-                    Err(Error::ConnectionLost)
-                }
-                Err(async_imap::error::Error::Validate(_)) => {
-                    Err(Error::BadFolderName(folder.as_ref().to_string()))
-                }
-                Err(err) => {
-                    self.config.write().await.selected_folder = None;
-                    self.trigger_reconnect();
-                    Err(Error::Other(err.to_string()))
+                match res {
+                    Ok(mailbox) => {
+                        let mut config = self.config.write().await;
+                        config.selected_folder = Some(folder.as_ref().to_string());
+                        config.selected_mailbox = Some(mailbox);
+                        Ok(())
+                    }
+                    Err(async_imap::error::Error::ConnectionLost) => {
+                        self.trigger_reconnect();
+                        self.config.write().await.selected_folder = None;
+                        Err(Error::ConnectionLost)
+                    }
+                    Err(async_imap::error::Error::Validate(_)) => {
+                        Err(Error::BadFolderName(folder.as_ref().to_string()))
+                    }
+                    Err(err) => {
+                        self.config.write().await.selected_folder = None;
+                        self.trigger_reconnect();
+                        Err(Error::Other(err.to_string()))
+                    }
                 }
+            } else {
+                Err(Error::NoSession)
             }
         } else {
-            Err(Error::NoSession)
+            Ok(())
         }
     }
 }

src/job.rs

@@ -1190,7 +1190,7 @@ fn perform_job_action(context: &Context, mut job: &mut Job, thread: Thread, trie
         Action::ImexImap => match JobImexImap(context, &job) {
             Ok(()) => Status::Finished(Ok(())),
             Err(err) => {
-                error!(context, "{}", err);
+                error!(context, "Import/export failed: {}", err);
                 Status::Finished(Err(err))
             }
         },

src/job_thread.rs

@@ -173,15 +173,14 @@ impl JobThread {
                 if !self.imap.can_idle() {
                     true // we have to do fake_idle
                 } else {
-                    if let Some(watch_folder) = self.get_watch_folder(context) {
-                        info!(context, "{} started...", prefix);
-                        let res = self.imap.idle(context, watch_folder);
-                        info!(context, "{} ended...", prefix);
-                        if let Err(err) = res {
-                            warn!(context, "{} failed: {} -> reconnecting", prefix, err);
-                            // something is Label { Label }orked, let's start afresh on the next occassion
-                            self.imap.disconnect(context);
-                        }
+                    let watch_folder = self.get_watch_folder(context);
+                    info!(context, "{} started...", prefix);
+                    let res = self.imap.idle(context, watch_folder);
+                    info!(context, "{} ended...", prefix);
+                    if let Err(err) = res {
+                        warn!(context, "{} failed: {} -> reconnecting", prefix, err);
+                        // something is borked, let's start afresh on the next occassion
+                        self.imap.disconnect(context);
                     }
                     false
                 }

src/message.rs

@@ -1215,7 +1215,7 @@ pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef
         }
         if let Some(error) = error {
             msg.param.set(Param::Error, error.as_ref());
-            error!(context, "{}", error.as_ref());
+            warn!(context, "Message failed: {}", error.as_ref());
         }
 
         if sql::execute(
@@ -1432,7 +1432,10 @@ pub(crate) fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
 ) -> Result<Option<(String, u32, MsgId)>, Error> {
-    ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
+    if rfc724_mid.is_empty() {
+        warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
+        return Ok(None);
+    }
 
     context
         .sql

src/mimeparser.rs

@@ -1,9 +1,8 @@
 use std::collections::{HashMap, HashSet};
 
-use anyhow::Context as _;
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
-use mailparse::{DispositionType, MailAddr, MailHeaderMap};
+use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
@@ -37,6 +36,11 @@ use crate::stock::StockMessage;
 pub struct MimeMessage {
     pub parts: Vec<Part>,
     header: HashMap<String, String>,
+
+    /// Addresses are normalized and lowercased:
+    pub recipients: Vec<SingleInfo>,
+    pub from: Vec<SingleInfo>,
+    pub chat_disposition_notification_to: Option<SingleInfo>,
     pub decrypting_failed: bool,
     pub signatures: HashSet<String>,
     pub gossipped_addr: HashSet<String>,
@@ -88,9 +92,19 @@ impl MimeMessage {
             .unwrap_or_default();
 
         let mut headers = Default::default();
+        let mut recipients = Default::default();
+        let mut from = Default::default();
+        let mut chat_disposition_notification_to = None;
 
         // init known headers with what mailparse provided us
-        MimeMessage::merge_headers(&mut headers, &mail.headers);
+        MimeMessage::merge_headers(
+            context,
+            &mut headers,
+            &mut recipients,
+            &mut from,
+            &mut chat_disposition_notification_to,
+            &mail.headers,
+        );
 
         // remove headers that are allowed _only_ in the encrypted part
         headers.remove("secure-join-fingerprint");
@@ -118,7 +132,14 @@ impl MimeMessage {
 
                     // let known protected headers from the decrypted
                     // part override the unencrypted top-level
-                    MimeMessage::merge_headers(&mut headers, &decrypted_mail.headers);
+                    MimeMessage::merge_headers(
+                        context,
+                        &mut headers,
+                        &mut recipients,
+                        &mut from,
+                        &mut chat_disposition_notification_to,
+                        &decrypted_mail.headers,
+                    );
 
                     (decrypted_mail, signatures)
                 } else {
@@ -142,6 +163,9 @@ impl MimeMessage {
         let mut parser = MimeMessage {
             parts: Vec::new(),
             header: headers,
+            recipients,
+            from,
+            chat_disposition_notification_to,
             decrypting_failed: false,
 
             // only non-empty if it was a valid autocrypt message
@@ -310,11 +334,9 @@ impl MimeMessage {
 
         // See if an MDN is requested from the other side
         if !self.decrypting_failed && !self.parts.is_empty() {
-            if let Some(ref dn_to_addr) =
-                self.parse_first_addr(context, HeaderDef::ChatDispositionNotificationTo)
-            {
-                if let Some(ref from_addr) = self.parse_first_addr(context, HeaderDef::From_) {
-                    if compare_addrs(from_addr, dn_to_addr) {
+            if let Some(ref dn_to) = self.chat_disposition_notification_to {
+                if let Some(ref from) = self.from.get(0) {
+                    if from.addr == dn_to.addr {
                         if let Some(part) = self.parts.last_mut() {
                             part.param.set_int(Param::WantsMdn, 1);
                         }
@@ -388,20 +410,6 @@ impl MimeMessage {
         self.header.get(headerdef.get_headername())
     }
 
-    fn parse_first_addr(&self, context: &Context, headerdef: HeaderDef) -> Option<MailAddr> {
-        if let Some(value) = self.get(headerdef.clone()) {
-            match mailparse::addrparse(&value) {
-                Ok(ref addrs) => {
-                    return addrs.first().cloned();
-                }
-                Err(err) => {
-                    warn!(context, "header {} parse error: {:?}", headerdef, err);
-                }
-            }
-        }
-        None
-    }
-
     fn parse_mime_recursive(
         &mut self,
         context: &Context,
@@ -745,17 +753,41 @@ impl MimeMessage {
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
-    fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
+    fn merge_headers(
+        context: &Context,
+        headers: &mut HashMap<String, String>,
+        recipients: &mut Vec<SingleInfo>,
+        from: &mut Vec<SingleInfo>,
+        chat_disposition_notification_to: &mut Option<SingleInfo>,
+        fields: &[mailparse::MailHeader<'_>],
+    ) {
         for field in fields {
             // lowercasing all headers is technically not correct, but makes things work better
             let key = field.get_key().to_lowercase();
             if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers)
                     is_known(&key) || key.starts_with("chat-")
             {
-                let value = field.get_value();
-                headers.insert(key.to_string(), value);
+                if key == HeaderDef::ChatDispositionNotificationTo.get_headername() {
+                    match addrparse_header(field) {
+                        Ok(addrlist) => {
+                            *chat_disposition_notification_to = addrlist.extract_single_info();
+                        }
+                        Err(e) => warn!(context, "Could not read {} address: {}", key, e),
+                    }
+                } else {
+                    let value = field.get_value();
+                    headers.insert(key.to_string(), value);
+                }
             }
         }
+        let recipients_new = get_recipients(fields);
+        if !recipients_new.is_empty() {
+            *recipients = recipients_new;
+        }
+        let from_new = get_from(fields);
+        if !from_new.is_empty() {
+            *from = from_new;
+        }
     }
 
     fn process_report(
@@ -823,23 +855,15 @@ fn update_gossip_peerstates(
     gossip_headers: Vec<String>,
 ) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
-    let mut recipients: Option<HashSet<String>> = None;
     let mut gossipped_addr: HashSet<String> = Default::default();
 
     for value in &gossip_headers {
         let gossip_header = value.parse::<Aheader>();
 
         if let Ok(ref header) = gossip_header {
-            if recipients.is_none() {
-                recipients = Some(get_recipients(
-                    mail.headers.iter().map(|v| (v.get_key(), v.get_value())),
-                ));
-            }
-
-            if recipients
-                .as_ref()
-                .unwrap()
-                .contains(&header.addr.to_lowercase())
+            if get_recipients(&mail.headers)
+                .iter()
+                .any(|info| info.addr == header.addr.to_lowercase())
             {
                 let mut peerstate = Peerstate::from_addr(context, &context.sql, &header.addr);
                 if let Some(ref mut peerstate) = peerstate {
@@ -877,13 +901,29 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
-pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
-    let ids = mailparse::msgidparse(value).context("failed to parse message id")?;
+pub(crate) fn parse_message_ids(ids: &str) -> Result<Vec<String>> {
+    // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>`
+    let mut msgids = Vec::new();
+    for id in ids.split_whitespace() {
+        let mut id = id.to_string();
+        if id.starts_with('<') {
+            id = id[1..].to_string();
+        }
+        if id.ends_with('>') {
+            id = id[..id.len() - 1].to_string();
+        }
+        if !id.is_empty() {
+            msgids.push(id);
+        }
+    }
+    Ok(msgids)
+}
 
-    if let Some(id) = ids.first() {
+pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
+    if let Some(id) = parse_message_ids(ids)?.first() {
         Ok(id.to_string())
     } else {
-        bail!("could not parse message_id: {}", value);
+        bail!("could not parse message_id: {}", ids);
     }
 }
 
@@ -984,6 +1024,11 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
     let desired_filename =
         desired_filename.or_else(|| ct.params.get("name").map(|s| s.to_string()));
 
+    // MS Outlook is known to specify filename in the "name" attribute of
+    // Content-Type and omit Content-Disposition.
+    let desired_filename =
+        desired_filename.or_else(|| mail.ctype.params.get("name").map(|s| s.to_string()));
+
     // If there is no filename, but part is an attachment, guess filename
     if ct.disposition == DispositionType::Attachment && desired_filename.is_none() {
         if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
@@ -999,50 +1044,50 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String
     }
 }
 
-// returned addresses are normalized and lowercased.
-fn get_recipients<S: AsRef<str>, T: Iterator<Item = (S, S)>>(headers: T) -> HashSet<String> {
-    let mut recipients: HashSet<String> = Default::default();
+/// Returned addresses are normalized and lowercased.
+fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
+    get_all_addresses_from_header(headers, |header_key| {
+        header_key == "to" || header_key == "cc"
+    })
+}
 
-    for (hkey, hvalue) in headers {
-        let hkey = hkey.as_ref().to_lowercase();
-        let hvalue = hvalue.as_ref();
-        if hkey == "to" || hkey == "cc" {
-            if let Ok(addrs) = mailparse::addrparse(hvalue) {
-                for addr in addrs.iter() {
-                    match addr {
-                        mailparse::MailAddr::Single(ref info) => {
-                            recipients.insert(addr_normalize(&info.addr).to_lowercase());
-                        }
-                        mailparse::MailAddr::Group(ref infos) => {
-                            for info in &infos.addrs {
-                                recipients.insert(addr_normalize(&info.addr).to_lowercase());
-                            }
+/// Returned addresses are normalized and lowercased.
+pub(crate) fn get_from(headers: &[MailHeader]) -> Vec<SingleInfo> {
+    get_all_addresses_from_header(headers, |header_key| header_key == "from")
+}
+
+fn get_all_addresses_from_header<F>(headers: &[MailHeader], pred: F) -> Vec<SingleInfo>
+where
+    F: Fn(String) -> bool,
+{
+    let mut result: Vec<SingleInfo> = Default::default();
+
+    headers
+        .iter()
+        .filter(|header| pred(header.get_key().to_lowercase()))
+        .filter_map(|header| mailparse::addrparse_header(header).ok())
+        .for_each(|addrs| {
+            for addr in addrs.iter() {
+                match addr {
+                    mailparse::MailAddr::Single(ref info) => {
+                        result.push(SingleInfo {
+                            addr: addr_normalize(&info.addr).to_lowercase(),
+                            display_name: info.display_name.clone(),
+                        });
+                    }
+                    mailparse::MailAddr::Group(ref infos) => {
+                        for info in &infos.addrs {
+                            result.push(SingleInfo {
+                                addr: addr_normalize(&info.addr).to_lowercase(),
+                                display_name: info.display_name.clone(),
+                            });
                         }
                     }
                 }
             }
-        }
-    }
+        });
 
-    recipients
-}
-
-/// Check if the only addrs match, ignoring names.
-fn compare_addrs(a: &mailparse::MailAddr, b: &mailparse::MailAddr) -> bool {
-    match a {
-        mailparse::MailAddr::Group(group_a) => match b {
-            mailparse::MailAddr::Group(group_b) => group_a
-                .addrs
-                .iter()
-                .zip(group_b.addrs.iter())
-                .all(|(a, b)| a.addr == b.addr),
-            _ => false,
-        },
-        mailparse::MailAddr::Single(single_a) => match b {
-            mailparse::MailAddr::Single(single_b) => single_a.addr == single_b.addr,
-            _ => false,
-        },
-    }
+    result
 }
 
 #[cfg(test)]
@@ -1091,12 +1136,11 @@ mod tests {
 
     #[test]
     fn test_get_recipients() {
-        let context = dummy_context();
         let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
-        let recipients = get_recipients(mimeparser.header.iter());
-        assert!(recipients.contains("abc@bcd.com"));
-        assert!(recipients.contains("def@def.de"));
+        let mail = mailparse::parse_mail(&raw[..]).unwrap();
+        let recipients = get_recipients(&mail.headers);
+        assert!(recipients.iter().any(|info| info.addr == "abc@bcd.com"));
+        assert!(recipients.iter().any(|info| info.addr == "def@def.de"));
         assert_eq!(recipients.len(), 2);
     }
 
@@ -1151,14 +1195,10 @@ mod tests {
 
         let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
 
-        let of = mimeparser
-            .parse_first_addr(&context.ctx, HeaderDef::From_)
-            .unwrap();
-        assert_eq!(of, mailparse::addrparse("hello@one.org").unwrap()[0]);
+        let of = &mimeparser.from[0];
+        assert_eq!(of.addr, "hello@one.org");
 
-        let of =
-            mimeparser.parse_first_addr(&context.ctx, HeaderDef::ChatDispositionNotificationTo);
-        assert!(of.is_none());
+        assert!(mimeparser.chat_disposition_notification_to.is_none());
     }
 
     #[test]
@@ -1616,4 +1656,142 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
         assert_eq!(message.parts[0].typ, Viewtype::Image);
         assert_eq!(message.parts[0].msg, "Test");
     }
+
+    // Outlook specifies filename in the "name" attribute of Content-Type
+    #[test]
+    fn parse_outlook_html_embedded_image() {
+        let context = dummy_context();
+        let raw = br##"From: Anonymous <anonymous@example.org>
+To: Anonymous <anonymous@example.org>
+Subject: Delta Chat is great stuff!
+Date: Tue, 5 May 2020 01:23:45 +0000
+MIME-Version: 1.0
+Content-Type: multipart/related;
+	boundary="----=_NextPart_000_0003_01D622B3.CA753E60"
+X-Mailer: Microsoft Outlook 15.0
+
+This is a multipart message in MIME format.
+
+------=_NextPart_000_0003_01D622B3.CA753E60
+Content-Type: multipart/alternative;
+	boundary="----=_NextPart_001_0004_01D622B3.CA753E60"
+
+
+------=_NextPart_001_0004_01D622B3.CA753E60
+Content-Type: text/plain;
+	charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+
+
+
+------=_NextPart_001_0004_01D622B3.CA753E60
+Content-Type: text/html;
+	charset="us-ascii"
+Content-Transfer-Encoding: quoted-printable
+
+<html>
+<body>
+<p>
+Test<img src="cid:image001.jpg@01D622B3.C9D8D750">
+</p>
+</body>
+</html>
+------=_NextPart_001_0004_01D622B3.CA753E60--
+
+------=_NextPart_000_0003_01D622B3.CA753E60
+Content-Type: image/jpeg;
+	name="image001.jpg"
+Content-Transfer-Encoding: base64
+Content-ID: <image001.jpg@01D622B3.C9D8D750>
+
+ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
+acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
+/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
+n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
+xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
+rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
+8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
+K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
+CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
+
+------=_NextPart_000_0003_01D622B3.CA753E60--
+"##;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Delta Chat is great stuff!".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Image);
+        assert_eq!(message.parts[0].msg, "Test");
+    }
+
+    #[test]
+    fn test_parse_message_id() {
+        let test = parse_message_id("<foobar>");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foobar");
+
+        let test = parse_message_id("<foo> <bar>");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("  < foo > <bar>");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("foo");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id(" foo ");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("foo bar");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("  foo  bar ");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "foo");
+
+        let test = parse_message_id("");
+        assert!(test.is_err());
+
+        let test = parse_message_id(" ");
+        assert!(test.is_err());
+
+        let test = parse_message_id("<>");
+        assert!(test.is_err());
+
+        let test = parse_message_id("<> bar");
+        assert!(test.is_ok());
+        assert_eq!(test.unwrap(), "bar");
+    }
+
+    #[test]
+    fn test_parse_message_ids() {
+        let test = parse_message_ids("  foo  bar <foobar>").unwrap();
+        assert_eq!(test.len(), 3);
+        assert_eq!(test[0], "foo");
+        assert_eq!(test[1], "bar");
+        assert_eq!(test[2], "foobar");
+
+        let test = parse_message_ids("  < foobar >").unwrap();
+        assert_eq!(test.len(), 1);
+        assert_eq!(test[0], "foobar");
+
+        let test = parse_message_ids("").unwrap();
+        assert!(test.is_empty());
+
+        let test = parse_message_ids(" ").unwrap();
+        assert!(test.is_empty());
+
+        let test = parse_message_ids("  < ").unwrap();
+        assert!(test.is_empty());
+    }
 }

src/provider/data.rs

@@ -18,6 +18,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // aol.md: aol.com
@@ -28,6 +29,7 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/aol",
         server: vec![
         ],
+        config_defaults: None,
     };
 
     // autistici.org.md: autistici.org
@@ -40,6 +42,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mail.autistici.org", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -52,6 +55,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imaps.bluewin.ch", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
@@ -73,6 +77,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.example.com", port: 1337, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // fastmail.md: fastmail.com
@@ -83,6 +88,7 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/fastmail",
         server: vec![
         ],
+        config_defaults: None,
     };
 
     // freenet.de.md: freenet.de
@@ -95,6 +101,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mx.freenet.de", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -107,6 +114,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.gmail.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -120,6 +128,7 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "mail.gmx.net", port: 465, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // i.ua.md: i.ua
@@ -135,6 +144,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.me.com", port: 993, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // kolst.com.md: kolst.com
@@ -159,6 +169,15 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.nauta.cu", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.nauta.cu", port: 25, username_pattern: EMAIL },
         ],
+        config_defaults: Some(vec![
+            ConfigDefault { key: Config::DeleteServerAfter, value: "1" },
+            ConfigDefault { key: Config::BccSelf, value: "0" },
+            ConfigDefault { key: Config::SentboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxWatch, value: "0" },
+            ConfigDefault { key: Config::MvboxMove, value: "0" },
+            ConfigDefault { key: Config::E2eeEnabled, value: "0" },
+            ConfigDefault { key: Config::MediaQuality, value: "1" },
+        ]),
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -171,6 +190,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap-mail.outlook.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // posteo.md: posteo.de
@@ -183,6 +203,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "posteo.de", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -193,6 +214,7 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/protonmail",
         server: vec![
         ],
+        config_defaults: None,
     };
 
     // riseup.net.md: riseup.net
@@ -201,6 +223,9 @@ lazy_static::lazy_static! {
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
+    // systemli.org.md: systemli.org
+    // - skipping provider with status OK and no special things to do
+
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
         status: Status::PREPARATION,
@@ -209,6 +234,7 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/t-online",
         server: vec![
         ],
+        config_defaults: None,
     };
 
     // testrun.md: testrun.org
@@ -222,6 +248,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // tiscali.it.md: tiscali.it
@@ -234,6 +261,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.tiscali.it", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // ukr.net.md: ukr.net
@@ -253,6 +281,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.web.de", port: 143, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
+        config_defaults: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
@@ -265,6 +294,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
@@ -275,6 +305,7 @@ lazy_static::lazy_static! {
         overview_page: "https://providers.delta.chat/yandex-ru",
         server: vec![
         ],
+        config_defaults: None,
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -287,6 +318,7 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.ziggo.nl", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
+        config_defaults: None,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [

src/provider/mod.rs

@@ -2,6 +2,7 @@
 
 mod data;
 
+use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
@@ -57,6 +58,12 @@ impl Server {
     }
 }
 
+#[derive(Debug)]
+pub struct ConfigDefault {
+    pub key: Config,
+    pub value: &'static str,
+}
+
 #[derive(Debug)]
 pub struct Provider {
     pub status: Status,
@@ -64,6 +71,7 @@ pub struct Provider {
     pub after_login_hint: &'static str,
     pub overview_page: &'static str,
     pub server: Vec<Server>,
+    pub config_defaults: Option<Vec<ConfigDefault>>,
 }
 
 impl Provider {

src/provider/update.py

@@ -9,6 +9,9 @@ out_all = ""
 out_domains = ""
 domains_dict = {}
 
+def camel(name):
+    words = name.split("_")
+    return "".join(w.capitalize() for i, w in enumerate(words))
 
 def cleanstr(s):
     s = s.strip()
@@ -31,6 +34,18 @@ def file2url(f):
     return "https://providers.delta.chat/" + f
 
 
+def process_config_defaults(data):
+    if not "config_defaults" in data:
+        return "None"
+    defaults = "Some(vec![\n"
+    config_defaults = data.get("config_defaults", "")
+    for key in config_defaults:
+        value = str(config_defaults[key])
+        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
+    defaults += "        ])"
+    return defaults
+
+
 def process_data(data, file):
     status = data.get("status", "")
     if status != "OK" and status != "PREPARATION" and status != "BROKEN":
@@ -83,6 +98,8 @@ def process_data(data, file):
             server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
+    config_defaults = process_config_defaults(data)
+
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -93,6 +110,7 @@ def process_data(data, file):
         provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
+        provider += "        config_defaults: " + config_defaults + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -103,7 +121,7 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "":
+    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
         out_all += "    // - skipping provider with status OK and no special things to do\n\n"
     else:
         out_all += provider

src/smtp/mod.rs

@@ -12,6 +12,7 @@ use crate::context::Context;
 use crate::events::Event;
 use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
+use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -171,7 +172,14 @@ impl Smtp {
         let mut trans = client.into_transport();
 
         trans.connect().await.map_err(|err| {
-            emit_event!(context, Event::ErrorNetwork(err.to_string()));
+            let message = {
+                context.stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("SMTP {}:{}", domain, port),
+                    err.to_string(),
+                )
+            };
+            emit_event!(context, Event::ErrorNetwork(message));
             Error::ConnectionFailure(err)
         })?;
 

src/sql.rs

@@ -242,7 +242,7 @@ impl Sql {
         match self.query_get_value_result(query, params) {
             Ok(res) => res,
             Err(err) => {
-                error!(context, "sql: Failed query_row: {}", err);
+                warn!(context, "sql: Failed query_row: {}", err);
                 None
             }
         }
@@ -400,7 +400,7 @@ fn open(
     }
     let mgr = r2d2_sqlite::SqliteConnectionManager::file(dbfile.as_ref())
         .with_flags(open_flags)
-        .with_init(|c| c.execute_batch("PRAGMA secure_delete=on;"));
+        .with_init(|c| c.execute_batch("PRAGMA journal_mode=WAL; PRAGMA secure_delete=on;"));
     let pool = r2d2::Pool::builder()
         .min_idle(Some(2))
         .max_size(10)