snap

* update changelog for 1.80.0

* bump version to 1.80.0

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.80.0
+
+### Changes
+- update provider database #3284
+- improve python bindings, tests and ci #3287 #3286 #3287 #3289 #3290 #3292
+
+### Fixes
+- fix escaping in generated QR-code-SVG #3295
+
+
 ## 1.79.0
 
 ### Changes

Cargo.lock

@@ -1067,7 +1067,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.79.0"
+version = "1.80.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1146,7 +1146,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.79.0"
+version = "1.80.0"
 dependencies = [
  "anyhow",
  "async-std",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.79.0"
+version = "1.80.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.79.0"
+version = "1.80.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/src/lib.rs

@@ -657,7 +657,7 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
 #[no_mangle]
 pub unsafe extern "C" fn dc_stop_io(context: *mut dc_context_t) {
     if context.is_null() {
-        eprintln!("ignoring careless call to dc_shutdown()");
+        eprintln!("ignoring careless call to dc_stop_io()");
         return;
     }
     let ctx = &*context;

python/examples/test_examples.py

@@ -22,7 +22,7 @@ def test_echo_quit_plugin(acfactory, lp):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
     lp.sec("creating a temp account to contact the bot")
-    ac1 = acfactory.get_one_online_account()
+    ac1, = acfactory.get_online_accounts(1)
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -42,7 +42,7 @@ def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
     botproc = acfactory.run_bot_process(group_tracking, ffi=False)
 
-    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
+    ac1, ac2 = acfactory.get_online_accounts(2)
 
     botproc.fnmatch_lines("""
         *ac_configure_completed*

python/mypy.ini

@@ -1,5 +1,8 @@
 [mypy]
 
+[mypy-py.*]
+ignore_missing_imports = True
+
 [mypy-deltachat.capi.*]
 ignore_missing_imports = True
 

python/src/deltachat/__init__.py

@@ -2,7 +2,7 @@ import sys
 
 from . import capi, const, hookspec # noqa
 from .capi import ffi  # noqa
-from .account import Account  # noqa
+from .account import Account, get_core_info  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa

python/src/deltachat/account.py

@@ -22,6 +22,29 @@ class MissingCredentials(ValueError):
     """ Account is missing `addr` and `mail_pw` config values. """
 
 
+def get_core_info():
+    """ get some system info. """
+    from tempfile import NamedTemporaryFile
+
+    with NamedTemporaryFile() as path:
+        path.close()
+        return get_dc_info_as_dict(ffi.gc(
+            lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
+            lib.dc_context_unref,
+        ))
+
+
+def get_dc_info_as_dict(dc_context):
+    lines = from_dc_charpointer(lib.dc_get_info(dc_context))
+    info_dict = {}
+    for line in lines.split("\n"):
+        if not line.strip():
+            continue
+        key, value = line.split("=", 1)
+        info_dict[key.lower()] = value
+    return info_dict
+
+
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
@@ -67,6 +90,9 @@ class Account(object):
         """ re-enable logging. """
         self._logging = True
 
+    def __repr__(self):
+        return "<Account path={}>".format(self.db_path)
+
     # def __del__(self):
     #    self.shutdown()
 
@@ -81,14 +107,7 @@ class Account(object):
 
     def get_info(self) -> Dict[str, str]:
         """ return dictionary of built config parameters. """
-        lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
-        d = {}
-        for line in lines.split("\n"):
-            if not line.strip():
-                continue
-            key, value = line.split("=", 1)
-            d[key.lower()] = value
-        return d
+        return get_dc_info_as_dict(self._dc_context)
 
     def dump_account_info(self, logfile):
         def log(*args, **kwargs):
@@ -129,6 +148,8 @@ class Account(object):
         namebytes = name.encode("utf8")
         if namebytes == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
+        if isinstance(value, (int, bool)):
+            value = str(int(value))
         if value is not None:
             valuebytes = value.encode("utf8")
         else:
@@ -169,7 +190,7 @@ class Account(object):
         :returns: None
         """
         for key, value in kwargs.items():
-            self.set_config(key, str(value))
+            self.set_config(key, value)
 
     def is_configured(self) -> bool:
         """ determine if the account is configured already; an initial connection
@@ -569,6 +590,8 @@ class Account(object):
         """ add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
         """
+        if name and self._pm.has_plugin(name=name):
+            self._pm.unregister(name=name)
         self._pm.register(plugin, name=name)
         self._pm.check_pending()
         return plugin
@@ -672,28 +695,25 @@ class Account(object):
         if self._dc_context is None:
             return
 
-        self.stop_io()
-
-        self.log("remove dc_context references")
-
-        # if _dc_context is unref'ed the event thread should quickly
-        # receive the termination signal. However, some python code might
-        # still hold a reference and so we use a secondary signal
-        # to make sure the even thread terminates if it receives any new
-        # event, indepedently from waiting for the core to send NULL to
-        # get_next_event().
+        # mark the event thread for shutdown (latest on next incoming event)
         self._event_thread.mark_shutdown()
-        self._dc_context = None
+
+        # stop_io also causes an info event which will wake up
+        # the EventThread's inner loop and let it notice the shutdown marker.
+        self.stop_io()
 
         self.log("wait for event thread to finish")
         try:
-            self._event_thread.wait(timeout=2)
+            self._event_thread.wait(timeout=5)
         except RuntimeError as e:
             self.log("Waiting for event thread failed: {}".format(e))
 
         if self._event_thread.is_alive():
             self.log("WARN: event thread did not terminate yet, ignoring.")
 
+        self.log("remove dc_context references, making the Account unusable")
+        self._dc_context = None
+
         self._shutdown_event.set()
 
         hook = hookspec.Global._get_plugin_manager().hook

python/src/deltachat/direct_imap.py

@@ -6,9 +6,9 @@ and for cleaning up inbox/mvbox for each test function run.
 import io
 import ssl
 import pathlib
+from contextlib import contextmanager
 from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
 import imaplib
-import deltachat
 from deltachat import const, Account
 from typing import List
 
@@ -18,44 +18,6 @@ FETCH = b'FETCH'
 ALL = "1:*"
 
 
-@deltachat.global_hookimpl
-def dc_account_extra_configure(account: Account):
-    """ Reset the account (we reuse accounts across tests)
-    and make 'account.direct_imap' available for direct IMAP ops.
-    """
-    try:
-
-        if not hasattr(account, "direct_imap"):
-            imap = DirectImap(account)
-
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete(ALL, expunge=True)
-                else:
-                    imap.conn.folder.delete(folder)
-                    # We just deleted the folder, so we have to make DC forget about it, too
-                    if account.get_config("configured_sentbox_folder") == folder:
-                        account.set_config("configured_sentbox_folder", None)
-
-            setattr(account, "direct_imap", imap)
-
-    except Exception as e:
-        # Uncaught exceptions here would lead to a timeout without any note written to the log
-        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
-        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
-        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
-
-
-@deltachat.global_hookimpl
-def dc_account_after_shutdown(account):
-    """ shutdown the imap connection if there is one. """
-    imap = getattr(account, "direct_imap", None)
-    if imap is not None:
-        imap.shutdown()
-        del account.direct_imap
-
-
 class DirectImap:
     def __init__(self, account: Account) -> None:
         self.account = account
@@ -91,10 +53,6 @@ class DirectImap:
         self.select_folder("INBOX")
 
     def shutdown(self):
-        try:
-            self.idle_done()
-        except (OSError, imaplib.IMAP4.abort):
-            pass
         try:
             self.conn.logout()
         except (OSError, imaplib.IMAP4.abort):
@@ -190,49 +148,14 @@ class DirectImap:
 
         print(stream.getvalue(), file=logfile)
 
-    def idle_start(self):
-        """ switch this connection to idle mode. non-blocking. """
-        assert not self._idling
-        res = self.conn.idle.start()
-        self._idling = True
-        return res
-
-    def idle_check(self, terminate=False, timeout=None) -> List[bytes]:
-        """ (blocking) wait for next idle message from server. """
-        assert self._idling
-        self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle.poll(timeout=timeout)
-        if terminate:
-            self.idle_done()
-        self.account.log("imap-direct: idle_check returned {!r}".format(res))
-        return res
-
-    def idle_wait_for_new_message(self, terminate=False, timeout=None) -> bytes:
-        while 1:
-            for item in self.idle_check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
-                    if terminate:
-                        self.idle_done()
-                    return item
-
-    def idle_wait_for_seen(self, terminate=False, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
-        """
-        while 1:
-            for item in self.idle_check(timeout=timeout):
-                if FETCH in item:
-                    self.account.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        if terminate:
-                            self.idle_done()
-                        return int(item.split(b' ')[1])
-
-    def idle_done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        if self._idling:
-            res = self.conn.idle.stop()
-            self._idling = False
-            return res
+    @contextmanager
+    def idle(self):
+        """ return Idle ContextManager. """
+        idle_manager = IdleManager(self)
+        try:
+            yield idle_manager
+        finally:
+            idle_manager.done()
 
     def append(self, folder: str, msg: str):
         """Upload a message to *folder*.
@@ -248,3 +171,41 @@ class DirectImap:
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]
+
+
+class IdleManager:
+    def __init__(self, direct_imap):
+        self.direct_imap = direct_imap
+        self.log = direct_imap.account.log
+        # fetch latest messages before starting idle so that it only
+        # returns messages that arrive anew
+        self.direct_imap.conn.fetch("1:*")
+        self.direct_imap.conn.idle.start()
+
+    def check(self, timeout=None) -> List[bytes]:
+        """ (blocking) wait for next idle message from server. """
+        self.log("imap-direct: calling idle_check")
+        res = self.direct_imap.conn.idle.poll(timeout=timeout)
+        self.log("imap-direct: idle_check returned {!r}".format(res))
+        return res
+
+    def wait_for_new_message(self, timeout=None) -> bytes:
+        while 1:
+            for item in self.check(timeout=timeout):
+                if b'EXISTS' in item or b'RECENT' in item:
+                    return item
+
+    def wait_for_seen(self, timeout=None) -> int:
+        """ Return first message with SEEN flag from a running idle-stream.
+        """
+        while 1:
+            for item in self.check(timeout=timeout):
+                if FETCH in item:
+                    self.log(str(item))
+                    if FLAGS in item and rb'\Seen' in item:
+                        return int(item.split(b' ')[1])
+
+    def done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        res = self.direct_imap.conn.idle.stop()
+        return res

python/src/deltachat/events.py

@@ -1,5 +1,8 @@
 import threading
+import sys
+import traceback
 import time
+import io
 import re
 import os
 from queue import Queue, Empty
@@ -29,10 +32,12 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account) -> None:
+    def __init__(self, account, logid=None, init_time=None) -> None:
         self.account = account
-        self.logid = self.account.get_config("displayname")
-        self.init_time = time.time()
+        self.logid = logid or self.account.get_config("displayname")
+        if init_time is None:
+            init_time = time.time()
+        self.init_time = init_time
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
@@ -156,14 +161,14 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
-    def wait_all_initial_fetches(self):
+    def wait_idle_inbox_ready(self):
         """Has to be called after start_io() to wait for fetch_existing_msgs to run
         so that new messages are not mistaken for old ones:
         - ac1 and ac2 are created
         - ac1 sends a message to ac2
         - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
         - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("Done fetching existing messages")
+        self.get_info_contains("INBOX: Idle entering")
 
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
@@ -225,9 +230,7 @@ class EventThread(threading.Thread):
         )
         while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL:
-                break
-            if self._marked_for_shutdown:
+            if event == ffi.NULL or self._marked_for_shutdown:
                 break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
@@ -241,15 +244,23 @@ class EventThread(threading.Thread):
 
             lib.dc_event_unref(event)
             ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            try:
+            with self.swallow_and_log_exception("ac_process_ffi_event {}".format(ffi_event)):
                 self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-                for name, kwargs in self._map_ffi_event(ffi_event):
-                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
-                    hook = getattr(self.account._pm.hook, name)
+            for name, kwargs in self._map_ffi_event(ffi_event):
+                hook = getattr(self.account._pm.hook, name)
+                info = "call {} kwargs={} failed".format(name, kwargs)
+                with self.swallow_and_log_exception(info):
                     hook(**kwargs)
-            except Exception:
-                if self.account._dc_context is not None:
-                    raise
+
+    @contextmanager
+    def swallow_and_log_exception(self, info):
+        try:
+            yield
+        except Exception as ex:
+            logfile = io.StringIO()
+            traceback.print_exception(*sys.exc_info(), file=logfile)
+            self.account.log("{}\nException {}\nTraceback:\n{}"
+                             .format(info, ex, logfile.getvalue()))
 
     def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name

python/src/deltachat/testplugin.py

@@ -8,35 +8,43 @@ import threading
 import fnmatch
 import time
 import weakref
-import tempfile
-from typing import List, Dict, Callable
+from queue import Queue
+from typing import List, Callable
 
 import pytest
 import requests
+import py
 
-from . import Account, const
-from .capi import lib
+from . import Account, const, account_hookimpl, get_core_info
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
-from deltachat import direct_imap
 
 import deltachat
 
 
 def pytest_addoption(parser):
-    parser.addoption(
+    group = parser.getgroup("deltachat testplugin options")
+    group.addoption(
         "--liveconfig", action="store", default=None,
         help="a file with >=2 lines where each line "
              "contains NAME=VALUE config settings for one account"
     )
-    parser.addoption(
+    group.addoption(
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    parser.addoption(
+    group.addoption(
         "--strict-tls", action="store_true",
         help="Never accept invalid TLS certificates for test accounts",
     )
+    group.addoption(
+        "--extra-info", action="store_true",
+        help="show more info on failures (imap server state, config)"
+    )
+    group.addoption(
+        "--debug-setup", action="store_true",
+        help="show events during configure and start io phases of online accounts"
+    )
 
 
 def pytest_configure(config):
@@ -91,9 +99,12 @@ def pytest_configure(config):
 
         @pytest.hookimpl(hookwrapper=True)
         def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
+            logging = item.config.getoption("--extra-info")
+            if logging:
+                self.enable_logging(item)
             yield
-            self.disable_logging(item)
+            if logging:
+                self.disable_logging(item)
 
     la = LoggingAspect()
     config.pluginmanager.register(la)
@@ -101,20 +112,12 @@ def pytest_configure(config):
 
 
 def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    try:
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-         info['journal_mode'],
-     )])
+    info = get_core_info()
+    summary = ['Deltachat core={} sqlite={} journal_mode={}'.format(
+        info['deltachat_core_version'],
+        info['sqlite_version'],
+        info['journal_mode'],
+    )]
 
     cfg = config.option.liveconfig
     if cfg:
@@ -126,57 +129,99 @@ def pytest_report_header(config, startdir):
     return summary
 
 
-class SessionLiveConfigFromFile:
-    def __init__(self, fn) -> None:
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index: int):
-        return self.configlist[index]
-
-    def exists(self) -> bool:
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    configlist: List[Dict[str, str]]
-
-    def __init__(self, url: str) -> None:
-        self.configlist = []
-        self.url = url
-
-    def get(self, index: int):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self) -> bool:
-        return bool(self.configlist)
-
-
 @pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
+def testprocess(request):
+    return TestProcess(pytestconfig=request.config)
+
+
+class TestProcess:
+    """ A pytest session-scoped instance to help with managing "live" account configurations.
+    """
+    def __init__(self, pytestconfig):
+        self.pytestconfig = pytestconfig
+        self._addr2files = {}
+        self._configlist = []
+
+    def get_liveconfig_producer(self):
+        """ provide live account configs, cached on a per-test-process scope
+        so that test functions can re-use already known live configs.
+        Depending on the --liveconfig option this comes from
+        a HTTP provider or a file with a line specifying each accounts config.
+        """
+        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
+        if not liveconfig_opt:
+            pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts")
+
+        if not liveconfig_opt.startswith("http"):
+            for line in open(liveconfig_opt):
+                if line.strip() and not line.strip().startswith('#'):
+                    d = {}
+                    for part in line.split():
+                        name, value = part.split("=")
+                        d[name] = value
+                    self._configlist.append(d)
+
+            yield from iter(self._configlist)
         else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
+            MAX_LIVE_CREATED_ACCOUNTS = 10
+            for index in range(MAX_LIVE_CREATED_ACCOUNTS):
+                try:
+                    yield self._configlist[index]
+                except IndexError:
+                    res = requests.post(liveconfig_opt)
+                    if res.status_code != 200:
+                        pytest.fail("newtmpuser count={} code={}: '{}'".format(
+                                    index, res.status_code, res.text))
+                    d = res.json()
+                    config = dict(addr=d["email"], mail_pw=d["password"])
+                    print("newtmpuser {}: addr={}".format(index, config["addr"]))
+                    self._configlist.append(config)
+                    yield config
+            pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS))
+
+    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
+        db_target_path = py.path.local(db_target_path)
+        assert not db_target_path.exists()
+
+        print("checking cache for", cache_addr)
+        try:
+            filescache = self._addr2files[cache_addr]
+        except KeyError:
+            print("CACHE FAIL for", cache_addr)
+            return False
+        else:
+            print("CACHE HIT for", cache_addr)
+            targetdir = db_target_path.dirpath()
+            write_dict_to_dir(filescache, targetdir)
+            return True
+
+    def cache_maybe_store_configured_db_files(self, acc):
+        addr = acc.get_config("addr")
+        assert acc.is_configured()
+        # don't overwrite existing entries
+        if addr not in self._addr2files:
+            print("storing cache for", addr)
+            basedir = py.path.local(acc.get_blobdir()).dirpath()
+            self._addr2files[addr] = create_dict_from_files_in_path(basedir)
+            return True
+
+
+def create_dict_from_files_in_path(path):
+    base = py.path.local(path)
+    cachedict = {}
+    for path in base.visit(fil=py.path.local.isfile):
+        cachedict[path.relto(base)] = path.read_binary()
+    return cachedict
+
+
+def write_dict_to_dir(dic, target_dir):
+    assert dic
+    target_dir = py.path.local(target_dir)
+    for relpath, content in dic.items():
+        path = target_dir.join(relpath)
+        if not path.dirpath().exists():
+            path.dirpath().ensure(dir=1)
+        path.write_binary(content)
 
 
 @pytest.fixture
@@ -209,256 +254,408 @@ def data(request):
     return Data()
 
 
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
+class ACSetup:
+    """ accounts setup helper to deal with multiple configure-process
+    and io & imap initialization phases. From tests, use the higher level
+    public ACFactory methods instead of its private helper class.
+    """
+    CONFIGURING = "CONFIGURING"
+    CONFIGURED = "CONFIGURED"
+    IDLEREADY = "IDLEREADY"
 
-    class AccountMaker:
-        _finalizers: List[Callable[[], None]]
-        _accounts: List[Account]
+    def __init__(self, testprocess, init_time):
+        self._configured_events = Queue()
+        self._account2state = {}
+        self._imap_cleaned = set()
+        self.testprocess = testprocess
+        self.init_time = init_time
 
-        def __init__(self) -> None:
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self._accounts = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
+    def log(self, *args):
+        print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args)
+
+    def add_configured(self, account):
+        """ add an already configured account. """
+        assert account.is_configured()
+        self._account2state[account] = self.CONFIGURED
+        self.log("added already configured account", account, account.get_config("addr"))
+        return
+
+    def start_configure(self, account, reconfigure=False):
+        """ add an account and start its configure process. """
+
+        if reconfigure:
+            assert account.is_configured()
+
+        class PendingTracker:
+            @account_hookimpl
+            def ac_configure_completed(this, success):
+                self._configured_events.put((account, success))
+
+        account.add_account_plugin(PendingTracker(), name="pending_tracker")
+        self._account2state[account] = self.CONFIGURING
+        account.configure(reconfigure=reconfigure)
+        self.log("started {}configure on".format("re-" if reconfigure else ""), account)
+
+    def wait_one_configured(self, account):
+        """ wait until this account has successfully configured. """
+        if self._account2state[account] == self.CONFIGURING:
+            while 1:
+                acc = self._pop_config_success()
+                if acc == account:
+                    break
+            self.init_imap(acc)
+            self.init_logging(acc)
+            acc._evtracker.consume_events()
+
+    def wait_all_configured(self):
+        """ Wait for all unconfigured accounts to become finished. """
+        while self.CONFIGURING in self._account2state.values():
+            self._pop_config_success()
+
+    def bring_online(self):
+        """ Wait for all accounts to become ready to receive messages.
+
+        This will initialize logging, start IO and the direct_imap attribute
+        for each account which either is CONFIGURED already or which is CONFIGURING
+        and successfully completing the configuration process.
+        """
+        print("wait_all_configured finds accounts=", self._account2state)
+        for acc, state in self._account2state.items():
+            if state == self.CONFIGURED:
+                self._onconfigure_start_io(acc)
+                self._account2state[acc] = self.IDLEREADY
+
+        while self.CONFIGURING in self._account2state.values():
+            acc = self._pop_config_success()
+            self._onconfigure_start_io(acc)
+            self._account2state[acc] = self.IDLEREADY
+        print("finished, account2state", self._account2state)
+
+    def _pop_config_success(self):
+        acc, success = self._configured_events.get()
+        if not success:
+            pytest.fail("configuring online account failed: {}".format(acc))
+        self._account2state[acc] = self.CONFIGURED
+        return acc
+
+    def _onconfigure_start_io(self, acc):
+        self.init_imap(acc)
+        self.init_logging(acc)
+        acc.start_io()
+        print(acc._logid, "waiting for inbox IDLE to become ready")
+        acc._evtracker.wait_idle_inbox_ready()
+        acc._evtracker.consume_events()
+        acc.log("inbox IDLE ready")
+
+    def init_logging(self, acc):
+        """ idempotent function for initializing logging (will replace existing logger). """
+        logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time)
+        acc.add_account_plugin(logger, name="logger-" + acc._logid)
+
+    def init_imap(self, acc):
+        """ initialize direct_imap and cleanup server state. """
+        from deltachat.direct_imap import DirectImap
+
+        assert acc.is_configured()
+        if not hasattr(acc, "direct_imap"):
+            acc.direct_imap = DirectImap(acc)
+        addr = acc.get_config("addr")
+        if addr not in self._imap_cleaned:
+            imap = acc.direct_imap
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete("1:*", expunge=True)
+                else:
+                    imap.conn.folder.delete(folder)
+            acc.log("imap cleaned for addr {}".format(addr))
+            self._imap_cleaned.add(addr)
+
+
+class ACFactory:
+    _finalizers: List[Callable[[], None]]
+    _accounts: List[Account]
+
+    def __init__(self, request, testprocess, tmpdir, data) -> None:
+        self.init_time = time.time()
+        self.tmpdir = tmpdir
+        self.pytestconfig = request.config
+        self.data = data
+        self.testprocess = testprocess
+        self._liveconfig_producer = testprocess.get_liveconfig_producer()
+
+        self._finalizers = []
+        self._accounts = []
+        self._acsetup = ACSetup(testprocess, self.init_time)
+        self._preconfigured_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
-            self.set_logging_default(False)
-            deltachat.register_global_plugin(direct_imap)
+        self.set_logging_default(False)
+        request.addfinalizer(self.finalize)
 
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
+    def log(self, *args):
+        print("[acfactory]", "{:.3f}".format(time.time() - self.init_time), *args)
 
-            while self._accounts:
-                acc = self._accounts.pop()
+    def finalize(self):
+        while self._finalizers:
+            fin = self._finalizers.pop()
+            fin()
+
+        while self._accounts:
+            acc = self._accounts.pop()
+            if acc is not None:
+                imap = getattr(acc, "direct_imap", None)
+                if imap is not None:
+                    imap.shutdown()
+                    del acc.direct_imap
                 acc.shutdown()
                 acc.disable_logging()
-            deltachat.unregister_global_plugin(direct_imap)
 
-        def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
-            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac.addr = ac.get_self_contact().addr
-            ac.set_config("displayname", logid)
-            if not quiet:
-                logger = FFIEventLogger(ac)
-                logger.init_time = self.init_time
-                ac.add_account_plugin(logger)
-            self._accounts.append(ac)
+    def get_next_liveconfig(self):
+        """ Base function to get functional online configurations
+        where we can make valid SMTP and IMAP connections with.
+        """
+        configdict = next(self._liveconfig_producer).copy()
+        if "e2ee_enabled" not in configdict:
+            configdict["e2ee_enabled"] = "1"
+
+        if self.pytestconfig.getoption("--strict-tls"):
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+        assert "addr" in configdict and "mail_pw" in configdict
+        return configdict
+
+    def _get_cached_account_copy(self, addr):
+        if addr in self.testprocess._addr2files:
+            return self._getaccount(addr)
+
+    def get_unconfigured_account(self):
+        return self._getaccount()
+
+    def _getaccount(self, try_cache_addr=None):
+        logid = "ac{}".format(len(self._accounts) + 1)
+
+        # we need to use fixed database basename for maybe_cache_* functions to work
+        path = self.tmpdir.mkdir(logid).join("dc.db")
+        if try_cache_addr:
+            self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
+        ac = Account(path.strpath, logging=self._logging)
+        ac._logid = logid  # later instantiated FFIEventLogger needs this
+        ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+        if self.pytestconfig.getoption("--debug-setup"):
+            self._acsetup.init_logging(ac)
+        self._accounts.append(ac)
+        return ac
+
+    def set_logging_default(self, logging):
+        self._logging = bool(logging)
+
+    def remove_preconfigured_keys(self):
+        self._preconfigured_keys = []
+
+    def _preconfigure_key(self, account, addr):
+        # Only set a preconfigured key if we haven't used it yet for another account.
+        try:
+            keyname = self._preconfigured_keys.pop(0)
+        except IndexError:
+            pass
+        else:
+            fname_pub = self.data.read_path("key/{name}-public.asc".format(name=keyname))
+            fname_sec = self.data.read_path("key/{name}-secret.asc".format(name=keyname))
+            if fname_pub and fname_sec:
+                account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                return True
+            else:
+                print("WARN: could not use preconfigured keys for {!r}".format(addr))
+
+    def get_pseudo_configured_account(self):
+        # do a pseudo-configured account
+        ac = self.get_unconfigured_account()
+        acname = ac._logid
+        addr = "{}@offline.org".format(acname)
+        ac.update_config(dict(
+            addr=addr, displayname=acname, mail_pw="123",
+            configured_addr=addr, configured_mail_pw="123",
+            configured="1",
+        ))
+        self._preconfigure_key(ac, addr)
+        self._acsetup.init_logging(ac)
+        return ac
+
+    # XXX deprecate the next function?
+    def new_online_configuring_account(self, cache=False, **kwargs):
+        configdict = self.get_next_liveconfig()
+        configdict.update(kwargs)
+        return self._setup_online_configuring_account(configdict, cache=cache)
+
+    def get_online_second_device(self, ac1, **kwargs):
+        ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
+        if ac2 is None:
+            # some tests setup the primary account without causing caching.
+            configdict = kwargs.copy()
+            configdict["addr"] = ac1.get_config("addr")
+            configdict["mail_pw"] = ac1.get_config("mail_pw")
+            ac2 = self._setup_online_configuring_account(configdict, cache=False)
+        elif kwargs:
+            ac2.update_config(kwargs)
+            self._acsetup.add_configured(ac2)
+
+        self.bring_accounts_online()
+        return ac2
+
+    def get_online_multidevice_setup(self, copied=True):
+        """ Provide two accounts. The second uses the same credentials
+        and if copied is True, also the same database and blobs.
+        You can use copy=False to get a typical configuration where
+        a user unsuspectingly sets up a second device and expects it to
+        "just work" not knowing that an export/import is required.
+        """
+        ac1, = self.get_online_accounts(1)
+        if copied:
+            ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
+            self._acsetup.add_configured(ac2)
+        else:
+            config2 = dict(addr=ac1.get_config("addr"), mail_pw=ac1.get_config("mail_pw"))
+            ac2 = self._setup_online_configuring_account(config2, cache=False)
+        self.bring_accounts_online()
+        return ac1, ac2
+
+    def get_online_devnull_email(self):
+        return self.get_next_liveconfig()["addr"]
+
+    def _setup_online_configuring_account(self, configdict, cache=False):
+        ac = self._get_cached_account_copy(configdict["addr"]) if cache else None
+        if ac is not None:
+            # make sure we consume a preconfig key, as if we had created a fresh account
+            self._preconfigured_keys.pop(0)
+            self._acsetup.add_configured(ac)
             return ac
+        ac = self.prepare_account_from_liveconfig(configdict)
+        self._acsetup.start_configure(ac)
+        return ac
 
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
+    def prepare_account_from_liveconfig(self, configdict):
+        ac = self.get_unconfigured_account()
+        assert "addr" in configdict and "mail_pw" in configdict, configdict
+        configdict.setdefault("bcc_self", False)
+        configdict.setdefault("mvbox_move", False)
+        configdict.setdefault("sentbox_watch", False)
+        ac.update_config(configdict)
+        self._preconfigure_key(ac, configdict["addr"])
+        return ac
 
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+    def wait_configured(self, account):
+        """ Wait until the specified account has successfully completed configure. """
+        self._acsetup.wait_one_configured(account)
 
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
-                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
-                if fname_pub and fname_sec:
-                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                    return True
-                else:
-                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
+    def bring_accounts_online(self):
+        print("bringing accounts online")
+        self._acsetup.bring_online()
+        print("all accounts online")
 
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
+    def get_online_configured_accounts(self, configlist):
+        accounts = [self.new_online_configuring_account(cache=True, **config)
+                    for config in configlist]
+        self._acsetup.wait_all_configured()
+        for acc in accounts:
+            self._acsetup.init_imap(acc)
+        return accounts
 
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
+    def force_reconfigure(self, account):
+        self._acsetup.start_configure(account, reconfigure=True)
 
-        def get_online_config(self, pre_generated_key=True, quiet=False):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
+    def get_online_accounts(self, num):
+        """ Return a list of configured and started Accounts.
 
-            if pytestconfig.getoption("--strict-tls"):
-                # Enable strict certificate checks for online accounts
-                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+        This function creates plain online accounts and fill
+        a testprocess-scoped cache and re-use these plain accounts
+        on the next test function.
+        """
+        accounts = [self.new_online_configuring_account(cache=True) for i in range(num)]
+        self.bring_accounts_online()
+        for acc in accounts:
+            self.testprocess.cache_maybe_store_configured_db_files(acc)
+        return accounts
 
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            return ac, dict(configdict)
+    def run_bot_process(self, module, ffi=True):
+        fn = module.__file__
 
-        def get_online_configuring_account(self, sentbox=False, move=False,
-                                           pre_generated_key=True, quiet=False, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key, quiet=quiet)
-            configdict.update(config)
-            configdict["mvbox_move"] = str(int(move))
-            configdict["sentbox_watch"] = str(int(sentbox))
-            ac.update_config(configdict)
-            ac._configtracker = ac.configure()
-            return ac
+        bot_cfg = self.get_next_liveconfig()
+        bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
 
-        def get_one_online_account(self, pre_generated_key=True, move=False):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, move=move)
-            self.wait_configure_and_start_io([ac1])
-            return ac1
+        # Forget ac as it will be opened by the bot subprocess
+        # but keep something in the list to not confuse account generation
+        self._accounts[self._accounts.index(bot_ac)] = None
 
-        def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
-            ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io([ac1, ac2])
-            return ac1, ac2
+        args = [
+            sys.executable,
+            "-u",
+            fn,
+            "--email", bot_cfg["addr"],
+            "--password", bot_cfg["mail_pw"],
+            bot_ac.db_path,
+        ]
+        if ffi:
+            args.insert(-1, "--show-ffi")
+        print("$", " ".join(args))
+        popen = subprocess.Popen(
+            args=args,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+            bufsize=0,                 # line buffering
+            close_fds=True,            # close all FDs other than 0/1/2
+            universal_newlines=True    # give back text
+        )
+        bot = BotProcess(popen, addr=bot_cfg["addr"])
+        self._finalizers.append(bot.kill)
+        return bot
 
-        def get_many_online_accounts(self, num, move=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=True)
-                        for i in range(num)]
-            self.wait_configure_and_start_io(accounts)
-            for acc in accounts:
-                acc.add_account_plugin(FFIEventLogger(acc))
-            return accounts
+    def dump_imap_summary(self, logfile):
+        for ac in self._accounts:
+            ac.dump_account_info(logfile=logfile)
+            imap = getattr(ac, "direct_imap", None)
+            if imap is not None:
+                imap.dump_imap_structures(self.tmpdir, logfile=logfile)
 
-        def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_move, sentbox_watch and the
-            direct_imap object of an online account. This simulates the user setting
-            up a new device without importing a backup.
+    def get_accepted_chat(self, ac1: Account, ac2):
+        ac2.create_chat(ac1)
+        return ac1.create_chat(ac2)
 
-            `pre_generated_key` only means that a key from python/tests/data/key is
-            used in order to speed things up.
-            """
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac.update_config(dict(
-                addr=account.get_config("addr"),
-                mail_pw=account.get_config("mail_pw"),
-                mvbox_move=account.get_config("mvbox_move"),
-                sentbox_watch=account.get_config("sentbox_watch"),
-            ))
-            if hasattr(account, "direct_imap"):
-                # Attach the existing direct_imap. If we did not do this, a new one would be created and
-                # delete existing messages (see dc_account_extra_configure(configure))
-                ac.direct_imap = account.direct_imap
-            ac._configtracker = ac.configure()
-            return ac
+    def introduce_each_other(self, accounts, sending=True):
+        to_wait = []
+        for i, acc in enumerate(accounts):
+            for acc2 in accounts[i + 1:]:
+                chat = self.get_accepted_chat(acc, acc2)
+                if sending:
+                    chat.send_text("hi")
+                    to_wait.append(acc2)
+                    acc2.create_chat(acc).send_text("hi back")
+                    to_wait.append(acc)
+        for acc in to_wait:
+            acc.log("waiting for incoming message")
+            acc._evtracker.wait_next_incoming_message()
 
-        def wait_configure_and_start_io(self, accounts=None):
-            if accounts is None:
-                accounts = self._accounts[:]
-            started_accounts = []
-            for acc in accounts:
-                if acc not in started_accounts:
-                    self.wait_configure(acc)
-                    acc.set_config("bcc_self", "0")
-                    if acc.is_configured():
-                        acc.start_io()
-                        started_accounts.append(acc)
-                    print("{}: {} account was started".format(
-                        acc.get_config("displayname"), acc.get_config("addr")))
-            for acc in started_accounts:
-                acc._evtracker.wait_all_initial_fetches()
 
-        def wait_configure(self, acc):
-            if hasattr(acc, "_configtracker"):
-                acc._configtracker.wait_finish()
-                acc._evtracker.consume_events()
-                acc.get_device_chat().mark_noticed()
-                del acc._configtracker
-
-        def run_bot_process(self, module, ffi=True):
-            fn = module.__file__
-
-            bot_ac, bot_cfg = self.get_online_config()
-
-            # Avoid starting ac so we don't interfere with the bot operating on
-            # the same database.
-            self._accounts.remove(bot_ac)
-
-            args = [
-                sys.executable,
-                "-u",
-                fn,
-                "--email", bot_cfg["addr"],
-                "--password", bot_cfg["mail_pw"],
-                bot_ac.db_path,
-            ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
-            print("$", " ".join(args))
-            popen = subprocess.Popen(
-                args=args,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
-                close_fds=True,            # close all FDs other than 0/1/2
-                universal_newlines=True    # give back text
-            )
-            bot = BotProcess(popen, bot_cfg)
-            self._finalizers.append(bot.kill)
-            return bot
-
-        def dump_imap_summary(self, logfile):
-            for ac in self._accounts:
-                ac.dump_account_info(logfile=logfile)
-                imap = getattr(ac, "direct_imap", None)
-                if imap is not None:
-                    try:
-                        imap.idle_done()
-                    except Exception:
-                        pass
-                    imap.dump_imap_structures(tmpdir, logfile=logfile)
-
-        def get_accepted_chat(self, ac1: Account, ac2: Account):
-            ac2.create_chat(ac1)
-            return ac1.create_chat(ac2)
-
-        def introduce_each_other(self, accounts, sending=True):
-            to_wait = []
-            for i, acc in enumerate(accounts):
-                for acc2 in accounts[i + 1:]:
-                    chat = self.get_accepted_chat(acc, acc2)
-                    if sending:
-                        chat.send_text("hi")
-                        to_wait.append(acc2)
-                        acc2.create_chat(acc).send_text("hi back")
-                        to_wait.append(acc)
-            for acc in to_wait:
-                acc._evtracker.wait_next_incoming_message()
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
+@pytest.fixture
+def acfactory(request, tmpdir, testprocess, data):
+    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testprocess, data=data)
     yield am
     if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        logfile = io.StringIO()
-        am.dump_imap_summary(logfile=logfile)
-        print(logfile.getvalue())
-        # request.node.add_report_section("call", "imap-server-state", s)
+        if testprocess.pytestconfig.getoption("--extra-info"):
+            logfile = io.StringIO()
+            am.dump_imap_summary(logfile=logfile)
+            print(logfile.getvalue())
+            # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
     stdout_queue: queue.Queue
 
-    def __init__(self, popen, bot_cfg) -> None:
+    def __init__(self, popen, addr) -> None:
         self.popen = popen
-        self.addr = bot_cfg["addr"]
+        self.addr = addr
 
         # we read stdout as quickly as we can in a thread and make
         # the (unicode) lines available for readers through a queue.

python/tests/bench_test_setup.py

@@ -8,11 +8,13 @@ to see timings of test setups.
 
 import pytest
 
+BENCH_NUM = 3
+
 
 class TestEmpty:
     def test_prepare_setup_measurings(self, acfactory):
-        acfactory.get_many_online_accounts(5)
+        acfactory.get_online_accounts(BENCH_NUM)
 
-    @pytest.mark.parametrize("num", range(0, 5))
+    @pytest.mark.parametrize("num", range(0, BENCH_NUM + 1))
     def test_setup_online_accounts(self, acfactory, num):
-        acfactory.get_many_online_accounts(num)
+        acfactory.get_online_accounts(num)

python/tests/stress_test_db.py

@@ -17,7 +17,7 @@ def test_db_busy_error(acfactory, tmpdir):
             print("%3.2f %s" % (time.time() - starttime, string))
 
     # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3, quiet=True)
+    accounts = acfactory.get_many_online_accounts(3)
     log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account

python/tests/test_0_complex_or_slow.py

@@ -0,0 +1,462 @@
+import sys
+import pytest
+
+
+class TestGroupStressTests:
+    def test_group_many_members_add_leave_remove(self, acfactory, lp):
+        accounts = acfactory.get_online_accounts(5)
+        acfactory.introduce_each_other(accounts)
+        ac1, ac5 = accounts.pop(), accounts.pop()
+
+        lp.sec("ac1: creating group chat with 3 other members")
+        chat = ac1.create_group_chat("title1", contacts=accounts)
+
+        lp.sec("ac1: send message to new group chat")
+        msg1 = chat.send_text("hello")
+        assert msg1.is_encrypted()
+        gossiped_timestamp = chat.get_summary()["gossiped_timestamp"]
+        assert gossiped_timestamp > 0
+
+        assert chat.num_contacts() == 3 + 1
+
+        lp.sec("ac2: checking that the chat arrived correctly")
+        ac2 = accounts[0]
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "hello"
+        print("chat is", msg2.chat)
+        assert msg2.chat.num_contacts() == 4
+
+        lp.sec("ac3: checking that 'ac4' is a known contact")
+        ac3 = accounts[1]
+        msg3 = ac3._evtracker.wait_next_incoming_message()
+        assert msg3.text == "hello"
+        ac3_contacts = ac3.get_contacts()
+        assert len(ac3_contacts) == 4
+        ac4_contacts = ac3.get_contacts(query=accounts[2].get_config("addr"))
+        assert len(ac4_contacts) == 1
+
+        lp.sec("ac2: removing one contact")
+        to_remove = ac2.create_contact(accounts[-1])
+        msg2.chat.remove_contact(to_remove)
+
+        lp.sec("ac1: receiving system message about contact removal")
+        sysmsg = ac1._evtracker.wait_next_incoming_message()
+        assert to_remove.addr in sysmsg.text
+        assert sysmsg.chat.num_contacts() == 3
+
+        # Receiving message about removed contact does not reset gossip
+        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
+
+        lp.sec("ac1: sending another message to the chat")
+        chat.send_text("hello2")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello2"
+        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
+
+        lp.sec("ac1: adding fifth member to the chat")
+        chat.add_contact(ac5)
+        # Adding contact to chat resets gossiped_timestamp
+        assert chat.get_summary()["gossiped_timestamp"] >= gossiped_timestamp
+
+        lp.sec("ac2: receiving system message about contact addition")
+        sysmsg = ac2._evtracker.wait_next_incoming_message()
+        assert ac5.get_config("configured_addr") in sysmsg.text
+        assert sysmsg.chat.num_contacts() == 4
+
+        lp.sec("ac5: waiting for message about addition to the chat")
+        sysmsg = ac5._evtracker.wait_next_incoming_message()
+        msg = sysmsg.chat.send_text("hello!")
+        # Message should be encrypted because keys of other members are gossiped
+        assert msg.is_encrypted()
+
+    def test_synchronize_member_list_on_group_rejoin(self, acfactory, lp):
+        """
+        Test that user recreates group member list when it joins the group again.
+        ac1 creates a group with two other accounts: ac2 and ac3
+        Then it removes ac2, removes ac3 and adds ac2 back.
+        ac2 did not see that ac3 is removed, so it should rebuild member list from scratch.
+        """
+        lp.sec("setting up accounts, accepted with each other")
+        accounts = acfactory.get_online_accounts(3)
+        acfactory.introduce_each_other(accounts)
+        ac1, ac2, ac3 = accounts
+
+        lp.sec("ac1: creating group chat with 2 other members")
+        chat = ac1.create_group_chat("title1", contacts=[ac2, ac3])
+        assert not chat.is_promoted()
+
+        lp.sec("ac1: send message to new group chat")
+        msg = chat.send_text("hello")
+        assert chat.is_promoted() and msg.is_encrypted()
+
+        assert chat.num_contacts() == 3
+
+        lp.sec("checking that the chat arrived correctly")
+        for ac in accounts[1:]:
+            msg = ac._evtracker.wait_next_incoming_message()
+            assert msg.text == "hello"
+            print("chat is", msg.chat)
+            assert msg.chat.num_contacts() == 3
+
+        lp.sec("ac1: removing ac2")
+        chat.remove_contact(ac2)
+
+        lp.sec("ac2: wait for a message about removal from the chat")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
+        lp.sec("ac1: removing ac3")
+        chat.remove_contact(ac3)
+
+        lp.sec("ac1: adding ac2 back")
+        # Group is promoted, message is sent automatically
+        assert chat.is_promoted()
+        chat.add_contact(ac2)
+
+        lp.sec("ac2: check that ac3 is removed")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
+        assert chat.num_contacts() == 2
+        assert msg.chat.num_contacts() == 2
+        acfactory.dump_imap_summary(sys.stdout)
+
+
+def test_qr_verified_group_and_chatting(acfactory, lp):
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
+    chat1 = ac1.create_group_chat("hello", verified=True)
+    assert chat1.is_protected()
+    qr = chat1.get_join_qr()
+    lp.sec("ac2: start QR-code based join-group protocol")
+    chat2 = ac2.qr_join_chat(qr)
+    assert chat2.id >= 10
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    lp.sec("ac2: read member added message")
+    msg = ac2._evtracker.wait_next_incoming_message()
+    assert msg.is_encrypted()
+    assert "added" in msg.text.lower()
+
+    lp.sec("ac1: send message")
+    msg_out = chat1.send_text("hello")
+    assert msg_out.is_encrypted()
+
+    lp.sec("ac2: read message and check it's verified chat")
+    msg = ac2._evtracker.wait_next_incoming_message()
+    assert msg.text == "hello"
+    assert msg.chat.is_protected()
+    assert msg.is_encrypted()
+
+    lp.sec("ac2: send message and let ac1 read it")
+    chat2.send_text("world")
+    msg = ac1._evtracker.wait_next_incoming_message()
+    assert msg.text == "world"
+    assert msg.is_encrypted()
+
+    lp.sec("ac1: create QR code and let ac3 scan it, starting the securejoin")
+    qr = ac1.get_setup_contact_qr()
+
+    lp.sec("ac3: start QR-code based setup contact protocol")
+    ch = ac3.qr_setup_contact(qr)
+    assert ch.id >= 10
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    lp.sec("ac1: add ac3 to verified group")
+    chat1.add_contact(ac3)
+    msg = ac2._evtracker.wait_next_incoming_message()
+    assert msg.is_encrypted()
+    assert msg.is_system_message()
+    assert not msg.error
+
+    lp.sec("ac2: send message and let ac3 read it")
+    chat2.send_text("hi")
+    # Skip system message about added member
+    ac3._evtracker.wait_next_incoming_message()
+    msg = ac3._evtracker.wait_next_incoming_message()
+    assert msg.text == "hi"
+    assert msg.is_encrypted()
+
+
+@pytest.mark.parametrize("mvbox_move", [False, True])
+def test_fetch_existing(acfactory, lp, mvbox_move):
+    """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
+    This way, we can already offer them some email addresses they can write to.
+
+    Also, the newest existing emails from each folder are fetched during onboarding.
+
+    Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
+
+    def assert_folders_configured(ac):
+        """There was a bug that scan_folders() set the configured folders to None under some circumstances.
+        So, check that they are still configured:"""
+        assert ac.get_config("configured_sentbox_folder") == "Sent"
+        if mvbox_move:
+            assert ac.get_config("configured_mvbox_folder")
+
+    ac1, ac2 = acfactory.get_online_configured_accounts([dict(mvbox_move=mvbox_move), dict()])
+    ac1.direct_imap.create_folder("Sent")
+    ac1.set_config("sentbox_watch", "1")
+
+    # We need to reconfigure to find the new "Sent" folder.
+    # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
+    # would also find the "Sent" folder, but it would be too late:
+    # The sentbox thread, started by `start_io()`, would have seen that there is no
+    # ConfiguredSentboxFolder and do nothing.
+    acfactory.force_reconfigure(ac1)
+    acfactory.bring_accounts_online()
+
+    assert_folders_configured(ac1)
+
+    assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
+    with ac1.direct_imap.idle() as idle1:
+        lp.sec("send out message with bcc to ourselves")
+        ac1.set_config("bcc_self", "1")
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        chat.send_text("message text")
+        assert_folders_configured(ac1)
+
+        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
+        assert idle1.wait_for_seen()
+    assert_folders_configured(ac1)
+
+    lp.sec("create a cloned ac1 and fetch contact history during configure")
+    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
+    assert_folders_configured(ac1_clone)
+
+    lp.sec("check that ac2 contact was fetchted during configure")
+    ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+    ac2_addr = ac2.get_config("addr")
+    assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+    assert_folders_configured(ac1_clone)
+
+    lp.sec("check that messages changed events arrive for the correct message")
+    msg = ac1_clone._evtracker.wait_next_messages_changed()
+    assert msg.text == "message text"
+    assert_folders_configured(ac1)
+    assert_folders_configured(ac1_clone)
+
+
+def test_fetch_existing_msgs_group_and_single(acfactory, lp):
+    """There was a bug concerning fetch-existing-msgs:
+
+    A sent a message to you, adding you to a group. This created a contact request.
+    You wrote a message to A, creating a chat.
+    ...but the group stayed blocked.
+    So, after fetch-existing-msgs you have one contact request and one chat with the same person.
+
+    See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    lp.sec("receive a message")
+    ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
+    ac1._evtracker.wait_next_incoming_message()
+
+    lp.sec("send out message with bcc to ourselves")
+    with ac1.direct_imap.idle() as idle1:
+        ac1.set_config("bcc_self", "1")
+        ac1_ac2_chat = ac1.create_chat(ac2)
+        ac1_ac2_chat.send_text("outgoing, encrypted direct message, creating a chat")
+        # wait until the bcc_self message arrives
+        assert idle1.wait_for_seen()
+
+    lp.sec("Clone online account and let it fetch the existing messages")
+    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
+
+    chats = ac1_clone.get_chats()
+    assert len(chats) == 4  # two newly created chats + self-chat + device-chat
+    group_chat = [c for c in chats if c.get_name() == "group name"][0]
+    assert group_chat.is_group()
+    private_chat, = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
+    assert not private_chat.is_group()
+
+    group_messages = group_chat.get_messages()
+    assert len(group_messages) == 1
+    assert group_messages[0].text == "incoming, unencrypted group message"
+    private_messages = private_chat.get_messages()
+    assert len(private_messages) == 1
+    assert private_messages[0].text.startswith("outgoing, encrypted")
+
+
+def test_undecipherable_group(acfactory, lp):
+    """Test how group messages that cannot be decrypted are
+    handled.
+
+    Group name is encrypted and plaintext subject is set to "..." in
+    this case, so we should assign the messages to existing chat
+    instead of creating a new one. Since there is no existing group
+    chat, the messages should be assigned to 1-1 chat with the sender
+    of the message.
+    """
+
+    lp.sec("creating and configuring three accounts")
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    acfactory.introduce_each_other([ac1, ac2, ac3])
+
+    lp.sec("ac3 reinstalls DC and generates a new key")
+    ac3.stop_io()
+    acfactory.remove_preconfigured_keys()
+    ac4 = acfactory.get_online_second_device(ac3)
+    # Create contacts to make sure incoming messages are not treated as contact requests
+
+    lp.sec("ac1: creating group chat with 2 other members")
+    chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
+
+    lp.sec("ac1: send message to new group chat")
+    msg = chat.send_text("hello")
+
+    lp.sec("ac2: checking that the chat arrived correctly")
+    msg = ac2._evtracker.wait_next_incoming_message()
+    assert msg.text == "hello"
+    assert msg.is_encrypted(), "Message is not encrypted"
+
+    # ac4 cannot decrypt the message.
+    # Error message should be assigned to the chat with ac1.
+    lp.sec("ac4: checking that message is assigned to the sender chat")
+    error_msg = ac4._evtracker.wait_next_incoming_message()
+    assert error_msg.error  # There is an error decrypting the message
+    assert error_msg.chat == chat41
+
+    lp.sec("ac2: sending a reply to the chat")
+    msg.chat.send_text("reply")
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.text == "reply"
+    assert reply.is_encrypted(), "Reply is not encrypted"
+
+    lp.sec("ac4: checking that reply is assigned to ac2 chat")
+    error_reply = ac4._evtracker.wait_next_incoming_message()
+    assert error_reply.error  # There is an error decrypting the message
+    assert error_reply.chat == chat42
+
+    # Test that ac4 replies to error messages don't appear in the
+    # group chat on ac1 and ac2.
+    lp.sec("ac4: replying to ac1 and ac2")
+
+    # Otherwise reply becomes a contact request.
+    chat41.send_text("I can't decrypt your message, ac1!")
+    chat42.send_text("I can't decrypt your message, ac2!")
+
+    msg = ac1._evtracker.wait_next_incoming_message()
+    assert msg.error is None
+    assert msg.text == "I can't decrypt your message, ac1!"
+    assert msg.is_encrypted(), "Message is not encrypted"
+    assert msg.chat == ac1.create_chat(ac3)
+
+    msg = ac2._evtracker.wait_next_incoming_message()
+    assert msg.error is None
+    assert msg.text == "I can't decrypt your message, ac2!"
+    assert msg.is_encrypted(), "Message is not encrypted"
+    assert msg.chat == ac2.create_chat(ac4)
+
+
+def test_ephemeral_timer(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    lp.sec("ac1: create chat with ac2")
+    chat1 = ac1.create_chat(ac2)
+    chat2 = ac2.create_chat(ac1)
+
+    lp.sec("ac1: set ephemeral timer to 60")
+    chat1.set_ephemeral_timer(60)
+
+    lp.sec("ac1: check that ephemeral timer is set for chat")
+    assert chat1.get_ephemeral_timer() == 60
+    chat1_summary = chat1.get_summary()
+    assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
+
+    lp.sec("ac2: receive system message about ephemeral timer modification")
+    ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+    system_message1 = ac2._evtracker.wait_next_incoming_message()
+    assert chat2.get_ephemeral_timer() == 60
+    assert system_message1.is_system_message()
+
+    # Disabled until markers are implemented
+    # assert "Ephemeral timer: 60\n" in system_message1.get_message_info()
+
+    lp.sec("ac2: send message to ac1")
+    sent_message = chat2.send_text("message")
+    assert sent_message.ephemeral_timer == 60
+    assert "Ephemeral timer: 60\n" in sent_message.get_message_info()
+
+    # Timer is started immediately for sent messages
+    assert sent_message.ephemeral_timestamp is not None
+    assert "Expires: " in sent_message.get_message_info()
+
+    lp.sec("ac1: waiting for message from ac2")
+    text_message = ac1._evtracker.wait_next_incoming_message()
+    assert text_message.text == "message"
+    assert text_message.ephemeral_timer == 60
+    assert "Ephemeral timer: 60\n" in text_message.get_message_info()
+
+    # Timer should not start until message is displayed
+    assert text_message.ephemeral_timestamp is None
+    assert "Expires: " not in text_message.get_message_info()
+    text_message.mark_seen()
+    text_message = ac1.get_message_by_id(text_message.id)
+    assert text_message.ephemeral_timestamp is not None
+    assert "Expires: " in text_message.get_message_info()
+
+    lp.sec("ac2: set ephemeral timer to 0")
+    chat2.set_ephemeral_timer(0)
+    ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+
+    lp.sec("ac1: receive system message about ephemeral timer modification")
+    ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+    system_message2 = ac1._evtracker.wait_next_incoming_message()
+    assert system_message2.ephemeral_timer is None
+    assert "Ephemeral timer: " not in system_message2.get_message_info()
+    assert chat1.get_ephemeral_timer() == 0
+
+
+def test_multidevice_sync_seen(acfactory, lp):
+    """Test that message marked as seen on one device is marked as seen on another."""
+    ac1, ac1_clone = acfactory.get_online_multidevice_setup()
+    ac2, = acfactory.get_online_accounts(1)
+
+    ac1.set_config("bcc_self", "1")
+    ac1_clone.set_config("bcc_self", "1")
+
+    ac1_chat = ac1.create_chat(ac2)
+    ac1_clone_chat = ac1_clone.create_chat(ac2)
+    ac2_chat = ac2.create_chat(ac1)
+
+    lp.sec("Send a message from ac2 to ac1 and check that it's 'fresh'")
+    ac2_chat.send_text("Hi")
+    ac1_message = ac1._evtracker.wait_next_incoming_message()
+    ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
+    assert ac1_chat.count_fresh_messages() == 1
+    assert ac1_clone_chat.count_fresh_messages() == 1
+    assert ac1_message.is_in_fresh
+    assert ac1_clone_message.is_in_fresh
+
+    lp.sec("ac1 marks message as seen on the first device")
+    ac1.mark_seen_messages([ac1_message])
+    assert ac1_message.is_in_seen
+
+    lp.sec("ac1 clone detects that message is marked as seen")
+    ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+    assert ev.data1 == ac1_clone_chat.id
+    assert ac1_clone_message.is_in_seen
+
+    lp.sec("Send an ephemeral message from ac2 to ac1")
+    ac2_chat.set_ephemeral_timer(60)
+    ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+    ac1._evtracker.wait_next_incoming_message()
+    ac1_clone._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
+    ac1_clone._evtracker.wait_next_incoming_message()
+
+    ac2_chat.send_text("Foobar")
+    ac1_message = ac1._evtracker.wait_next_incoming_message()
+    ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
+    assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
+    assert "Expires: " not in ac1_clone_message.get_message_info()
+    assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
+    assert "Expires: " not in ac1_clone_message.get_message_info()
+
+    ac1.mark_seen_messages([ac1_message])
+    assert ac1_message.is_in_seen
+    assert "Expires: " in ac1_message.get_message_info()
+    ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
+    assert ev.data1 == ac1_clone_chat.id
+    assert ac1_clone_message.is_in_seen
+    # Test that the timer is started on the second device after synchronizing the seen status.
+    assert "Expires: " in ac1_clone_message.get_message_info()

python/tests/test_2_increation.py

@@ -33,7 +33,7 @@ def wait_msgs_changed(account, msgs_list):
 
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1, ac2 = acfactory.get_online_accounts(2)
         chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
@@ -43,7 +43,7 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1, ac2 = acfactory.get_online_accounts(2)
         chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
@@ -56,7 +56,7 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1, ac2 = acfactory.get_online_accounts(2)
 
         chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?

python/tests/test_3_offline.py

@@ -0,0 +1,634 @@
+from __future__ import print_function
+import pytest
+import os
+import time
+from deltachat import const, Account
+from deltachat.message import Message
+from deltachat.hookspec import account_hookimpl
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
+from datetime import datetime, timedelta, timezone
+
+
+@pytest.mark.parametrize("msgtext,res", [
+    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+        ("removed", "tmp1@x.org", "tmp2@x.org")),
+    ("Member With space (tmp1@x.org) removed by me",
+        ("removed", "tmp1@x.org", "me")),
+    ("Group left by some one (tmp1@x.org).",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Group left by tmp1@x.org.",
+        ("removed", "tmp1@x.org", "tmp1@x.org")),
+    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
+    ("Member nothing bla bla", None),
+    ("Another unknown system message", None),
+])
+def test_parse_system_add_remove(msgtext, res):
+    from deltachat.message import parse_system_add_remove
+
+    out = parse_system_add_remove(msgtext)
+    assert out == res
+
+
+class TestOfflineAccountBasic:
+    def test_wrong_db(self, tmpdir):
+        p = tmpdir.join("hello.db")
+        p.write("123")
+        account = Account(p.strpath)
+        assert not account.is_open()
+
+    def test_os_name(self, tmpdir):
+        p = tmpdir.join("hello.db")
+        # we can't easily test if os_name is used in X-Mailer
+        # outgoing messages without a full Online test
+        # but we at least check Account accepts the arg
+        ac1 = Account(p.strpath, os_name="solarpunk")
+        ac1.get_info()
+
+    def test_preconfigure_keypair(self, acfactory, data):
+        ac = acfactory.get_unconfigured_account()
+        alice_public = data.read_path("key/alice-public.asc")
+        alice_secret = data.read_path("key/alice-secret.asc")
+        assert alice_public and alice_secret
+        ac._preconfigure_keypair("alice@example.org", alice_public, alice_secret)
+
+    def test_getinfo(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        d = ac1.get_info()
+        assert d["arch"]
+        assert d["number_of_chats"] == "0"
+        assert d["bcc_self"] == "0"
+
+    def test_is_not_configured(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        assert not ac1.is_configured()
+        with pytest.raises(ValueError):
+            ac1.check_is_configured()
+
+    def test_wrong_config_keys(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        with pytest.raises(KeyError):
+            ac1.set_config("lqkwje", "value")
+        with pytest.raises(KeyError):
+            ac1.get_config("lqkwje")
+
+    def test_set_config_int_conversion(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        ac1.set_config("mvbox_move", False)
+        assert ac1.get_config("mvbox_move") == "0"
+        ac1.set_config("mvbox_move", True)
+        assert ac1.get_config("mvbox_move") == "1"
+        ac1.set_config("mvbox_move", 0)
+        assert ac1.get_config("mvbox_move") == "0"
+        ac1.set_config("mvbox_move", 1)
+        assert ac1.get_config("mvbox_move") == "1"
+
+    def test_update_config(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        ac1.update_config(dict(mvbox_move=False))
+        assert ac1.get_config("mvbox_move") == "0"
+
+    def test_has_savemime(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        assert "save_mime_headers" in ac1.get_config("sys.config_keys").split()
+
+    def test_has_bccself(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        assert "bcc_self" in ac1.get_config("sys.config_keys").split()
+        assert ac1.get_config("bcc_self") == "0"
+
+    def test_selfcontact_if_unconfigured(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        assert not ac1.get_self_contact().addr
+
+    def test_selfcontact_configured(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        me = ac1.get_self_contact()
+        assert me.display_name
+        assert me.addr
+
+    def test_get_config_fails(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        with pytest.raises(KeyError):
+            ac1.get_config("123123")
+
+    def test_empty_group_bcc_self_enabled(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        ac1.set_config("bcc_self", "1")
+        chat = ac1.create_group_chat(name="group1")
+        msg = chat.send_text("msg1")
+        assert msg in chat.get_messages()
+
+    def test_empty_group_bcc_self_disabled(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        ac1.set_config("bcc_self", "0")
+        chat = ac1.create_group_chat(name="group1")
+        msg = chat.send_text("msg1")
+        assert msg in chat.get_messages()
+
+
+class TestOfflineContact:
+    def test_contact_attr(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some1@example.org", name="some1")
+        str(contact1)
+        repr(contact1)
+        assert contact1 == contact2
+        assert contact1.id
+        assert contact1.addr == "some1@example.org"
+        assert contact1.display_name == "some1"
+        assert not contact1.is_blocked()
+        assert not contact1.is_verified()
+
+    def test_get_blocked(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some2@example.org", name="some2")
+        ac1.create_contact("some3@example.org", name="some3")
+        assert ac1.get_blocked_contacts() == []
+        contact1.block()
+        assert ac1.get_blocked_contacts() == [contact1]
+        contact2.block()
+        blocked = ac1.get_blocked_contacts()
+        assert len(blocked) == 2 and contact1 in blocked and contact2 in blocked
+        contact2.unblock()
+        assert ac1.get_blocked_contacts() == [contact1]
+
+    def test_create_self_contact(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact1 = ac1.create_contact(ac1.get_config("addr"))
+        assert contact1.id == 1
+
+    def test_get_contacts_and_delete(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contacts = ac1.get_contacts()
+        assert len(contacts) == 1
+        assert contact1 in contacts
+
+        assert not ac1.get_contacts(query="some2")
+        assert ac1.get_contacts(query="some1")
+        assert not ac1.get_contacts(only_verified=True)
+        assert len(ac1.get_contacts(with_self=True)) == 2
+
+        assert ac1.delete_contact(contact1)
+        assert contact1 not in ac1.get_contacts()
+
+    def test_get_contacts_and_delete_fails(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact1 = ac1.create_contact("some1@example.com", name="some1")
+        msg = contact1.create_chat().send_text("one message")
+        assert not ac1.delete_contact(contact1)
+        assert not msg.filemime
+
+    def test_create_chat_flexibility(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        ac2 = acfactory.get_pseudo_configured_account()
+        chat1 = ac1.create_chat(ac2)
+        chat2 = ac1.create_chat(ac2.get_self_contact().addr)
+        assert chat1 == chat2
+        ac3 = acfactory.get_unconfigured_account()
+        with pytest.raises(ValueError):
+            ac1.create_chat(ac3)
+
+    def test_contact_rename(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact = ac1.create_contact("some1@example.com", name="some1")
+        chat = ac1.create_chat(contact)
+        assert chat.get_name() == "some1"
+        ac1.create_contact("some1@example.com", name="renamed")
+        ev = ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
+        assert ev.data1 == chat.id
+        assert chat.get_name() == "renamed"
+
+
+class TestOfflineChat:
+    @pytest.fixture
+    def ac1(self, acfactory):
+        return acfactory.get_pseudo_configured_account()
+
+    @pytest.fixture
+    def chat1(self, ac1):
+        return ac1.create_contact("some1@example.org", name="some1").create_chat()
+
+    def test_display(self, chat1):
+        str(chat1)
+        repr(chat1)
+
+    def test_is_group(self, chat1):
+        assert not chat1.is_group()
+
+    def test_chat_by_id(self, chat1):
+        chat2 = chat1.account.get_chat_by_id(chat1.id)
+        assert chat2 == chat1
+        with pytest.raises(ValueError):
+            chat1.account.get_chat_by_id(123123)
+
+    def test_chat_idempotent(self, chat1, ac1):
+        contact1 = chat1.get_contacts()[0]
+        chat2 = contact1.create_chat()
+        assert chat2.id == chat1.id
+        assert chat2.get_name() == chat1.get_name()
+        assert chat1 == chat2
+        assert not (chat1 != chat2)
+
+        for ichat in ac1.get_chats():
+            if ichat.id == chat1.id:
+                break
+        else:
+            pytest.fail("could not find chat")
+
+    def test_group_chat_add_second_account(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        ac2 = acfactory.get_pseudo_configured_account()
+        chat = ac1.create_group_chat(name="title1")
+        with pytest.raises(ValueError):
+            chat.add_contact(ac2.get_self_contact())
+        contact = chat.add_contact(ac2)
+        assert contact.addr == ac2.get_config("addr")
+        assert contact.name == ac2.get_config("displayname")
+        assert contact.account == ac1
+        chat.remove_contact(ac2)
+
+    def test_group_chat_creation(self, ac1):
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact2 = ac1.create_contact("some2@example.org", name="some2")
+        chat = ac1.create_group_chat(name="title1", contacts=[contact1, contact2])
+        assert chat.get_name() == "title1"
+        assert contact1 in chat.get_contacts()
+        assert contact2 in chat.get_contacts()
+        assert not chat.is_promoted()
+        chat.set_name("title2")
+        assert chat.get_name() == "title2"
+
+        d = chat.get_summary()
+        print(d)
+        assert d["id"] == chat.id
+        assert d["type"] == chat.get_type()
+        assert d["name"] == chat.get_name()
+        assert d["archived"] == chat.is_archived()
+        # assert d["param"] == chat.param
+        assert d["color"] == chat.get_color()
+        assert d["profile_image"] == "" if chat.get_profile_image() is None else chat.get_profile_image()
+        assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
+
+    def test_group_chat_creation_with_translation(self, ac1):
+        ac1.set_stock_translation(const.DC_STR_MSGGRPNAME, "abc %1$s xyz %2$s")
+        ac1._evtracker.consume_events()
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(const.DC_STR_FILE, "xyz %1$s")
+        ac1._evtracker.get_matching("DC_EVENT_WARNING")
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(const.DC_STR_CONTACT_NOT_VERIFIED, "xyz %2$s")
+        ac1._evtracker.get_matching("DC_EVENT_WARNING")
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(500, "xyz %1$s")
+        ac1._evtracker.get_matching("DC_EVENT_WARNING")
+        chat = ac1.create_group_chat(name="homework", contacts=[])
+        assert chat.get_name() == "homework"
+        chat.send_text("Now we have a group for homework")
+        assert chat.is_promoted()
+        chat.set_name("Homework")
+        assert chat.get_messages()[-1].text == "abc homework xyz Homework by me."
+
+    @pytest.mark.parametrize("verified", [True, False])
+    def test_group_chat_qr(self, acfactory, ac1, verified):
+        ac2 = acfactory.get_pseudo_configured_account()
+        chat = ac1.create_group_chat(name="title1", verified=verified)
+        assert chat.is_group()
+        qr = chat.get_join_qr()
+        assert ac2.check_qr(qr).is_ask_verifygroup
+
+    def test_removing_blocked_user_from_group(self, ac1, lp):
+        """
+        Test that blocked contact is not unblocked when removed from a group.
+        See https://github.com/deltachat/deltachat-core-rust/issues/2030
+        """
+        lp.sec("Create a group chat with a contact")
+        contact = ac1.create_contact("some1@example.org")
+        group = ac1.create_group_chat("title", contacts=[contact])
+        group.send_text("First group message")
+
+        lp.sec("ac1 blocks contact")
+        contact.block()
+        assert contact.is_blocked()
+
+        lp.sec("ac1 removes contact from their group")
+        group.remove_contact(contact)
+        assert contact.is_blocked()
+
+        lp.sec("ac1 adding blocked contact unblocks it")
+        group.add_contact(contact)
+        assert not contact.is_blocked()
+
+    def test_get_set_profile_image_simple(self, ac1, data):
+        chat = ac1.create_group_chat(name="title1")
+        p = data.get_path("d.png")
+        chat.set_profile_image(p)
+        p2 = chat.get_profile_image()
+        assert open(p, "rb").read() == open(p2, "rb").read()
+        chat.remove_profile_image()
+        assert chat.get_profile_image() is None
+
+    def test_mute(self, ac1):
+        chat = ac1.create_group_chat(name="title1")
+        assert not chat.is_muted()
+        assert chat.get_mute_duration() == 0
+        chat.mute()
+        assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
+        chat.unmute()
+        assert not chat.is_muted()
+        chat.mute(50)
+        assert chat.is_muted()
+        assert chat.get_mute_duration() <= 50
+        with pytest.raises(ValueError):
+            chat.mute(-51)
+
+        # Regression test, this caused Rust panic previously
+        chat.mute(2**63 - 1)
+        assert chat.is_muted()
+        assert chat.get_mute_duration() == -1
+
+    def test_delete_and_send_fails(self, ac1, chat1):
+        chat1.delete()
+        ac1._evtracker.wait_next_messages_changed()
+        with pytest.raises(ValueError):
+            chat1.send_text("msg1")
+
+    def test_prepare_message_and_send(self, ac1, chat1):
+        msg = chat1.prepare_message(Message.new_empty(chat1.account, "text"))
+        msg.set_text("hello world")
+        assert msg.text == "hello world"
+        assert msg.id > 0
+        chat1.send_prepared(msg)
+        assert "Sent" in msg.get_message_info()
+        str(msg)
+        repr(msg)
+        assert msg == ac1.get_message_by_id(msg.id)
+
+    def test_prepare_file(self, ac1, chat1):
+        blobdir = ac1.get_blobdir()
+        p = os.path.join(blobdir, "somedata.txt")
+        with open(p, "w") as f:
+            f.write("some data")
+        message = chat1.prepare_message_file(p)
+        assert message.id > 0
+        message.set_text("hello world")
+        assert message.is_out_preparing()
+        assert message.text == "hello world"
+        chat1.send_prepared(message)
+        assert "Sent" in message.get_message_info()
+
+    def test_message_eq_contains(self, chat1):
+        msg = chat1.send_text("msg1")
+        assert msg in chat1.get_messages()
+        assert not (msg not in chat1.get_messages())
+        str(msg)
+        repr(msg)
+
+    def test_message_send_text(self, chat1):
+        msg = chat1.send_text("msg1")
+        assert msg
+        assert msg.is_text()
+        assert not msg.is_audio()
+        assert not msg.is_video()
+        assert not msg.is_gif()
+        assert not msg.is_file()
+        assert not msg.is_image()
+
+        assert not msg.is_in_fresh()
+        assert not msg.is_in_noticed()
+        assert not msg.is_in_seen()
+        assert msg.is_out_pending()
+        assert not msg.is_out_failed()
+        assert not msg.is_out_delivered()
+        assert not msg.is_out_mdn_received()
+
+    def test_message_image(self, chat1, data, lp):
+        with pytest.raises(ValueError):
+            chat1.send_image(path="notexists")
+        fn = data.get_path("d.png")
+        lp.sec("sending image")
+        chat1.account._evtracker.consume_events()
+        msg = chat1.send_image(fn)
+        chat1.account._evtracker.get_matching("DC_EVENT_NEW_BLOB_FILE")
+        assert msg.is_image()
+        assert msg
+        assert msg.id > 0
+        assert os.path.exists(msg.filename)
+        assert msg.filemime == "image/png"
+
+    @pytest.mark.parametrize("typein,typeout", [
+            (None, "application/octet-stream"),
+            ("text/plain", "text/plain"),
+            ("image/png", "image/png"),
+    ])
+    def test_message_file(self, ac1, chat1, data, lp, typein, typeout):
+        lp.sec("sending file")
+        fn = data.get_path("r.txt")
+        msg = chat1.send_file(fn, typein)
+        assert msg
+        assert msg.id > 0
+        assert msg.is_file()
+        assert os.path.exists(msg.filename)
+        assert msg.filename.endswith(msg.basename)
+        assert msg.filemime == typeout
+        msg2 = chat1.send_file(fn, typein)
+        assert msg2 != msg
+        assert msg2.filename != msg.filename
+
+    def test_create_contact(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        email = "hello <hello@example.org>"
+        contact1 = ac1.create_contact(email)
+        assert contact1.addr == "hello@example.org"
+        assert contact1.name == "hello"
+        contact1 = ac1.create_contact(email, name="world")
+        assert contact1.name == "world"
+        contact2 = ac1.create_contact("display1 <x@example.org>", "real")
+        assert contact2.name == "real"
+
+    def test_create_chat_simple(self, acfactory):
+        ac1 = acfactory.get_pseudo_configured_account()
+        contact1 = ac1.create_contact("some1@example.org", name="some1")
+        contact1.create_chat().send_text("hello")
+
+    def test_chat_message_distinctions(self, ac1, chat1):
+        past1s = datetime.now(timezone.utc) - timedelta(seconds=1)
+        msg = chat1.send_text("msg1")
+        ts = msg.time_sent
+        assert msg.time_received is None
+        assert ts.strftime("Y")
+        assert past1s < ts
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+    def test_set_config_after_configure_is_forbidden(self, ac1):
+        assert ac1.get_config("mail_pw")
+        assert ac1.is_configured()
+        with pytest.raises(ValueError):
+            ac1.set_config("addr", "123@example.org")
+
+    def test_import_export_one_contact(self, acfactory, tmpdir):
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account()
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+        assert not backupdir.listdir()
+        ac1.stop_io()
+        path = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path)
+        ac2 = acfactory.get_unconfigured_account()
+        ac2.import_all(path)
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_set_get_draft(self, chat1):
+        msg = Message.new_empty(chat1.account, "text")
+        msg1 = chat1.prepare_message(msg)
+        msg1.set_text("hello")
+        chat1.set_draft(msg1)
+        msg1.set_text("obsolete")
+        msg2 = chat1.get_draft()
+        assert msg2.text == "hello"
+        chat1.set_draft(None)
+        assert chat1.get_draft() is None
+
+    def test_qr_setup_contact(self, acfactory, lp):
+        ac1 = acfactory.get_pseudo_configured_account()
+        ac2 = acfactory.get_pseudo_configured_account()
+        qr = ac1.get_setup_contact_qr()
+        assert qr.startswith("OPENPGP4FPR:")
+        res = ac2.check_qr(qr)
+        assert res.is_ask_verifycontact()
+        assert not res.is_ask_verifygroup()
+        assert res.contact_id == 10
+
+    def test_quote(self, chat1):
+        """Offline quoting test"""
+        msg = Message.new_empty(chat1.account, "text")
+        msg.set_text("Multi\nline\nmessage")
+        assert msg.quoted_text is None
+
+        # Prepare message to assign it a Message-Id.
+        # Messages without Message-Id cannot be quoted.
+        msg = chat1.prepare_message(msg)
+
+        reply_msg = Message.new_empty(chat1.account, "text")
+        reply_msg.set_text("reply")
+        reply_msg.quote = msg
+        assert reply_msg.quoted_text == "Multi\nline\nmessage"
+
+    def test_group_chat_many_members_add_remove(self, ac1, lp):
+        lp.sec("ac1: creating group chat with 10 other members")
+        chat = ac1.create_group_chat(name="title1")
+        # promote chat
+        chat.send_text("hello")
+        assert chat.is_promoted()
+
+        # activate local plugin
+        in_list = []
+
+        class InPlugin:
+            @account_hookimpl
+            def ac_member_added(self, chat, contact, actor):
+                in_list.append(("added", chat, contact, actor))
+
+            @account_hookimpl
+            def ac_member_removed(self, chat, contact, actor):
+                in_list.append(("removed", chat, contact, actor))
+
+        ac1.add_account_plugin(InPlugin())
+
+        # perform add contact many times
+        contacts = []
+        for i in range(10):
+            lp.sec("create contact")
+            contact = ac1.create_contact("some{}@example.org".format(i))
+            contacts.append(contact)
+            lp.sec("add contact")
+            chat.add_contact(contact)
+
+        assert chat.num_contacts() == 11
+
+        # let's make sure the events perform plugin hooks
+        def wait_events(cond):
+            now = time.time()
+            while time.time() < now + 5:
+                if cond():
+                    break
+                time.sleep(0.1)
+            else:
+                pytest.fail("failed to get events")
+
+        wait_events(lambda: len(in_list) == 10)
+
+        assert len(in_list) == 10
+        chat_contacts = chat.get_contacts()
+        for in_cmd, in_chat, in_contact, in_actor in in_list:
+            assert in_cmd == "added"
+            assert in_chat == chat
+            assert in_contact in chat_contacts
+            assert in_actor is None
+            chat_contacts.remove(in_contact)
+
+        assert chat_contacts[0].id == 1  # self contact
+
+        in_list[:] = []
+
+        lp.sec("ac1: removing two contacts and checking things are right")
+        chat.remove_contact(contacts[9])
+        chat.remove_contact(contacts[3])
+        assert chat.num_contacts() == 9
+
+        wait_events(lambda: len(in_list) == 2)
+        assert len(in_list) == 2
+        assert in_list[0][0] == "removed"
+        assert in_list[0][1] == chat
+        assert in_list[0][2] == contacts[9]
+        assert in_list[1][0] == "removed"
+        assert in_list[1][1] == chat
+        assert in_list[1][2] == contacts[3]
+
+    def test_audit_log_view_without_daymarker(self, ac1, lp):
+        lp.sec("ac1: test audit log (show only system messages)")
+        chat = ac1.create_group_chat(name="audit log sample data")
+        # promote chat
+        chat.send_text("hello")
+        assert chat.is_promoted()
+
+        lp.sec("create test data")
+        chat.add_contact(ac1.create_contact("some-1@example.org"))
+        chat.set_name("audit log test group")
+        chat.send_text("a message in between")
+
+        lp.sec("check message count of all messages")
+        assert len(chat.get_messages()) == 4
+
+        lp.sec("check message count of only system messages (without daymarkers)")
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
+            lib.dc_array_unref
+        )
+        assert len(list(iter_array(dc_array, lambda x: x))) == 2

python/tests/test_4_lowlevel.py

@@ -0,0 +1,199 @@
+
+import os
+
+from queue import Queue
+from deltachat import capi, cutil, const
+from deltachat import register_global_plugin
+from deltachat.hookspec import global_hookimpl
+from deltachat.capi import ffi
+from deltachat.capi import lib
+from deltachat.testplugin import ACSetup
+# from deltachat.account import EventLogger
+
+
+class TestACSetup:
+    def test_basic_states(self, acfactory, monkeypatch, testprocess):
+        pc = ACSetup(init_time=0.0, testprocess=testprocess)
+        acc = acfactory.get_unconfigured_account()
+        monkeypatch.setattr(acc, "configure", lambda **kwargs: None)
+        pc.start_configure(acc)
+        assert pc._account2state[acc] == pc.CONFIGURING
+        pc._configured_events.put((acc, True))
+        monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
+        pc.wait_one_configured(acc)
+        assert pc._account2state[acc] == pc.CONFIGURED
+        monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None)
+        pc.bring_online()
+        assert pc._account2state[acc] == pc.IDLEREADY
+
+    def test_two_accounts_one_waited_all_started(self, monkeypatch, acfactory, testprocess):
+        pc = ACSetup(init_time=0.0, testprocess=testprocess)
+        monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
+        monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None)
+        ac1 = acfactory.get_unconfigured_account()
+        monkeypatch.setattr(ac1, "configure", lambda **kwargs: None)
+        pc.start_configure(ac1)
+        ac2 = acfactory.get_unconfigured_account()
+        monkeypatch.setattr(ac2, "configure", lambda **kwargs: None)
+        pc.start_configure(ac2)
+        assert pc._account2state[ac1] == pc.CONFIGURING
+        assert pc._account2state[ac2] == pc.CONFIGURING
+        pc._configured_events.put((ac1, True))
+        pc.wait_one_configured(ac1)
+        assert pc._account2state[ac1] == pc.CONFIGURED
+        assert pc._account2state[ac2] == pc.CONFIGURING
+        pc._configured_events.put((ac2, True))
+        pc.bring_online()
+        assert pc._account2state[ac1] == pc.IDLEREADY
+        assert pc._account2state[ac2] == pc.IDLEREADY
+
+    def test_store_and_retrieve_configured_account_cache(self, acfactory, tmpdir):
+        ac1 = acfactory.get_pseudo_configured_account()
+        holder = acfactory._acsetup.testprocess
+        assert holder.cache_maybe_store_configured_db_files(ac1)
+        assert not holder.cache_maybe_store_configured_db_files(ac1)
+        acdir = tmpdir.mkdir("newaccount")
+        addr = ac1.get_config("addr")
+        target_db_path = acdir.join("db").strpath
+        assert holder.cache_maybe_retrieve_configured_db_files(addr, target_db_path)
+        assert len(os.listdir(acdir)) >= 2
+
+
+def test_liveconfig_caching(acfactory, monkeypatch):
+    prod = [
+        {"addr": "1@example.org", "mail_pw": "123"},
+    ]
+    acfactory._liveconfig_producer = iter(prod)
+    d1 = acfactory.get_next_liveconfig()
+    d1["hello"] = "world"
+    acfactory._liveconfig_producer = iter(prod)
+    d2 = acfactory.get_next_liveconfig()
+    assert "hello" not in d2
+
+
+def test_empty_context():
+    ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
+    capi.lib.dc_context_unref(ctx)
+
+
+def test_dc_close_events(tmpdir, acfactory):
+    ac1 = acfactory.get_unconfigured_account()
+
+    # register after_shutdown function
+    shutdowns = Queue()
+
+    class ShutdownPlugin:
+        @global_hookimpl
+        def dc_account_after_shutdown(self, account):
+            assert account._dc_context is None
+            shutdowns.put(account)
+    register_global_plugin(ShutdownPlugin())
+    assert hasattr(ac1, "_dc_context")
+    ac1.shutdown()
+    shutdowns.get(timeout=2)
+
+
+def test_wrong_db(tmpdir):
+    p = tmpdir.join("hello.db")
+    # write an invalid database file
+    p.write("x123" * 10)
+
+    context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
+    assert not lib.dc_context_is_open(context)
+
+
+def test_empty_blobdir(tmpdir):
+    db_fname = tmpdir.join("hello.db")
+    # Apparently some client code expects this to be the same as passing NULL.
+    ctx = ffi.gc(
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
+        lib.dc_context_unref,
+    )
+    assert ctx != ffi.NULL
+
+
+def test_event_defines():
+    assert const.DC_EVENT_INFO == 100
+    assert const.DC_CONTACT_ID_SELF
+
+
+def test_sig():
+    sig = capi.lib.dc_event_has_string_data
+    assert not sig(const.DC_EVENT_MSGS_CHANGED)
+    assert sig(const.DC_EVENT_INFO)
+    assert sig(const.DC_EVENT_WARNING)
+    assert sig(const.DC_EVENT_ERROR)
+    assert sig(const.DC_EVENT_SMTP_CONNECTED)
+    assert sig(const.DC_EVENT_IMAP_CONNECTED)
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
+    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
+
+
+def test_markseen_invalid_message_ids(acfactory):
+    ac1 = acfactory.get_pseudo_configured_account()
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
+    chat.send_text("one messae")
+    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    msg_ids = [9]
+    lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+
+
+def test_get_special_message_id_returns_empty_message(acfactory):
+    ac1 = acfactory.get_pseudo_configured_account()
+    for i in range(1, 10):
+        msg = ac1.get_message_by_id(i)
+        assert msg.id == 0
+
+
+def test_provider_info_none():
+    ctx = ffi.gc(
+        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
+
+
+def test_get_info_open(tmpdir):
+    db_fname = tmpdir.join("test.db")
+    ctx = ffi.gc(
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
+        lib.dc_context_unref,
+    )
+    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
+    assert 'deltachat_core_version' in info
+    assert 'database_dir' in info
+
+
+def test_logged_hook_failure(acfactory):
+    ac1 = acfactory.get_pseudo_configured_account()
+    cap = []
+    ac1.log = cap.append
+    with ac1._event_thread.swallow_and_log_exception("some"):
+        0/0
+    assert cap
+    assert "some" in str(cap)
+    assert "ZeroDivisionError" in str(cap)
+    assert "Traceback" in str(cap)
+
+
+def test_logged_ac_process_ffi_failure(acfactory):
+    from deltachat import account_hookimpl
+
+    ac1 = acfactory.get_pseudo_configured_account()
+
+    class FailPlugin:
+        @account_hookimpl
+        def ac_process_ffi_event(ffi_event):
+            0/0
+
+    cap = Queue()
+    ac1.log = cap.put
+    ac1.add_account_plugin(FailPlugin())
+    # cause any event eg contact added/changed
+    ac1.create_contact("something@example.org")
+    res = cap.get(timeout=10)
+    assert "ac_process_ffi_event" in res
+    assert "ZeroDivisionError" in res
+    assert "Traceback" in res

python/tests/test_lowlevel.py

@@ -1,105 +0,0 @@
-from __future__ import print_function
-
-from queue import Queue
-from deltachat import capi, cutil, const
-from deltachat import register_global_plugin
-from deltachat.hookspec import global_hookimpl
-from deltachat.capi import ffi
-from deltachat.capi import lib
-# from deltachat.account import EventLogger
-
-
-def test_empty_context():
-    ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_context_unref(ctx)
-
-
-def test_dc_close_events(tmpdir, acfactory):
-    ac1 = acfactory.get_unconfigured_account()
-
-    # register after_shutdown function
-    shutdowns = Queue()
-
-    class ShutdownPlugin:
-        @global_hookimpl
-        def dc_account_after_shutdown(self, account):
-            assert account._dc_context is None
-            shutdowns.put(account)
-    register_global_plugin(ShutdownPlugin())
-    assert hasattr(ac1, "_dc_context")
-    ac1.shutdown()
-    shutdowns.get(timeout=2)
-
-
-def test_wrong_db(tmpdir):
-    p = tmpdir.join("hello.db")
-    # write an invalid database file
-    p.write("x123" * 10)
-
-    context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
-    assert not lib.dc_context_is_open(context)
-
-
-def test_empty_blobdir(tmpdir):
-    db_fname = tmpdir.join("hello.db")
-    # Apparently some client code expects this to be the same as passing NULL.
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
-        lib.dc_context_unref,
-    )
-    assert ctx != ffi.NULL
-
-
-def test_event_defines():
-    assert const.DC_EVENT_INFO == 100
-    assert const.DC_CONTACT_ID_SELF
-
-
-def test_sig():
-    sig = capi.lib.dc_event_has_string_data
-    assert not sig(const.DC_EVENT_MSGS_CHANGED)
-    assert sig(const.DC_EVENT_INFO)
-    assert sig(const.DC_EVENT_WARNING)
-    assert sig(const.DC_EVENT_ERROR)
-    assert sig(const.DC_EVENT_SMTP_CONNECTED)
-    assert sig(const.DC_EVENT_IMAP_CONNECTED)
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
-    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
-
-
-def test_markseen_invalid_message_ids(acfactory):
-    ac1 = acfactory.get_configured_offline_account()
-
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
-    chat = contact1.create_chat()
-    chat.send_text("one messae")
-    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    msg_ids = [9]
-    lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
-
-
-def test_get_special_message_id_returns_empty_message(acfactory):
-    ac1 = acfactory.get_configured_offline_account()
-    for i in range(1, 10):
-        msg = ac1.get_message_by_id(i)
-        assert msg.id == 0
-
-
-def test_provider_info_none():
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
-
-
-def test_get_info_open(tmpdir):
-    db_fname = tmpdir.join("test.db")
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
-        lib.dc_context_unref,
-    )
-    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' in info

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n1 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

src/context.rs

@@ -202,6 +202,11 @@ impl Context {
 
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
+        // Sending an event wakes up event pollers (get_next_event)
+        // so the caller of stop_io() can arrange for proper termination.
+        // For this, the caller needs to instruct the event poller
+        // to terminate on receiving the next event and then call stop_io()
+        // which will emit the below event(s)
         info!(self, "stopping IO");
 
         if let Err(err) = self.inner.stop_io().await {

src/provider/data.rs

@@ -44,7 +44,7 @@ static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
             protocol: Smtp,
             socket: Starttls,
             hostname: "newyear.aktivix.org",
-            port: 25,
+            port: 587,
             username_pattern: Email,
         },
     ],
@@ -851,10 +851,6 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::MvboxMove,
             value: "0",
         },
-        ConfigDefault {
-            key: Config::E2eeEnabled,
-            value: "0",
-        },
         ConfigDefault {
             key: Config::MediaQuality,
             value: "1",
@@ -983,7 +979,7 @@ static P_QQ: Lazy<Provider> = Lazy::new(|| {
     overview_page: "https://providers.delta.chat/qq",
     server: vec![
         Server { protocol: Imap, socket: Ssl, hostname: "imap.qq.com", port: 993, username_pattern: Emaillocalpart },
-        Server { protocol: Smtp, socket: Starttls, hostname: "smtp.qq.com", port: 465, username_pattern: Email },
+        Server { protocol: Smtp, socket: Ssl, hostname: "smtp.qq.com", port: 465, username_pattern: Email },
     ],
     config_defaults: None,
     strict_tls: true,
@@ -1402,7 +1398,9 @@ static P_YGGMAIL: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: Imap, socket: Plain, hostname: "localhost", port: 1143, username_pattern: Email },
         Server { protocol: Smtp, socket: Plain, hostname: "localhost", port: 1025, username_pattern: Email },
     ],
-    config_defaults: None,
+    config_defaults: Some(vec![
+        ConfigDefault { key: Config::MvboxMove, value: "0" },
+    ]),
     strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
@@ -1853,4 +1851,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2022, 1, 31));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2022, 5, 3));

src/qr_code_generator.rs

@@ -65,13 +65,12 @@ async fn generate_verification_qr(context: &Context) -> Result<String> {
 }
 
 fn inner_generate_secure_join_qr_code(
-    raw_qrcode_description: &str,
+    qrcode_description: &str,
     qrcode_content: &str,
     color: &str,
     avatar: Option<Vec<u8>>,
     avatar_letter: char,
 ) -> Result<String> {
-    let qrcode_description = &escaper::encode_minimal(raw_qrcode_description);
     // config
     let width = 515.0;
     let height = 630.0;
@@ -262,3 +261,21 @@ fn inner_generate_secure_join_qr_code(
 
     Ok(svg)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_svg_escaping() {
+        let svg = inner_generate_secure_join_qr_code(
+            "descr123 \" < > &",
+            "qr-code-content",
+            "#000000",
+            None,
+            'X',
+        )
+        .unwrap();
+        assert!(svg.contains("descr123 &quot; &lt; &gt; &amp;"))
+    }
+}

src/scheduler.rs

@@ -378,7 +378,7 @@ impl Scheduler {
     /// Start the scheduler, returns error if it is already running.
     pub async fn start(&mut self, ctx: Context) -> Result<()> {
         if self.is_running() {
-            bail!("scheduler is already stopped");
+            bail!("scheduler is already started");
         }
 
         let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?;

src/smtp.rs

@@ -420,7 +420,7 @@ pub(crate) async fn send_msg_to_smtp(
     }
     info!(
         context,
-        "Retry number {} to send message {} over SMTP", retries, msg_id
+        "Try number {} to send message {} over SMTP", retries, msg_id
     );
 
     let recipients_list = recipients