snap

python/mypy.ini

@@ -1,5 +1,8 @@
 [mypy]
 
+[mypy-py.*]
+ignore_missing_imports = True
+
 [mypy-deltachat.capi.*]
 ignore_missing_imports = True
 

python/src/deltachat/testplugin.py

@@ -13,6 +13,7 @@ from typing import List, Callable
 
 import pytest
 import requests
+import py
 
 from . import Account, const, account_hookimpl, get_core_info
 from .events import FFIEventLogger, FFIEventTracker
@@ -178,6 +179,50 @@ class TestProcess:
                     yield config
             pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS))
 
+    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
+        db_target_path = py.path.local(db_target_path)
+        assert not db_target_path.exists()
+
+        print("checking cache for", cache_addr)
+        try:
+            filescache = self._addr2files[cache_addr]
+        except KeyError:
+            print("CACHE FAIL for", cache_addr)
+            return False
+        else:
+            print("CACHE HIT for", cache_addr)
+            targetdir = db_target_path.dirpath()
+            write_dict_to_dir(filescache, targetdir)
+            return True
+
+    def cache_maybe_store_configured_db_files(self, acc):
+        addr = acc.get_config("addr")
+        assert acc.is_configured()
+        # don't overwrite existing entries
+        if addr not in self._addr2files:
+            print("storing cache for", addr)
+            basedir = py.path.local(acc.get_blobdir()).dirpath()
+            self._addr2files[addr] = create_dict_from_files_in_path(basedir)
+            return True
+
+
+def create_dict_from_files_in_path(path):
+    base = py.path.local(path)
+    cachedict = {}
+    for path in base.visit(fil=py.path.local.isfile):
+        cachedict[path.relto(base)] = path.read_binary()
+    return cachedict
+
+
+def write_dict_to_dir(dic, target_dir):
+    assert dic
+    target_dir = py.path.local(target_dir)
+    for relpath, content in dic.items():
+        path = target_dir.join(relpath)
+        if not path.dirpath().exists():
+            path.dirpath().ensure(dir=1)
+        path.write_binary(content)
+
 
 @pytest.fixture
 def data(request):
@@ -225,8 +270,22 @@ class ACSetup:
         self.testprocess = testprocess
         self.init_time = init_time
 
+    def log(self, *args):
+        print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args)
+
+    def add_configured(self, account):
+        """ add an already configured account. """
+        assert account.is_configured()
+        self._account2state[account] = self.CONFIGURED
+        self.log("added already configured account", account, account.get_config("addr"))
+        return
+
     def start_configure(self, account, reconfigure=False):
         """ add an account and start its configure process. """
+
+        if reconfigure:
+            assert account.is_configured()
+
         class PendingTracker:
             @account_hookimpl
             def ac_configure_completed(this, success):
@@ -235,7 +294,7 @@ class ACSetup:
         account.add_account_plugin(PendingTracker(), name="pending_tracker")
         self._account2state[account] = self.CONFIGURING
         account.configure(reconfigure=reconfigure)
-        print("started configure on pending", account)
+        self.log("started {}configure on".format("re-" if reconfigure else ""), account)
 
     def wait_one_configured(self, account):
         """ wait until this account has successfully configured. """
@@ -248,6 +307,11 @@ class ACSetup:
             self.init_logging(acc)
             acc._evtracker.consume_events()
 
+    def wait_all_configured(self):
+        """ Wait for all unconfigured accounts to become finished. """
+        while self.CONFIGURING in self._account2state.values():
+            self._pop_config_success()
+
     def bring_online(self):
         """ Wait for all accounts to become ready to receive messages.
 
@@ -328,6 +392,9 @@ class ACFactory:
         self.set_logging_default(False)
         request.addfinalizer(self.finalize)
 
+    def log(self, *args):
+        print("[acfactory]", "{:.3f}".format(time.time() - self.init_time), *args)
+
     def finalize(self):
         while self._finalizers:
             fin = self._finalizers.pop()
@@ -359,9 +426,20 @@ class ACFactory:
         assert "addr" in configdict and "mail_pw" in configdict
         return configdict
 
+    def _get_cached_account_copy(self, addr):
+        if addr in self.testprocess._addr2files:
+            return self._getaccount(addr)
+
     def get_unconfigured_account(self):
+        return self._getaccount()
+
+    def _getaccount(self, try_cache_addr=None):
         logid = "ac{}".format(len(self._accounts) + 1)
-        path = self.tmpdir.join(logid)
+
+        # we need to use fixed database basename for maybe_cache_* functions to work
+        path = self.tmpdir.mkdir(logid).join("dc.db")
+        if try_cache_addr:
+            self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
         ac = Account(path.strpath, logging=self._logging)
         ac._logid = logid  # later instantiated FFIEventLogger needs this
         ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
@@ -394,7 +472,7 @@ class ACFactory:
     def get_pseudo_configured_account(self):
         # do a pseudo-configured account
         ac = self.get_unconfigured_account()
-        acname = os.path.basename(ac.db_path)
+        acname = ac._logid
         addr = "{}@offline.org".format(acname)
         ac.update_config(dict(
             addr=addr, displayname=acname, mail_pw="123",
@@ -405,16 +483,54 @@ class ACFactory:
         self._acsetup.init_logging(ac)
         return ac
 
-    def new_online_configuring_account(self, cloned_from=None, **kwargs):
-        if cloned_from is None:
-            configdict = self.get_next_liveconfig()
-        else:
-            # XXX we might want to transfer the key to the new account
-            configdict = dict(
-                addr=cloned_from.get_config("addr"),
-                mail_pw=cloned_from.get_config("mail_pw"),
-            )
+    # XXX deprecate the next function?
+    def new_online_configuring_account(self, cache=False, **kwargs):
+        configdict = self.get_next_liveconfig()
         configdict.update(kwargs)
+        return self._setup_online_configuring_account(configdict, cache=cache)
+
+    def get_online_second_device(self, ac1, **kwargs):
+        ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
+        if ac2 is None:
+            # some tests setup the primary account without causing caching.
+            configdict = kwargs.copy()
+            configdict["addr"] = ac1.get_config("addr")
+            configdict["mail_pw"] = ac1.get_config("mail_pw")
+            ac2 = self._setup_online_configuring_account(configdict, cache=False)
+        elif kwargs:
+            ac2.update_config(kwargs)
+            self._acsetup.add_configured(ac2)
+
+        self.bring_accounts_online()
+        return ac2
+
+    def get_online_multidevice_setup(self, copied=True):
+        """ Provide two accounts. The second uses the same credentials
+        and if copied is True, also the same database and blobs.
+        You can use copy=False to get a typical configuration where
+        a user unsuspectingly sets up a second device and expects it to
+        "just work" not knowing that an export/import is required.
+        """
+        ac1, = self.get_online_accounts(1)
+        if copied:
+            ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
+            self._acsetup.add_configured(ac2)
+        else:
+            config2 = dict(addr=ac1.get_config("addr"), mail_pw=ac1.get_config("mail_pw"))
+            ac2 = self._setup_online_configuring_account(config2, cache=False)
+        self.bring_accounts_online()
+        return ac1, ac2
+
+    def get_online_devnull_email(self):
+        return self.get_next_liveconfig()["addr"]
+
+    def _setup_online_configuring_account(self, configdict, cache=False):
+        ac = self._get_cached_account_copy(configdict["addr"]) if cache else None
+        if ac is not None:
+            # make sure we consume a preconfig key, as if we had created a fresh account
+            self._preconfigured_keys.pop(0)
+            self._acsetup.add_configured(ac)
+            return ac
         ac = self.prepare_account_from_liveconfig(configdict)
         self._acsetup.start_configure(ac)
         return ac
@@ -438,10 +554,28 @@ class ACFactory:
         self._acsetup.bring_online()
         print("all accounts online")
 
+    def get_online_configured_accounts(self, configlist):
+        accounts = [self.new_online_configuring_account(cache=True, **config)
+                    for config in configlist]
+        self._acsetup.wait_all_configured()
+        for acc in accounts:
+            self._acsetup.init_imap(acc)
+        return accounts
+
+    def force_reconfigure(self, account):
+        self._acsetup.start_configure(account, reconfigure=True)
+
     def get_online_accounts(self, num):
-        # to reduce number of log events logging starts after accounts can receive
-        accounts = [self.new_online_configuring_account() for i in range(num)]
+        """ Return a list of configured and started Accounts.
+
+        This function creates plain online accounts and fill
+        a testprocess-scoped cache and re-use these plain accounts
+        on the next test function.
+        """
+        accounts = [self.new_online_configuring_account(cache=True) for i in range(num)]
         self.bring_accounts_online()
+        for acc in accounts:
+            self.testprocess.cache_maybe_store_configured_db_files(acc)
         return accounts
 
     def run_bot_process(self, module, ffi=True):
@@ -485,7 +619,7 @@ class ACFactory:
             if imap is not None:
                 imap.dump_imap_structures(self.tmpdir, logfile=logfile)
 
-    def get_accepted_chat(self, ac1: Account, ac2: Account):
+    def get_accepted_chat(self, ac1: Account, ac2):
         ac2.create_chat(ac1)
         return ac1.create_chat(ac2)
 

python/tests/test_0_complex_or_slow.py

@@ -192,9 +192,7 @@ def test_fetch_existing(acfactory, lp, mvbox_move):
         if mvbox_move:
             assert ac.get_config("configured_mvbox_folder")
 
-    ac1 = acfactory.new_online_configuring_account(mvbox_move=mvbox_move)
-    ac2 = acfactory.new_online_configuring_account()
-    acfactory.wait_configured(ac1)
+    ac1, ac2 = acfactory.get_online_configured_accounts([dict(mvbox_move=mvbox_move), dict()])
     ac1.direct_imap.create_folder("Sent")
     ac1.set_config("sentbox_watch", "1")
 
@@ -203,8 +201,9 @@ def test_fetch_existing(acfactory, lp, mvbox_move):
     # would also find the "Sent" folder, but it would be too late:
     # The sentbox thread, started by `start_io()`, would have seen that there is no
     # ConfiguredSentboxFolder and do nothing.
-    acfactory._acsetup.start_configure(ac1, reconfigure=True)
+    acfactory.force_reconfigure(ac1)
     acfactory.bring_accounts_online()
+
     assert_folders_configured(ac1)
 
     assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
@@ -220,10 +219,7 @@ def test_fetch_existing(acfactory, lp, mvbox_move):
     assert_folders_configured(ac1)
 
     lp.sec("create a cloned ac1 and fetch contact history during configure")
-    ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
-    ac1_clone.set_config("fetch_existing_msgs", "1")
-    acfactory.wait_configured(ac1_clone)
-    ac1_clone.start_io()
+    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
     assert_folders_configured(ac1_clone)
 
     lp.sec("check that ac2 contact was fetchted during configure")
@@ -248,10 +244,7 @@ def test_fetch_existing_msgs_group_and_single(acfactory, lp):
     So, after fetch-existing-msgs you have one contact request and one chat with the same person.
 
     See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account()
-
-    acfactory.bring_accounts_online()
+    ac1, ac2 = acfactory.get_online_accounts(2)
 
     lp.sec("receive a message")
     ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
@@ -266,12 +259,7 @@ def test_fetch_existing_msgs_group_and_single(acfactory, lp):
         assert idle1.wait_for_seen()
 
     lp.sec("Clone online account and let it fetch the existing messages")
-    ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
-    ac1_clone.set_config("fetch_existing_msgs", "1")
-    acfactory.wait_configured(ac1_clone)
-
-    ac1_clone.start_io()
-    ac1_clone._evtracker.wait_idle_inbox_ready()
+    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
 
     chats = ac1_clone.get_chats()
     assert len(chats) == 4  # two newly created chats + self-chat + device-chat
@@ -284,8 +272,8 @@ def test_fetch_existing_msgs_group_and_single(acfactory, lp):
     assert len(group_messages) == 1
     assert group_messages[0].text == "incoming, unencrypted group message"
     private_messages = private_chat.get_messages()
-    # We can't decrypt the message in this chat, so the chat is empty:
-    assert len(private_messages) == 0
+    assert len(private_messages) == 1
+    assert private_messages[0].text.startswith("outgoing, encrypted")
 
 
 def test_undecipherable_group(acfactory, lp):
@@ -307,13 +295,8 @@ def test_undecipherable_group(acfactory, lp):
     lp.sec("ac3 reinstalls DC and generates a new key")
     ac3.stop_io()
     acfactory.remove_preconfigured_keys()
-    ac4 = acfactory.new_online_configuring_account(cloned_from=ac3)
-    acfactory.wait_configured(ac4)
+    ac4 = acfactory.get_online_second_device(ac3)
     # Create contacts to make sure incoming messages are not treated as contact requests
-    chat41 = ac4.create_chat(ac1)
-    chat42 = ac4.create_chat(ac2)
-    ac4.start_io()
-    ac4._evtracker.wait_idle_inbox_ready()
 
     lp.sec("ac1: creating group chat with 2 other members")
     chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
@@ -426,10 +409,8 @@ def test_ephemeral_timer(acfactory, lp):
 
 def test_multidevice_sync_seen(acfactory, lp):
     """Test that message marked as seen on one device is marked as seen on another."""
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account()
-    ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
+    ac1, ac1_clone = acfactory.get_online_multidevice_setup()
+    ac2, = acfactory.get_online_accounts(1)
 
     ac1.set_config("bcc_self", "1")
     ac1_clone.set_config("bcc_self", "1")

python/tests/test_1_online.py

@@ -92,16 +92,13 @@ def test_export_import_self_keys(acfactory, tmpdir, lp):
 
 
 def test_one_account_send_bcc_setting(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account()
-    ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
+    ac1, ac1_clone = acfactory.get_online_multidevice_setup(copied=False)
+    ac2, = acfactory.get_online_accounts(1)
 
     # test if sent messages are copied to it via BCC.
-
     chat = acfactory.get_accepted_chat(ac1, ac2)
     self_addr = ac1.get_config("addr")
-    other_addr = ac2.get_config("addr")
+    ac2_addr = ac2.get_config("addr")
 
     lp.sec("send out message without bcc to ourselves")
     ac1.set_config("bcc_self", "0")
@@ -112,9 +109,9 @@ def test_one_account_send_bcc_setting(acfactory, lp):
     ev = ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
     assert ac1.get_config("bcc_self") == "0"
 
-    # make sure we are not sending message to ourselves
+    # make sure we are NOT sending message to ourselves
     assert self_addr not in ev.data2
-    assert other_addr in ev.data2
+    assert ac2_addr in ev.data2
 
     lp.sec("ac1: setting bcc_self=1")
     ac1.set_config("bcc_self", "1")
@@ -129,7 +126,7 @@ def test_one_account_send_bcc_setting(acfactory, lp):
 
         # now make sure we are sending message to ourselves too
         assert self_addr in ev.data2
-        assert other_addr in ev.data2
+        assert ac2_addr in ev.data2
         assert idle1.wait_for_seen()
 
     # Second client receives only second message, but not the first
@@ -863,20 +860,12 @@ def test_dont_show_emails(acfactory, lp):
 
 
 def test_no_old_msg_is_fresh(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account()
-    ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
-
-    ac1.set_config("e2ee_enabled", "0")
-    ac1_clone.set_config("e2ee_enabled", "0")
-    ac2.set_config("e2ee_enabled", "0")
+    ac1, ac1_clone = acfactory.get_online_multidevice_setup()
+    ac2, = acfactory.get_online_accounts(1)
 
     ac1_clone.set_config("bcc_self", "1")
-
     ac1.create_chat(ac2)
     ac1_clone.create_chat(ac2)
-
     ac1.get_device_chat().mark_noticed()
 
     lp.sec("Send a first message from ac2 to ac1 and check that it's 'fresh'")
@@ -888,7 +877,6 @@ def test_no_old_msg_is_fresh(acfactory, lp):
     lp.sec("Send a message from ac1_clone to ac2 and check that ac1 marks the first message as 'noticed'")
     ac1_clone.create_chat(ac2).send_text("Hi back")
     ev = ac1._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-
     assert ev.data1 == first_msg_id.chat.id
     assert ac1.create_chat(ac2).count_fresh_messages() == 0
     assert len(list(ac1.get_fresh_messages())) == 0
@@ -1239,9 +1227,7 @@ def test_ac_setup_message(acfactory, lp):
     # note that the receiving account needs to be configured and running
     # before ther setup message is send. DC does not read old messages
     # as of Jul2019
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
+    ac1, ac2 = acfactory.get_online_multidevice_setup(copied=False)
 
     lp.sec("trigger ac setup message and return setupcode")
     assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
@@ -1261,9 +1247,7 @@ def test_ac_setup_message(acfactory, lp):
 
 
 def test_ac_setup_message_twice(acfactory, lp):
-    ac1 = acfactory.new_online_configuring_account()
-    ac2 = acfactory.new_online_configuring_account(cloned_from=ac1)
-    acfactory.bring_accounts_online()
+    ac1, ac2 = acfactory.get_online_multidevice_setup(copied=False)
 
     lp.sec("trigger ac setup message but ignore")
     assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]

python/tests/test_4_lowlevel.py

@@ -1,4 +1,5 @@
-from __future__ import print_function
+
+import os
 
 from queue import Queue
 from deltachat import capi, cutil, const
@@ -46,6 +47,17 @@ class TestACSetup:
         assert pc._account2state[ac1] == pc.IDLEREADY
         assert pc._account2state[ac2] == pc.IDLEREADY
 
+    def test_store_and_retrieve_configured_account_cache(self, acfactory, tmpdir):
+        ac1 = acfactory.get_pseudo_configured_account()
+        holder = acfactory._acsetup.testprocess
+        assert holder.cache_maybe_store_configured_db_files(ac1)
+        assert not holder.cache_maybe_store_configured_db_files(ac1)
+        acdir = tmpdir.mkdir("newaccount")
+        addr = ac1.get_config("addr")
+        target_db_path = acdir.join("db").strpath
+        assert holder.cache_maybe_retrieve_configured_db_files(addr, target_db_path)
+        assert len(os.listdir(acdir)) >= 2
+
 
 def test_liveconfig_caching(acfactory, monkeypatch):
     prod = [