From c8bfa98b6bdcffd9a6b49711d551e3fe4e26f6e8 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 5 May 2022 13:13:54 +0200 Subject: [PATCH] actually enable online account caching. previously it was creating (>100) online test accounts --- python/src/deltachat/direct_imap.py | 3 ++ python/src/deltachat/testplugin.py | 44 ++++++++++++++--------------- python/tests/bench_test_setup.py | 6 ++-- python/tests/test_1_online.py | 2 +- python/tests/test_4_lowlevel.py | 24 ++++++++++++---- 5 files changed, 48 insertions(+), 31 deletions(-) diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index 240fa4988..94943cb71 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -177,6 +177,9 @@ class IdleManager: def __init__(self, direct_imap): self.direct_imap = direct_imap self.log = direct_imap.account.log + # fetch latest messages before starting idle so that it only + # returns messages that arrive anew + self.direct_imap.conn.fetch("1:*") self.direct_imap.conn.idle.start() def check(self, timeout=None) -> List[bytes]: diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 1b2590e7c..7ccbe4279 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -138,6 +138,8 @@ class TestProcess: """ def __init__(self, pytestconfig): self.pytestconfig = pytestconfig + self._addr2files = {} + self._configlist = [] def get_liveconfig_producer(self): """ provide live account configs, cached on a per-test-process scope @@ -149,7 +151,6 @@ class TestProcess: if not liveconfig_opt: pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts") - configlist = [] if not liveconfig_opt.startswith("http"): for line in open(liveconfig_opt): if line.strip() and not line.strip().startswith('#'): @@ -157,14 +158,14 @@ class TestProcess: for part in line.split(): name, value = part.split("=") d[name] = value - configlist.append(d) + self._configlist.append(d) - yield from iter(configlist) + yield from iter(self._configlist) else: MAX_LIVE_CREATED_ACCOUNTS = 10 for index in range(MAX_LIVE_CREATED_ACCOUNTS): try: - yield configlist[index] + yield self._configlist[index] except IndexError: res = requests.post(liveconfig_opt) if res.status_code != 200: @@ -173,7 +174,7 @@ class TestProcess: d = res.json() config = dict(addr=d["email"], mail_pw=d["password"]) print("newtmpuser {}: addr={}".format(index, config["addr"])) - configlist.append(config) + self._configlist.append(config) yield config pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS)) @@ -217,10 +218,11 @@ class ACSetup: CONFIGURED = "CONFIGURED" IDLEREADY = "IDLEREADY" - def __init__(self, init_time): + def __init__(self, testprocess, init_time): self._configured_events = Queue() self._account2state = {} self._imap_cleaned = set() + self.testprocess = testprocess self.init_time = init_time def start_configure(self, account, reconfigure=False): @@ -242,7 +244,8 @@ class ACSetup: acc = self._pop_config_success() if acc == account: break - self.init_direct_imap_and_logging(acc) + self.init_imap(acc) + self.init_logging(acc) acc._evtracker.consume_events() def bring_online(self): @@ -272,25 +275,24 @@ class ACSetup: return acc def _onconfigure_start_io(self, acc): + self.init_imap(acc) + self.init_logging(acc) acc.start_io() print(acc._logid, "waiting for inbox IDLE to become ready") acc._evtracker.wait_idle_inbox_ready() - self.init_direct_imap_and_logging(acc) acc._evtracker.consume_events() acc.log("inbox IDLE ready") - def init_direct_imap_and_logging(self, acc): - """ idempotent function for initializing direct_imap and logging for an account. """ - self.init_direct_imap(acc) - self.init_logging(acc) - def init_logging(self, acc): + """ idempotent function for initializing logging (will replace existing logger). """ logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time) - acc.add_account_plugin(logger, name=acc._logid) + acc.add_account_plugin(logger, name="logger-" + acc._logid) - def init_direct_imap(self, acc): - """ idempotent function for initializing direct_imap.""" + def init_imap(self, acc): + """ initialize direct_imap and cleanup server state. """ from deltachat.direct_imap import DirectImap + + assert acc.is_configured() if not hasattr(acc, "direct_imap"): acc.direct_imap = DirectImap(acc) addr = acc.get_config("addr") @@ -315,11 +317,12 @@ class ACFactory: self.tmpdir = tmpdir self.pytestconfig = request.config self.data = data + self.testprocess = testprocess self._liveconfig_producer = testprocess.get_liveconfig_producer() self._finalizers = [] self._accounts = [] - self._acsetup = ACSetup(self.init_time) + self._acsetup = ACSetup(testprocess, self.init_time) self._preconfigured_keys = ["alice", "bob", "charlie", "dom", "elena", "fiona"] self.set_logging_default(False) @@ -344,7 +347,7 @@ class ACFactory: """ Base function to get functional online configurations where we can make valid SMTP and IMAP connections with. """ - configdict = next(self._liveconfig_producer) + configdict = next(self._liveconfig_producer).copy() if "e2ee_enabled" not in configdict: configdict["e2ee_enabled"] = "1" @@ -480,10 +483,6 @@ class ACFactory: ac.dump_account_info(logfile=logfile) imap = getattr(ac, "direct_imap", None) if imap is not None: - try: - imap.idle_done() - except Exception: - pass imap.dump_imap_structures(self.tmpdir, logfile=logfile) def get_accepted_chat(self, ac1: Account, ac2: Account): @@ -501,6 +500,7 @@ class ACFactory: acc2.create_chat(acc).send_text("hi back") to_wait.append(acc) for acc in to_wait: + acc.log("waiting for incoming message") acc._evtracker.wait_next_incoming_message() diff --git a/python/tests/bench_test_setup.py b/python/tests/bench_test_setup.py index 9636d64c7..5dc5e3877 100644 --- a/python/tests/bench_test_setup.py +++ b/python/tests/bench_test_setup.py @@ -8,11 +8,13 @@ to see timings of test setups. import pytest +BENCH_NUM = 3 + class TestEmpty: def test_prepare_setup_measurings(self, acfactory): - acfactory.get_online_accounts(5) + acfactory.get_online_accounts(BENCH_NUM) - @pytest.mark.parametrize("num", range(0, 5)) + @pytest.mark.parametrize("num", range(0, BENCH_NUM + 1)) def test_setup_online_accounts(self, acfactory, num): acfactory.get_online_accounts(num) diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index af7057af1..7d19ec57d 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -419,8 +419,8 @@ def test_send_and_receive_message_markseen(acfactory, lp): assert msg2.chat.id == msg4.chat.id assert ev.data1 == msg2.chat.id assert ev.data2 == 0 + idle2.wait_for_seen() - idle2.wait_for_new_message() lp.step("1") for i in range(2): ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ") diff --git a/python/tests/test_4_lowlevel.py b/python/tests/test_4_lowlevel.py index efede80b8..9983c23ba 100644 --- a/python/tests/test_4_lowlevel.py +++ b/python/tests/test_4_lowlevel.py @@ -11,23 +11,23 @@ from deltachat.testplugin import ACSetup class TestACSetup: - def test_basic_states(self, acfactory, monkeypatch): - pc = ACSetup(init_time=0.0) + def test_basic_states(self, acfactory, monkeypatch, testprocess): + pc = ACSetup(init_time=0.0, testprocess=testprocess) acc = acfactory.get_unconfigured_account() monkeypatch.setattr(acc, "configure", lambda **kwargs: None) pc.start_configure(acc) assert pc._account2state[acc] == pc.CONFIGURING pc._configured_events.put((acc, True)) - monkeypatch.setattr(pc, "init_direct_imap", lambda *args, **kwargs: None) + monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None) pc.wait_one_configured(acc) assert pc._account2state[acc] == pc.CONFIGURED monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None) pc.bring_online() assert pc._account2state[acc] == pc.IDLEREADY - def test_two_accounts_one_waited_all_started(self, monkeypatch, acfactory): - pc = ACSetup(init_time=0.0) - monkeypatch.setattr(pc, "init_direct_imap", lambda *args, **kwargs: None) + def test_two_accounts_one_waited_all_started(self, monkeypatch, acfactory, testprocess): + pc = ACSetup(init_time=0.0, testprocess=testprocess) + monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None) monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None) ac1 = acfactory.get_unconfigured_account() monkeypatch.setattr(ac1, "configure", lambda **kwargs: None) @@ -47,6 +47,18 @@ class TestACSetup: assert pc._account2state[ac2] == pc.IDLEREADY +def test_liveconfig_caching(acfactory, monkeypatch): + prod = [ + {"addr": "1@example.org", "mail_pw": "123"}, + ] + acfactory._liveconfig_producer = iter(prod) + d1 = acfactory.get_next_liveconfig() + d1["hello"] = "world" + acfactory._liveconfig_producer = iter(prod) + d2 = acfactory.get_next_liveconfig() + assert "hello" not in d2 + + def test_empty_context(): ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL) capi.lib.dc_context_unref(ctx)