From 0105c831f1b075e5048cecf55b5e3e71b164cd97 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 8 Jun 2020 11:15:53 +0200 Subject: [PATCH] make direct_imap a permanent feature of online accounts --- python/src/deltachat/account.py | 2 +- python/src/deltachat/direct_imap.py | 65 +++++++++++++++++++++++++---- python/src/deltachat/hookspec.py | 10 ++++- python/src/deltachat/testplugin.py | 23 +++++----- python/src/deltachat/tracker.py | 8 +++- python/tests/test_direct_imap.py | 23 ++++------ 6 files changed, 92 insertions(+), 39 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index bf399c2de..91b17db01 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -581,7 +581,7 @@ class Account(object): raise MissingCredentials("addr or mail_pwd not set in config") if hasattr(self, "_configtracker"): self.remove_account_plugin(self._configtracker) - self._configtracker = ConfigureTracker() + self._configtracker = ConfigureTracker(self) self.add_account_plugin(self._configtracker) lib.dc_configure(self._dc_context) diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index e1feeca8d..e53317d80 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -1,19 +1,50 @@ +""" +Internal Python-level IMAP handling used by the testplugin +and for cleaning up inbox/mvbox for each test function run. +""" + import io import email import ssl import pathlib from imapclient import IMAPClient from imapclient.exceptions import IMAPClientError +import deltachat SEEN = b'\\Seen' +DELETED = b'\\Deleted' FLAGS = b'FLAGS' FETCH = b'FETCH' +ALL = "1:*" -class ImapConn: - def __init__(self, account): +@deltachat.global_hookimpl +def dc_account_extra_configure(account): + """ Reset the account (we reuse accounts across tests) + and make 'account.direct_imap' available for direct IMAP ops. + """ + imap = DirectImap(account, account.logid) + if imap.select_config_folder("mvbox"): + imap.delete(ALL, expunge=True) + assert imap.select_config_folder("inbox") + imap.delete(ALL, expunge=True) + setattr(account, "direct_imap", imap) + + +@deltachat.global_hookimpl +def dc_account_after_shutdown(account): + """ shutdown the imap connection if there is one. """ + imap = getattr(account, "direct_imap", None) + if imap is not None: + imap.shutdown() + del account.direct_imap + + +class DirectImap: + def __init__(self, account, logid): self.account = account + self.logid = logid self._idling = False self.connect() @@ -49,25 +80,39 @@ class ImapConn: return self.conn.select_folder(foldername) def select_config_folder(self, config_name): + """ Return info about selected folder if it is + configured, otherwise None. """ if "_" not in config_name: config_name = "configured_{}_folder".format(config_name) foldername = self.account.get_config(config_name) - return self.select_folder(foldername) + if foldername: + return self.select_folder(foldername) def list_folders(self): + """ return list of all existing folder names""" assert not self._idling folders = [] for meta, sep, foldername in self.conn.list_folders(): folders.append(foldername) return folders + def delete(self, range, expunge=True): + """ delete a range of messages (imap-syntax). + If expunge is true, perform the expunge-operation + to make sure the messages are really gone and not + just flagged as deleted. + """ + self.conn.set_flags(range, [DELETED]) + if expunge: + self.conn.expunge() + def get_all_messages(self): assert not self._idling - return self.conn.fetch("1:*", [FLAGS]) + return self.conn.fetch(ALL, [FLAGS]) def get_unread_messages(self): assert not self._idling - res = self.conn.fetch("1:*", [FLAGS]) + res = self.conn.fetch(ALL, [FLAGS]) return [uid for uid in res if SEEN not in res[uid][FLAGS]] @@ -103,8 +148,6 @@ class ImapConn: kwargs["file"] = stream print(*args, **kwargs) - acinfo = self.account.logid + "-" + self.account.get_config("addr") - empty_folders = [] for imapfolder in self.list_folders(): self.select_folder(imapfolder) @@ -114,12 +157,13 @@ class ImapConn: continue log("---------", imapfolder, len(messages), "messages ---------") - # request message content without auto-marking it as seen + # get message content without auto-marking it as seen + # fetching 'RFC822' would mark it as seen. requested = [b'BODY.PEEK[HEADER]', FLAGS] for uid, data in self.conn.fetch(messages, requested).items(): body_bytes = data[b'BODY[HEADER]'] flags = data[FLAGS] - path = pathlib.Path(str(dir)).joinpath("IMAP", acinfo, imapfolder) + path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder) path.mkdir(parents=True, exist_ok=True) fn = path.joinpath(str(uid)) fn.write_bytes(body_bytes) @@ -133,12 +177,14 @@ class ImapConn: print(stream.getvalue(), file=logfile) def idle(self): + """ switch this connection to idle mode. non-blocking. """ assert not self._idling res = self.conn.idle() self._idling = True return res def idle_check(self, terminate=False): + """ (blocking) wait for next idle message from server. """ assert self._idling self.account.log("imap-direct: calling idle_check") res = self.conn.idle_check() @@ -147,6 +193,7 @@ class ImapConn: return res def idle_done(self): + """ send idle-done to server if we are currently in idle mode. """ if self._idling: res = self.conn.idle_done() self._idling = False diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index b20d56027..9fbf17346 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -43,7 +43,7 @@ class PerAccount: @account_hookspec def ac_configure_completed(self, success): - """ Called when a configure process completed. """ + """ Called after a configure process completed. """ @account_hookspec def ac_incoming_message(self, message): @@ -88,6 +88,14 @@ class Global: def dc_account_init(self, account): """ called when `Account::__init__()` function starts executing. """ + @global_hookspec + def dc_account_extra_configure(self, account): + """ Called when account configuration successfully finished. + + This hook can be used to perform extra work before + ac_configure_completed is called. + """ + @global_hookspec def dc_account_after_shutdown(self, account): """ Called after the account has been shutdown. """ diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index e0ec44cee..1d1c804e3 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -17,7 +17,7 @@ from . import Account, const from .capi import lib from .events import FFIEventLogger, FFIEventTracker from _pytest._code import Source -from deltachat.direct_imap import ImapConn +from deltachat import direct_imap import deltachat @@ -215,6 +215,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): self._generated_keys = ["alice", "bob", "charlie", "dom", "elena", "fiona"] self.set_logging_default(False) + deltachat.register_global_plugin(direct_imap) def finalize(self): while self._finalizers: @@ -225,13 +226,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): acc = self._accounts.pop() acc.shutdown() acc.disable_logging() - - def new_imap_conn(self, account, config_folder=None): - imap_conn = ImapConn(account) - self._finalizers.append(imap_conn.shutdown) - if config_folder is not None: - imap_conn.select_config_folder(config_folder) - return imap_conn + deltachat.unregister_global_plugin(direct_imap) def make_account(self, path, logid, quiet=False): ac = Account(path, logging=self._logging, logid=logid) @@ -383,10 +378,14 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): def dump_imap_summary(self, logfile): for ac in self._accounts: - imap = self.new_imap_conn(ac) - imap.dump_account_info(logfile=logfile) - imap.dump_imap_structures(tmpdir, logfile=logfile) - imap.shutdown() + imap = getattr(ac, "direct_imap", None) + if imap is not None: + try: + imap.idle_done() + except Exception: + pass + imap.dump_account_info(logfile=logfile) + imap.dump_imap_structures(tmpdir, logfile=logfile) def get_chat(self, ac1, ac2, both=True): chat12, chat21 = self.get_chats(ac1, ac2, both=both) diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index bc8122b25..25f3c0da7 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -2,7 +2,7 @@ from queue import Queue from threading import Event -from .hookspec import account_hookimpl +from .hookspec import account_hookimpl, Global class ImexFailed(RuntimeError): @@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError): class ConfigureTracker: ConfigureFailed = ConfigureFailed - def __init__(self): + def __init__(self, account): + self.account = account self._configure_events = Queue() self._smtp_finished = Event() self._imap_finished = Event() self._ffi_events = [] self._progress = Queue() + self._gm = Global._get_plugin_manager() @account_hookimpl def ac_process_ffi_event(self, ffi_event): @@ -59,6 +61,8 @@ class ConfigureTracker: @account_hookimpl def ac_configure_completed(self, success): + if success: + self._gm.hook.dc_account_extra_configure(account=self.account) self._configure_events.put(success) def wait_smtp_connected(self): diff --git a/python/tests/test_direct_imap.py b/python/tests/test_direct_imap.py index 6a80b8f05..e301847e3 100644 --- a/python/tests/test_direct_imap.py +++ b/python/tests/test_direct_imap.py @@ -7,9 +7,9 @@ def test_basic_imap_api(acfactory, tmpdir): ac1, ac2 = acfactory.get_two_online_accounts() chat12 = acfactory.get_chat(ac1, ac2) - imap2 = acfactory.new_imap_conn(ac2) + imap2 = ac2.direct_imap - imap2.idle() + ac2.direct_imap.idle() chat12.send_text("hello") ac2._evtracker.wait_next_incoming_message() @@ -26,38 +26,33 @@ class TestDirectImap: def test_mark_read_on_server(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts(move=False) - imap1 = acfactory.new_imap_conn(ac1, config_folder="inbox") - assert imap1.get_unread_cnt() == 0 chat12, chat21 = acfactory.get_chats(ac1, ac2) # send a message and check IMAP read flag - imap1.idle() + ac1.direct_imap.idle() chat21.send_text("Text message") msg_in = ac1._evtracker.wait_next_incoming_message() assert list(ac1.get_fresh_messages()) - imap1.idle_check() + ac1.direct_imap.idle_check() msg_in.mark_seen() - imap1.idle_check(terminate=True) - assert imap1.get_unread_cnt() == 0 + ac1.direct_imap.idle_check(terminate=True) + assert ac1.direct_imap.get_unread_cnt() == 0 def test_mark_bcc_read_on_server(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts(move=True) - - imap1_mvbox = acfactory.new_imap_conn(ac1, config_folder="mvbox") - chat = acfactory.get_chat(ac1, ac2) ac1.set_config("bcc_self", "1") # wait for seen/read message to appear in mvbox - imap1_mvbox.idle() + ac1.direct_imap.select_config_folder("mvbox") + ac1.direct_imap.idle() chat.send_text("Text message") ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") while 1: - res = imap1_mvbox.idle_check() + res = ac1.direct_imap.idle_check() for item in res: - uid = item[0] if item[1] == FETCH: if item[2][0] == FLAGS and SEEN in item[2][1]: return