make direct_imap a permanent feature of online accounts

python/src/deltachat/account.py

@@ -581,7 +581,7 @@ class Account(object):
             raise MissingCredentials("addr or mail_pwd not set in config")
         if hasattr(self, "_configtracker"):
             self.remove_account_plugin(self._configtracker)
-        self._configtracker = ConfigureTracker()
+        self._configtracker = ConfigureTracker(self)
         self.add_account_plugin(self._configtracker)
         lib.dc_configure(self._dc_context)
 

python/src/deltachat/direct_imap.py

@@ -1,19 +1,50 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
 import io
 import email
 import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
+import deltachat
 
 
 SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
+ALL = "1:*"
 
 
-class ImapConn:
+@deltachat.global_hookimpl
-    def __init__(self, account):
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    imap = DirectImap(account, account.logid)
+    if imap.select_config_folder("mvbox"):
+        imap.delete(ALL, expunge=True)
+    assert imap.select_config_folder("inbox")
+    imap.delete(ALL, expunge=True)
+    setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account, logid):
         self.account = account
+        self.logid = logid
         self._idling = False
         self.connect()
 
@@ -49,25 +80,39 @@ class ImapConn:
         return self.conn.select_folder(foldername)
 
     def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
         if "_" not in config_name:
             config_name = "configured_{}_folder".format(config_name)
         foldername = self.account.get_config(config_name)
-        return self.select_folder(foldername)
+        if foldername:
+            return self.select_folder(foldername)
 
     def list_folders(self):
+        """ return list of all existing folder names"""
         assert not self._idling
         folders = []
         for meta, sep, foldername in self.conn.list_folders():
             folders.append(foldername)
         return folders
 
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
     def get_all_messages(self):
         assert not self._idling
-        return self.conn.fetch("1:*", [FLAGS])
+        return self.conn.fetch(ALL, [FLAGS])
 
     def get_unread_messages(self):
         assert not self._idling
-        res = self.conn.fetch("1:*", [FLAGS])
+        res = self.conn.fetch(ALL, [FLAGS])
         return [uid for uid in res
                 if SEEN not in res[uid][FLAGS]]
 
@@ -103,8 +148,6 @@ class ImapConn:
             kwargs["file"] = stream
             print(*args, **kwargs)
 
-        acinfo = self.account.logid + "-" + self.account.get_config("addr")
-
         empty_folders = []
         for imapfolder in self.list_folders():
             self.select_folder(imapfolder)
@@ -114,12 +157,13 @@ class ImapConn:
                 continue
 
             log("---------", imapfolder, len(messages), "messages ---------")
-            # request message content without auto-marking it as seen
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
             requested = [b'BODY.PEEK[HEADER]', FLAGS]
             for uid, data in self.conn.fetch(messages, requested).items():
                 body_bytes = data[b'BODY[HEADER]']
                 flags = data[FLAGS]
-                path = pathlib.Path(str(dir)).joinpath("IMAP", acinfo, imapfolder)
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
                 fn = path.joinpath(str(uid))
                 fn.write_bytes(body_bytes)
@@ -133,12 +177,14 @@ class ImapConn:
         print(stream.getvalue(), file=logfile)
 
     def idle(self):
+        """ switch this connection to idle mode. non-blocking. """
         assert not self._idling
         res = self.conn.idle()
         self._idling = True
         return res
 
     def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
         res = self.conn.idle_check()
@@ -147,6 +193,7 @@ class ImapConn:
         return res
 
     def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
         if self._idling:
             res = self.conn.idle_done()
             self._idling = False

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -88,6 +88,14 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
+    @global_hookspec
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/testplugin.py

@@ -17,7 +17,7 @@ from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
-from deltachat.direct_imap import ImapConn
+from deltachat import direct_imap
 
 import deltachat
 
@@ -215,6 +215,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -225,13 +226,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
-
+            deltachat.unregister_global_plugin(direct_imap)
-        def new_imap_conn(self, account, config_folder=None):
-            imap_conn = ImapConn(account)
-            self._finalizers.append(imap_conn.shutdown)
-            if config_folder is not None:
-                imap_conn.select_config_folder(config_folder)
-            return imap_conn
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging, logid=logid)
@@ -383,10 +378,14 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
-                imap = self.new_imap_conn(ac)
+                imap = getattr(ac, "direct_imap", None)
-                imap.dump_account_info(logfile=logfile)
+                if imap is not None:
-                imap.dump_imap_structures(tmpdir, logfile=logfile)
+                    try:
-                imap.shutdown()
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
 
         def get_chat(self, ac1, ac2, both=True):
             chat12, chat21 = self.get_chats(ac1, ac2, both=both)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -59,6 +61,8 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
 
     def wait_smtp_connected(self):

python/tests/test_direct_imap.py

@@ -7,9 +7,9 @@ def test_basic_imap_api(acfactory, tmpdir):
     ac1, ac2 = acfactory.get_two_online_accounts()
     chat12 = acfactory.get_chat(ac1, ac2)
 
-    imap2 = acfactory.new_imap_conn(ac2)
+    imap2 = ac2.direct_imap
 
-    imap2.idle()
+    ac2.direct_imap.idle()
     chat12.send_text("hello")
     ac2._evtracker.wait_next_incoming_message()
 
@@ -26,38 +26,33 @@ class TestDirectImap:
     def test_mark_read_on_server(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=False)
 
-        imap1 = acfactory.new_imap_conn(ac1, config_folder="inbox")
-        assert imap1.get_unread_cnt() == 0
         chat12, chat21 = acfactory.get_chats(ac1, ac2)
 
         # send a message and check IMAP read flag
-        imap1.idle()
+        ac1.direct_imap.idle()
         chat21.send_text("Text message")
 
         msg_in = ac1._evtracker.wait_next_incoming_message()
         assert list(ac1.get_fresh_messages())
 
-        imap1.idle_check()
+        ac1.direct_imap.idle_check()
         msg_in.mark_seen()
-        imap1.idle_check(terminate=True)
+        ac1.direct_imap.idle_check(terminate=True)
-        assert imap1.get_unread_cnt() == 0
+        assert ac1.direct_imap.get_unread_cnt() == 0
 
     def test_mark_bcc_read_on_server(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
-
-        imap1_mvbox = acfactory.new_imap_conn(ac1, config_folder="mvbox")
-
         chat = acfactory.get_chat(ac1, ac2)
         ac1.set_config("bcc_self", "1")
         # wait for seen/read message to appear in mvbox
-        imap1_mvbox.idle()
+        ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle()
         chat.send_text("Text message")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
         while 1:
-            res = imap1_mvbox.idle_check()
+            res = ac1.direct_imap.idle_check()
             for item in res:
-                uid = item[0]
                 if item[1] == FETCH:
                     if item[2][0] == FLAGS and SEEN in item[2][1]:
                         return