fixing imap interactions

python/src/deltachat/direct_imap.py

@@ -6,9 +6,15 @@ from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
 
 
+SEEN = b'\\Seen'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+
+
 class ImapConn:
     def __init__(self, account):
         self.account = account
+        self._idling = False
         self.connect()
 
     def connect(self):
@@ -26,22 +32,21 @@ class ImapConn:
         self.conn = IMAPClient(host, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
-        self._original_msg_count = {}
         self.select_folder("INBOX")
 
     def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
         try:
             self.conn.logout()
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
     def select_folder(self, foldername):
-        res = self.conn.select_folder(foldername)
+        assert not self._idling
-        self.foldername = foldername
+        return self.conn.select_folder(foldername)
-        msg_count = res[b'UIDNEXT'] - 1
-        # memorize initial message count on first select
-        self._original_msg_count.setdefault(foldername, msg_count)
-        return res
 
     def select_config_folder(self, config_name):
         if "_" not in config_name:
@@ -50,26 +55,31 @@ class ImapConn:
         return self.select_folder(foldername)
 
     def list_folders(self):
+        assert not self._idling
         folders = []
         for meta, sep, foldername in self.conn.list_folders():
             folders.append(foldername)
         return folders
 
+    def get_all_messages(self):
+        assert not self._idling
+        return self.conn.fetch("1:*", [FLAGS])
+
     def get_unread_messages(self):
-        return self.conn.search("UNSEEN")
+        assert not self._idling
+        res = self.conn.fetch("1:*", [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.set_flags(messages, ['\\SEEN'])
+            res = self.conn.set_flags(messages, [SEEN])
             print("marked seen:", messages, res)
 
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
-    def get_new_email_cnt(self):
-        return self.get_unread_cnt() - self._original_msg_count[self.foldername]
-
     def dump_account_info(self, logfile):
         def log(*args, **kwargs):
             kwargs["file"] = logfile
@@ -86,6 +96,7 @@ class ImapConn:
         log("")
 
     def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
         stream = io.StringIO()
 
         def log(*args, **kwargs):
@@ -97,20 +108,22 @@ class ImapConn:
         empty_folders = []
         for imapfolder in self.list_folders():
             self.select_folder(imapfolder)
-            messages = self.conn.search('ALL')
+            messages = list(self.get_all_messages())
             if not messages:
                 empty_folders.append(imapfolder)
                 continue
 
             log("---------", imapfolder, len(messages), "messages ---------")
-            for uid, data in self.conn.fetch(messages, [b'RFC822', b'FLAGS']).items():
+            # request message content without auto-marking it as seen
-                body_bytes = data[b'RFC822']
+            requested = [b'BODY.PEEK[HEADER]', FLAGS]
-                flags = data[b'FLAGS']
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[HEADER]']
+                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", acinfo, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
                 fn = path.joinpath(str(uid))
                 fn.write_bytes(body_bytes)
-                log("Message", uid, "saved as", fn)
+                log("Message", uid, fn)
                 email_message = email.message_from_bytes(body_bytes)
                 log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
 
@@ -118,3 +131,23 @@ class ImapConn:
             log("--------- EMPTY FOLDERS:", empty_folders)
 
         print(stream.getvalue(), file=logfile)
+
+    def idle(self):
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check()
+        if terminate:
+            self.idle_done()
+        return res
+
+    def idle_done(self):
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/testplugin.py

@@ -318,7 +318,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
             ac1.wait_configure_finish()
             ac1.start_io()

python/tests/test_direct_imap.py

@@ -1,93 +1,63 @@
 import sys
-import time
+
+from deltachat.direct_imap import SEEN, FLAGS, FETCH
 
 
-def test_basic_message_seen(acfactory, tmpdir):
+def test_basic_imap_api(acfactory, tmpdir):
     ac1, ac2 = acfactory.get_two_online_accounts()
     chat12 = acfactory.get_chat(ac1, ac2)
 
+    imap2 = acfactory.new_imap_conn(ac2)
+
+    imap2.idle()
     chat12.send_text("hello")
     ac2._evtracker.wait_next_incoming_message()
 
-    imap2 = acfactory.new_imap_conn(ac2)
+    imap2.idle_check(terminate=True)
-    imap2.dump_imap_structures(tmpdir, logfile=sys.stdout)
     assert imap2.get_unread_cnt() == 1
     imap2.mark_all_read()
     assert imap2.get_unread_cnt() == 0
+
+    imap2.dump_imap_structures(tmpdir, logfile=sys.stdout)
     imap2.shutdown()
 
 
 class TestDirectImap:
     def test_mark_read_on_server(self, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account()
+        ac1, ac2 = acfactory.get_two_online_accounts(move=False)
-        ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True)
 
-        ac1.wait_configure_finish()
+        imap1 = acfactory.new_imap_conn(ac1, config_folder="inbox")
-        ac1.start_io()
+        assert imap1.get_unread_cnt() == 0
-        ac2.wait_configure_finish()
+        chat12, chat21 = acfactory.get_chats(ac1, ac2)
-        ac2.start_io()
 
-        imap2 = acfactory.new_imap_conn(ac2, config_folder="mvbox")
+        # send a message and check IMAP read flag
-        imap2.mark_all_read()
+        imap1.idle()
-        assert imap2.get_unread_cnt() == 0
+        chat21.send_text("Text message")
 
-        chat, chat_on_ac2 = acfactory.get_chats(ac1, ac2)
+        msg_in = ac1._evtracker.wait_next_incoming_message()
+        assert list(ac1.get_fresh_messages())
 
-        chat.send_text("Text message")
+        imap1.idle_check()
-
+        msg_in.mark_seen()
-        incoming_on_ac2 = ac2._evtracker.wait_next_incoming_message()
+        imap1.idle_check(terminate=True)
-        lp.sec("Incoming: "+incoming_on_ac2.text)
+        assert imap1.get_unread_cnt() == 0
-
-        assert list(ac2.get_fresh_messages())
-
-        for i in range(0, 20):
-            if imap2.get_unread_cnt() == 1:
-                break
-            time.sleep(1)  # We might need to wait because Imaplib is slower than DC-Core
-        assert imap2.get_unread_cnt() == 1
-
-        chat_on_ac2.mark_noticed()
-        incoming_on_ac2.mark_seen()
-        ac2._evtracker.wait_next_messages_changed()
-
-        assert not list(ac2.get_fresh_messages())
-
-        # The new messages should be seen now.
-        for i in range(0, 20):
-            if imap2.get_unread_cnt() == 0:
-                break
-            time.sleep(1)  # We might need to wait because Imaplib is slower than DC-Core
-        assert imap2.get_unread_cnt() == 0
 
     def test_mark_bcc_read_on_server(self, acfactory, lp):
-        ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True)
+        ac1, ac2 = acfactory.get_two_online_accounts(move=True)
-        ac2 = acfactory.get_online_configuring_account()
 
-        ac1.wait_configure_finish()
+        imap1_mvbox = acfactory.new_imap_conn(ac1, config_folder="mvbox")
-        ac1.start_io()
-        ac2.wait_configure_finish()
-        ac2.start_io()
-
-        imap1 = acfactory.new_imap_conn(ac1, config_folder="mvbox")
-        imap1.mark_all_read()
-        assert imap1.get_unread_cnt() == 0
 
         chat = acfactory.get_chat(ac1, ac2)
-
         ac1.set_config("bcc_self", "1")
+        # wait for seen/read message to appear in mvbox
+        imap1_mvbox.idle()
         chat.send_text("Text message")
-
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
-        for i in range(0, 20):
+        while 1:
-            if imap1.get_new_email_cnt() == 1:
+            res = imap1_mvbox.idle_check()
-                break
+            for item in res:
-            time.sleep(1)  # We might need to wait because Imaplib is slower than DC-Core
+                uid = item[0]
-        assert imap1.get_new_email_cnt() == 1
+                if item[1] == FETCH:
-
+                    if item[2][0] == FLAGS and SEEN in item[2][1]:
-        for i in range(0, 20):
+                        return
-            if imap1.get_unread_cnt() == 0:
-                break
-            time.sleep(1)  # We might need to wait because Imaplib is slower than DC-Core
-
-        assert imap1.get_unread_cnt() == 0