replaced imapclient python library with imap-tools in the tests. works with testrun.org locally

python/setup.py

@@ -11,7 +11,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
         setup_requires=[
             'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
             'pkgconfig',

python/src/deltachat/direct_imap.py

@@ -4,25 +4,23 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
-import email
 import ssl
 import pathlib
-from imapclient import IMAPClient
+from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
-from imapclient.exceptions import IMAPClientError
 import imaplib
 import deltachat
 from deltachat import const, Account
 
 
-SEEN = b'\\Seen'
+SEEN = MailMessageFlags.SEEN
-DELETED = b'\\Deleted'
+DELETED = MailMessageFlags.DELETED
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
 @deltachat.global_hookimpl
-def dc_account_extra_configure(account):
+def dc_account_extra_configure(account: Account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
@@ -36,7 +34,7 @@ def dc_account_extra_configure(account):
                     assert imap.select_folder(folder)
                     imap.delete(ALL, expunge=True)
                 else:
-                    imap.conn.delete_folder(folder)
+                    imap.conn.folder.delete(folder)
                     # We just deleted the folder, so we have to make DC forget about it, too
                     if account.get_config("configured_sentbox_folder") == folder:
                         account.set_config("configured_sentbox_folder", None)
@@ -86,37 +84,34 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
-        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
         try:
-            self.conn.idle_done()
+            self.idle_done()
-        except (OSError, IMAPClientError):
+        except (OSError, imaplib.IMAP4.abort):
             pass
         try:
             self.conn.logout()
-        except (OSError, IMAPClientError):
+        except (OSError, imaplib.IMAP4.abort):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.create_folder(foldername)
+            self.conn.folder.create(foldername)
-        except imaplib.IMAP4.error as e:
+        except errors.MailboxFolderCreateError as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername):
+    def select_folder(self, foldername: str) -> tuple:
         assert not self._idling
-        return self.conn.select_folder(foldername)
+        return self.conn.folder.set(foldername)
 
-    def select_config_folder(self, config_name):
+    def select_config_folder(self, config_name: str):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -125,50 +120,36 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self):
+    def list_folders(self) -> [str]:
         """ return list of all existing folder names"""
         assert not self._idling
-        folders = []
+        return [folder.name for folder in self.conn.folder.list()]
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
 
-    def delete(self, range, expunge=True):
+    def delete(self, uid_list: str, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.set_flags(range, [DELETED])
+        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self):
+    def get_all_messages(self) -> [MailMessage]:
         assert not self._idling
+        return [mail for mail in self.conn.fetch()]
 
-        # Flush unsolicited responses. IMAPClient has problems
+    def get_unread_messages(self) -> [str]:
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
         assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
+        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.set_flags(messages, [SEEN])
+            res = self.conn.flag(messages, SEEN, True)
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self):
+    def get_unread_cnt(self) -> int:
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -191,20 +172,20 @@ class DirectImap:
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
             requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
+            for msg in self.conn.fetch(mark_seen=False):
-                body_bytes = data[b'BODY[]']
+                body = getattr(msg.obj, "text", None)
-                if not body_bytes:
+                if not body:
-                    log("Message", uid, "has empty body")
+                    body = getattr(msg.obj, "html", None)
+                if not body:
+                    log("Message", msg.uid, "has empty body")
                     continue
 
-                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
+                fn = path.joinpath(str(msg.uid))
-                fn.write_bytes(body_bytes)
+                fn.write_bytes(body)
-                log("Message", uid, fn)
+                log("Message", msg.uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
+                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -214,59 +195,59 @@ class DirectImap:
     def idle_start(self):
         """ switch this connection to idle mode. non-blocking. """
         assert not self._idling
-        res = self.conn.idle()
+        res = self.conn.idle.start()
         self._idling = True
         return res
 
-    def idle_check(self, terminate=False):
+    def idle_check(self, terminate=False, timeout=60) -> [bytes]:
         """ (blocking) wait for next idle message from server. """
         assert self._idling
         self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check()
+        res = self.conn.idle.poll(timeout=timeout)
         if terminate:
             self.idle_done()
         self.account.log("imap-direct: idle_check returned {!r}".format(res))
         return res
 
-    def idle_wait_for_new_message(self, terminate=False):
+    def idle_wait_for_new_message(self, terminate=False, timeout=60) -> bytes:
         while 1:
-            for item in self.idle_check():
+            for item in self.idle_check(timeout=timeout):
-                if item[1] in (b'EXISTS', b'RECENT'):
+                if b'EXISTS' in item or b'RECENT' in item:
                     if terminate:
                         self.idle_done()
                     return item
 
-    def idle_wait_for_seen(self, terminate=False):
+    def idle_wait_for_seen(self, terminate=False, timeout=60) -> int:
         """ Return first message with SEEN flag
         from a running idle-stream REtiurn.
         """
         while 1:
-            for item in self.idle_check():
+            for item in self.idle_check(timeout=timeout):
-                if item[1] == FETCH:
+                if FETCH in item:
-                    if item[2][0] == FLAGS:
+                    self.account.log(str(item))
-                        if SEEN in item[2][1]:
+                    if FLAGS in item and bytes(SEEN, encoding='ascii') in item:
-                            if terminate:
+                        if terminate:
-                                self.idle_done()
+                            self.idle_done()
-                            return item[0]
+                        return int(item.split(b' ')[1])
 
     def idle_done(self):
         """ send idle-done to server if we are currently in idle mode. """
         if self._idling:
-            res = self.conn.idle_done()
+            res = self.conn.idle.stop()
             self._idling = False
             return res
 
-    def append(self, folder, msg):
+    def append(self, folder: str, msg: str):
         """Upload a message to *folder*.
         Trailing whitespace or a linebreak at the beginning will be removed automatically.
         """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
+        self.conn.append(bytes(msg, encoding='ascii'), folder)
 
-    def get_uid_by_message_id(self, message_id):
+    def get_uid_by_message_id(self, message_id) -> str:
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]

python/tests/test_account.py

@@ -11,6 +11,7 @@ from deltachat.hookspec import account_hookimpl
 from deltachat.capi import ffi, lib
 from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
+from imap_tools import AND, U
 
 
 @pytest.mark.parametrize("msgtext,res", [
@@ -1058,12 +1059,9 @@ class TestOnlineAccount:
         # Accept the contact request.
         msg.chat.accept()
         ac2.mark_seen_messages([msg])
-        ac2.direct_imap.idle_wait_for_seen(terminate=True)
+        uid = ac2.direct_imap.idle_wait_for_seen(terminate=True)
 
-        fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values())
+        assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1
-        flags = fetch[-1][b'FLAGS']
-        is_seen = b'\\Seen' in flags
-        assert is_seen
 
     def test_multidevice_sync_seen(self, acfactory, lp):
         """Test that message marked as seen on one device is marked as seen on another."""
@@ -2296,7 +2294,7 @@ class TestOnlineAccount:
         ac1.direct_imap.delete("1:*", expunge=False)
         ac1.start_io()
 
-        for ev in ac1._evtracker.iter_events():
+        for ev in ac1._evtracker.iter_events(timeout=60):
             if ev.name == "DC_EVENT_MSGS_CHANGED":
                 pytest.fail("A deleted message was shown to the user")
 
@@ -2552,7 +2550,9 @@ class TestOnlineAccount:
 
         lp.sec("imap2: test that only one message is left")
         imap2 = ac2.direct_imap
-
+        imap2.idle_start()
+        imap2.idle_wait_for_new_message(timeout=600)
+        imap2.idle_done()
         assert len(imap2.get_all_messages()) == 1
 
     def test_configure_error_msgs(self, acfactory):
@@ -2858,15 +2858,15 @@ class TestOnlineAccount:
         ac2 = acfactory.get_online_configuring_account()
         acfactory.wait_configure(ac1)
 
-        ac1.direct_imap.conn.delete_folder("DeltaChat")
+        ac1.direct_imap.conn.folder.delete("DeltaChat")
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0
+        assert "DeltaChat" not in ac1.direct_imap.list_folders()
         acfactory.wait_configure_and_start_io()
 
         ac2.create_chat(ac1).send_text("hello")
         msg = ac1._evtracker.wait_next_incoming_message()
         assert msg.text == "hello"
 
-        assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1
+        assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
 class TestGroupStressTests: