From 612a9d012caae058f4f2395f6ac5fda2a2da7183 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 8 Jun 2020 00:02:59 +0200 Subject: [PATCH] snap using imapclient --- python/src/deltachat/direct_imap.py | 163 ++++++++++++---------------- python/src/deltachat/testplugin.py | 26 +++-- python/tests/test_account.py | 137 ++++------------------- python/tests/test_direct_imap.py | 94 ++++++++++++++++ 4 files changed, 202 insertions(+), 218 deletions(-) create mode 100644 python/tests/test_direct_imap.py diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index f396bb8f8..01e8e0a3d 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -1,51 +1,47 @@ -import imaplib +import io +import email +import ssl import pathlib - - -def db_folder_attr(name): - def fget(s): - return s.db_folder.get(name, 1) - - def fset(s, val): - s.db_folder[name] = val - return property(fget, fset, None, None) +from imapclient import IMAPClient +from imapclient.exceptions import IMAPClientError class ImapConn: def __init__(self, account): self.account = account - host = account.get_config("configured_mail_server") - user = account.get_config("addr") - pw = account.get_config("mail_pw") - self.connection = imaplib.IMAP4_SSL(host) - self.connection.login(user, pw) + self.connect() + + def connect(self): + ssl_context = ssl.create_default_context() + + # don't check if certificate hostname doesn't match target hostname + ssl_context.check_hostname = False + + # don't check if the certificate is trusted by a certificate authority + ssl_context.verify_mode = ssl.CERT_NONE + + host = self.account.get_config("configured_mail_server") + user = self.account.get_config("addr") + pw = self.account.get_config("mail_pw") + self.conn = IMAPClient(host, ssl_context=ssl_context) + self.conn.login(user, pw) + self._original_msg_count = {} self.select_folder("INBOX") def shutdown(self): try: - self.connection.close() - except Exception: - pass - try: - self.connection.logout() - except Exception: + self.conn.logout() + except (OSError, IMAPClientError): print("Could not logout direct_imap conn") def select_folder(self, foldername): - status, messages = self.connection.select(foldername) - if status != "OK": - raise ConnectionError("Could not select {}: status={} message={}".format( - foldername, status, messages)) + res = self.conn.select_folder(foldername) self.foldername = foldername - try: - msg_count = int(messages[0]) - except IndexError: - msg_count = 0 - + msg_count = res[b'UIDNEXT'] - 1 # memorize initial message count on first select self._original_msg_count.setdefault(foldername, msg_count) - return messages + return res def select_config_folder(self, config_name): if "_" not in config_name: @@ -54,66 +50,33 @@ class ImapConn: return self.select_folder(foldername) def list_folders(self): - res = self.connection.list() - # XXX this parsing is hairy, maybe use imapclient library - # instead of imaplib? - if res[0] != "OK": - raise ConnectionError(str(res)) - folders = [] - for entry in res[1]: - entry = entry.decode() - i = entry.find('"') - assert entry[i + 2] == '"' - folder_name = entry[i + 3:].strip() - folders.append(folder_name) + for meta, sep, foldername in self.conn.list_folders(): + folders.append(foldername) return folders + def get_unread_messages(self): + return self.conn.search("UNSEEN") + def mark_all_read(self): - # result, data = self.connection.uid('search', None, "(UNSEEN)") - result, data = self.connection.search(None, 'UnSeen') - try: - mails_uid = data[0].split() - print("New mails") - - # self.connection.store(data[0].replace(' ',','),'+FLAGS','\Seen') - for e_id in mails_uid: - self.connection.store(e_id, '+FLAGS', '\\Seen') - print("marked:", e_id) - - return True - except IndexError: - print("No unread") - return False + messages = self.get_unread_messages() + if messages: + res = self.conn.set_flags(messages, ['\\SEEN']) + print("marked seen:", messages, res) def get_unread_cnt(self): - # result, data = self.connection.uid('search', None, "(UNSEEN)") - result, data = self.connection.search(None, 'UnSeen') - try: - mails_uid = data[0].split() - - return len(mails_uid) - except IndexError: - return 0 + return len(self.get_unread_messages()) def get_new_email_cnt(self): - messages = self.select_folder(self.foldername) - try: - return int(messages[0]) - self._original_msg_count[self.foldername] - except IndexError: - return int(messages[0]) - - def dump_imap_structures(self, dir, file): - ac = self.account - acinfo = ac.logid + "-" + ac.get_config("addr") + return self.get_unread_cnt() - self._original_msg_count[self.foldername] + def dump_account_info(self, logfile): def log(*args, **kwargs): - kwargs["file"] = file + kwargs["file"] = logfile print(*args, **kwargs) - log("================= ACCOUNT", acinfo, "=================") cursor = 0 - for name, val in ac.get_info().items(): + for name, val in self.account.get_info().items(): entry = "{}={}".format(name.upper(), val) if cursor + len(entry) > 80: log("") @@ -122,22 +85,36 @@ class ImapConn: cursor += len(entry) + 1 log("") + def dump_imap_structures(self, dir, logfile): + stream = io.StringIO() + + def log(*args, **kwargs): + kwargs["file"] = stream + print(*args, **kwargs) + + acinfo = self.account.logid + "-" + self.account.get_config("addr") + + empty_folders = [] for imapfolder in self.list_folders(): self.select_folder(imapfolder) - c = self.connection - typ, data = c.search(None, 'ALL') - c._get_tagged_response - log("-----------------", imapfolder, "-----------------") - for num in data[0].split(): - typ, data = c.fetch(num, '(RFC822)') - body = data[0][1] + messages = self.conn.search('ALL') + if not messages: + empty_folders.append(imapfolder) + continue - typ, data = c.fetch(num, '(UID FLAGS)') - info = data[0].decode() - - path = pathlib.Path(dir.strpath).joinpath("IMAP-MESSAGES", acinfo, imapfolder) + log("---------", imapfolder, len(messages), "messages ---------") + for uid, data in self.conn.fetch(messages, [b'RFC822', b'FLAGS']).items(): + body_bytes = data[b'RFC822'] + flags = data[b'FLAGS'] + path = pathlib.Path(str(dir)).joinpath("IMAP", acinfo, imapfolder) path.mkdir(parents=True, exist_ok=True) - num = info.split()[0] - fn = path.joinpath(num) - fn.write_bytes(body) - log("Message", info, "saved as", fn) + fn = path.joinpath(str(uid)) + fn.write_bytes(body_bytes) + log("Message", uid, "saved as", fn) + email_message = email.message_from_bytes(body_bytes) + log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id")) + + if empty_folders: + log("--------- EMPTY FOLDERS:", empty_folders) + + print(stream.getvalue(), file=logfile) diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 1d4617c05..87ad508d6 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -381,19 +381,31 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): self._finalizers.append(bot.kill) return bot - def dump_imap_structures(self, file): + def dump_imap_summary(self, logfile): for ac in self._accounts: - conn = self.new_imap_conn(ac) - conn.dump_imap_structures(tmpdir, file=file) + imap = self.new_imap_conn(ac) + imap.dump_account_info(logfile=logfile) + imap.dump_imap_structures(tmpdir, logfile=logfile) + imap.shutdown() + + def get_chat(self, ac1, ac2): + chat12, chat21 = self.get_chats(ac1, ac2) + return chat12 + + def get_chats(self, ac1, ac2): + chat12 = ac1.create_chat_by_contact( + ac1.create_contact(email=ac2.get_config("addr"))) + chat21 = ac2.create_chat_by_contact( + ac2.create_contact(email=ac1.get_config("addr"))) + return chat12, chat21 am = AccountMaker() request.addfinalizer(am.finalize) yield am if hasattr(request.node, "rep_call") and request.node.rep_call.failed: - file = io.StringIO() - am.dump_imap_structures(file=file) - s = file.getvalue() - print(s) + logfile = io.StringIO() + am.dump_imap_summary(logfile=logfile) + print(logfile.getvalue()) # request.node.add_report_section("call", "imap-server-state", s) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 1f1a6b918..4a149e60e 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -11,15 +11,6 @@ from deltachat.hookspec import account_hookimpl from datetime import datetime, timedelta -def get_chat(ac1, ac2, both_created=False): - c2 = ac1.create_contact(email=ac2.get_config("addr")) - chat = ac1.create_chat_by_contact(c2) - assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL - if both_created: - ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr"))) - return chat - - @pytest.mark.parametrize("msgtext,res", [ ("Member Me (tmp1@x.org) removed by tmp2@x.org.", ("removed", "tmp1@x.org")), ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")), @@ -539,7 +530,7 @@ class TestOnlineAccount: ac1.start_io() ac2.wait_configure_finish() ac2.start_io() - chat = get_chat(ac1, ac2, both_created=True) + chat = acfactory.get_chat(ac1, ac2, both_created=True) lp.sec("ac1: send unencrypted message to ac2") chat.send_text("message1") @@ -600,7 +591,7 @@ class TestOnlineAccount: ac1_clone.wait_configure_finish() ac1_clone.start_io() - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) self_addr = ac1.get_config("addr") other_addr = ac2.get_config("addr") @@ -640,7 +631,7 @@ class TestOnlineAccount: def test_send_file_twice_unicode_filename_mangling(self, tmpdir, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) basename = "somedäüta.html.zip" p = os.path.join(tmpdir.strpath, basename) @@ -672,7 +663,7 @@ class TestOnlineAccount: def test_send_file_html_attachment(self, tmpdir, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) basename = "test.html" content = "textdata" @@ -710,7 +701,7 @@ class TestOnlineAccount: ac1.start_io() lp.sec("ac1: send message and wait for ac2 to receive it") - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) chat.send_text("message1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL @@ -723,7 +714,7 @@ class TestOnlineAccount: ac2.start_io() ac1.wait_configure_finish() ac1.start_io() - chat = get_chat(ac1, ac2) + chat, _ = get_chat(ac1, ac2) chat.send_text("message1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL @@ -738,7 +729,7 @@ class TestOnlineAccount: ac1.wait_configure_finish() ac1.start_io() - chat = get_chat(ac1, ac2) + chat, _ = get_chat(ac1, ac2) chat.send_text("message1") chat.send_text("message2") chat.send_text("message3") @@ -748,7 +739,7 @@ class TestOnlineAccount: def test_forward_messages(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() - chat = get_chat(ac1, ac2) + chat, _ = get_chat(ac1, ac2) lp.sec("ac1: send message to ac2") msg_out = chat.send_text("message2") @@ -781,7 +772,7 @@ class TestOnlineAccount: def test_forward_own_message(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() - chat = get_chat(ac1, ac2, both_created=True) + chat = acfactory.get_chat(ac1, ac2, both_created=True) lp.sec("sending message") msg_out = chat.send_text("message2") @@ -823,7 +814,7 @@ class TestOnlineAccount: ac1.set_config("displayname", "ä name") lp.sec("ac1: create chat with ac2") - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") @@ -889,7 +880,7 @@ class TestOnlineAccount: ac1, ac2 = acfactory.get_two_online_accounts(move=True) lp.sec("ac1: create chat with ac2") - chat = get_chat(ac1, ac2, both_created=True) + chat = acfactory.get_chat(ac1, ac2, both_created=True) # make sure mdns are enabled (usually enabled by default already) ac1.set_config("mdns_enabled", "1") @@ -924,7 +915,7 @@ class TestOnlineAccount: ac1, ac2 = acfactory.get_two_online_accounts() lp.sec("ac1: create chat with ac2") - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") @@ -974,7 +965,7 @@ class TestOnlineAccount: ac2.set_config("save_mime_headers", "1") lp.sec("ac1: create chat with ac2") - chat = get_chat(ac1, ac2, both_created=True) + chat = acfactory.get_chat(ac1, ac2, both_created=True) lp.sec("sending multi-line non-unicode message from ac1 to ac2") text1 = "hello\nworld" @@ -999,7 +990,7 @@ class TestOnlineAccount: ac1, ac2 = acfactory.get_two_online_accounts() lp.sec("ac1: create chat with ac2") - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") @@ -1053,7 +1044,7 @@ class TestOnlineAccount: lp.sec("configure ac2 to save mime headers, create ac1/ac2 chat") ac2.set_config("save_mime_headers", "1") - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") @@ -1069,7 +1060,7 @@ class TestOnlineAccount: def test_send_mark_seen_clean_incoming_events(self, acfactory, lp, data): ac1, ac2 = acfactory.get_two_online_accounts() - chat = get_chat(ac1, ac2, both_created=True) + chat = acfactory.get_chat(ac1, ac2, both_created=True) message_queue = queue.Queue() @@ -1098,7 +1089,7 @@ class TestOnlineAccount: def test_send_and_receive_image(self, acfactory, lp, data): ac1, ac2 = acfactory.get_two_online_accounts() - chat = get_chat(ac1, ac2) + chat = acfactory.get_chat(ac1, ac2) message_queue = queue.Queue() @@ -1305,7 +1296,7 @@ class TestOnlineAccount: ac1.set_avatar(p) lp.sec("ac1: create 1:1 chat with ac2") - chat = get_chat(ac1, ac2, both_created=True) + chat = acfactory.get_chat(ac1, ac2, both_created=True) msg = chat.send_text("hi -- do you see my brand new avatar?") assert not msg.is_encrypted() @@ -1484,8 +1475,7 @@ class TestOnlineAccount: ac1, ac2 = acfactory.get_two_online_accounts() lp.sec("ac1: create chat with ac2") - chat1 = get_chat(ac1, ac2) - chat2 = get_chat(ac2, ac1) + chat1, chat2 = get_chat(ac1, ac2) assert not chat1.is_sending_locations() with pytest.raises(ValueError): @@ -1729,92 +1719,3 @@ class TestOnlineConfigureFails: ac1._configtracker.wait_progress(0) ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "could not connect" in ev.data2.lower() - - -class TestDirectImap: - def test_basic_imap(self, acfactory): - ac1, ac2 = acfactory.get_two_online_accounts() - imap1 = acfactory.new_imap_conn(ac1) - res = imap1.list_folders() - for folder_name in res: - imap1.select_folder(folder_name) - - file = io.StringIO() - acfactory.dump_imap_structures(file=file) - out = file.getvalue().lower() - assert "arch" in out - - def test_mark_read_on_server(self, acfactory, lp): - ac1 = acfactory.get_online_configuring_account() - ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True) - - ac1.wait_configure_finish() - ac1.start_io() - ac2.wait_configure_finish() - ac2.start_io() - - imap2 = acfactory.new_imap_conn(ac2, config_folder="mvbox") - imap2.mark_all_read() - assert imap2.get_unread_cnt() == 0 - - chat = get_chat(ac1, ac2) - chat_on_ac2 = get_chat(ac2, ac1) - - chat.send_text("Text message") - - incoming_on_ac2 = ac2._evtracker.wait_next_incoming_message() - lp.sec("Incoming: "+incoming_on_ac2.text) - - assert list(ac2.get_fresh_messages()) - - for i in range(0, 20): - if imap2.get_unread_cnt() == 1: - break - time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core - assert imap2.get_unread_cnt() == 1 - - chat_on_ac2.mark_noticed() - incoming_on_ac2.mark_seen() - ac2._evtracker.wait_next_messages_changed() - - assert not list(ac2.get_fresh_messages()) - - # The new messages should be seen now. - for i in range(0, 20): - if imap2.get_unread_cnt() == 0: - break - time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core - assert imap2.get_unread_cnt() == 0 - - def test_mark_bcc_read_on_server(self, acfactory, lp): - ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True) - ac2 = acfactory.get_online_configuring_account() - - ac1.wait_configure_finish() - ac1.start_io() - ac2.wait_configure_finish() - ac2.start_io() - - imap1 = acfactory.new_imap_conn(ac1, config_folder="mvbox") - imap1.mark_all_read() - assert imap1.get_unread_cnt() == 0 - - chat = get_chat(ac1, ac2) - - ac1.set_config("bcc_self", "1") - chat.send_text("Text message") - - ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") - - for i in range(0, 20): - if imap1.get_new_email_cnt() == 1: - break - time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core - assert imap1.get_new_email_cnt() == 1 - - for i in range(0, 20): - if imap1.get_unread_cnt() == 0: - break - time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core - - assert imap1.get_unread_cnt() == 0 diff --git a/python/tests/test_direct_imap.py b/python/tests/test_direct_imap.py new file mode 100644 index 000000000..0c8ac6b75 --- /dev/null +++ b/python/tests/test_direct_imap.py @@ -0,0 +1,94 @@ +import time +import sys + + +def test_basic_message_seen(acfactory, tmpdir): + ac1, ac2 = acfactory.get_two_online_accounts() + chat12 = acfactory.get_chat(ac1, ac2) + + chat12.send_text("hello") + msg = ac2._evtracker.wait_next_incoming_message() + + # imap2.dump_imap_structures(tmpdir, logfile=sys.stdout) + + imap2 = acfactory.new_imap_conn(ac2) + assert imap2.get_unread_cnt() == 1 + imap2.mark_all_read() + assert imap2.get_unread_cnt() == 0 + imap2.shutdown() + + +class TestDirectImap: + def test_mark_read_on_server(self, acfactory, lp): + ac1 = acfactory.get_online_configuring_account() + ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True) + + ac1.wait_configure_finish() + ac1.start_io() + ac2.wait_configure_finish() + ac2.start_io() + + imap2 = acfactory.new_imap_conn(ac2, config_folder="mvbox") + imap2.mark_all_read() + assert imap2.get_unread_cnt() == 0 + + chat, chat_on_ac2 = acfactory.get_chats(ac1, ac2) + + chat.send_text("Text message") + + incoming_on_ac2 = ac2._evtracker.wait_next_incoming_message() + lp.sec("Incoming: "+incoming_on_ac2.text) + + assert list(ac2.get_fresh_messages()) + + for i in range(0, 20): + if imap2.get_unread_cnt() == 1: + break + time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core + assert imap2.get_unread_cnt() == 1 + + chat_on_ac2.mark_noticed() + incoming_on_ac2.mark_seen() + ac2._evtracker.wait_next_messages_changed() + + assert not list(ac2.get_fresh_messages()) + + # The new messages should be seen now. + for i in range(0, 20): + if imap2.get_unread_cnt() == 0: + break + time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core + assert imap2.get_unread_cnt() == 0 + + def test_mark_bcc_read_on_server(self, acfactory, lp): + ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True) + ac2 = acfactory.get_online_configuring_account() + + ac1.wait_configure_finish() + ac1.start_io() + ac2.wait_configure_finish() + ac2.start_io() + + imap1 = acfactory.new_imap_conn(ac1, config_folder="mvbox") + imap1.mark_all_read() + assert imap1.get_unread_cnt() == 0 + + chat = acfactory.get_chat(ac1, ac2) + + ac1.set_config("bcc_self", "1") + chat.send_text("Text message") + + ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + + for i in range(0, 20): + if imap1.get_new_email_cnt() == 1: + break + time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core + assert imap1.get_new_email_cnt() == 1 + + for i in range(0, 20): + if imap1.get_unread_cnt() == 0: + break + time.sleep(1) # We might need to wait because Imaplib is slower than DC-Core + + assert imap1.get_unread_cnt() == 0