Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-05-08 09:26:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3260 from deltachat/imap-tools

Browse Source 
replaced imapclient python library with imap-tools
This commit is contained in:
missytake
2022-04-29 17:58:50 +02:00
committed by GitHub
parent b7ac81701a b97b374487
commit d4e0009b89
4 changed files with 74 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/mypy.ini
View File

@@ -17,3 +17,7 @@ ignore_missing_imports = True
[mypy-_pytest.*] [mypy-_pytest.*]
ignore_missing_imports = True ignore_missing_imports = True
[mypy-imap_tools.*]
ignore_missing_imports = True

2
python/setup.py
View File

@@ -11,7 +11,7 @@ def main():
description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat', description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
long_description=long_description, long_description=long_description,
author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors', author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'], install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
setup_requires=[ setup_requires=[
'setuptools_scm', # required for compatibility with `python3 setup.py sdist` 'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
'pkgconfig', 'pkgconfig',

132
python/src/deltachat/direct_imap.py
View File

@@ -4,25 +4,22 @@ and for cleaning up inbox/mvbox for each test function run.
""" """
import io import io
import email
import ssl import ssl
import pathlib import pathlib
from imapclient import IMAPClient from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
from imapclient.exceptions import IMAPClientError
import imaplib import imaplib
import deltachat import deltachat
from deltachat import const, Account from deltachat import const, Account
from typing import List
SEEN = b'\\Seen'
DELETED = b'\\Deleted'
FLAGS = b'FLAGS' FLAGS = b'FLAGS'
FETCH = b'FETCH' FETCH = b'FETCH'
ALL = "1:*" ALL = "1:*"
@deltachat.global_hookimpl @deltachat.global_hookimpl
def dc_account_extra_configure(account): def dc_account_extra_configure(account: Account):
""" Reset the account (we reuse accounts across tests) """ Reset the account (we reuse accounts across tests)
and make 'account.direct_imap' available for direct IMAP ops. and make 'account.direct_imap' available for direct IMAP ops.
""" """
@@ -36,7 +33,7 @@ def dc_account_extra_configure(account):
assert imap.select_folder(folder) assert imap.select_folder(folder)
imap.delete(ALL, expunge=True) imap.delete(ALL, expunge=True)
else: else:
imap.conn.delete_folder(folder) imap.conn.folder.delete(folder)
# We just deleted the folder, so we have to make DC forget about it, too # We just deleted the folder, so we have to make DC forget about it, too
if account.get_config("configured_sentbox_folder") == folder: if account.get_config("configured_sentbox_folder") == folder:
account.set_config("configured_sentbox_folder", None) account.set_config("configured_sentbox_folder", None)
@@ -86,37 +83,34 @@ class DirectImap:
ssl_context.verify_mode = ssl.CERT_NONE ssl_context.verify_mode = ssl.CERT_NONE
if security == const.DC_SOCKET_STARTTLS: if security == const.DC_SOCKET_STARTTLS:
self.conn = IMAPClient(host, port, ssl=False) self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
self.conn.starttls(ssl_context) elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
elif security == const.DC_SOCKET_PLAIN: self.conn = MailBox(host, port, ssl_context=ssl_context)
self.conn = IMAPClient(host, port, ssl=False)
elif security == const.DC_SOCKET_SSL:
self.conn = IMAPClient(host, port, ssl_context=ssl_context)
self.conn.login(user, pw) self.conn.login(user, pw)
self.select_folder("INBOX") self.select_folder("INBOX")
def shutdown(self): def shutdown(self):
try: try:
self.conn.idle_done() self.idle_done()
except (OSError, IMAPClientError): except (OSError, imaplib.IMAP4.abort):
pass pass
try: try:
self.conn.logout() self.conn.logout()
except (OSError, IMAPClientError): except (OSError, imaplib.IMAP4.abort):
print("Could not logout direct_imap conn") print("Could not logout direct_imap conn")
def create_folder(self, foldername): def create_folder(self, foldername):
try: try:
self.conn.create_folder(foldername) self.conn.folder.create(foldername)
except imaplib.IMAP4.error as e: except errors.MailboxFolderCreateError as e:
print("Can't create", foldername, "probably it already exists:", str(e)) print("Can't create", foldername, "probably it already exists:", str(e))
def select_folder(self, foldername): def select_folder(self, foldername: str) -> tuple:
assert not self._idling assert not self._idling
return self.conn.select_folder(foldername) return self.conn.folder.set(foldername)
def select_config_folder(self, config_name): def select_config_folder(self, config_name: str):
""" Return info about selected folder if it is """ Return info about selected folder if it is
configured, otherwise None. """ configured, otherwise None. """
if "_" not in config_name: if "_" not in config_name:
@@ -125,50 +119,36 @@ class DirectImap:
if foldername: if foldername:
return self.select_folder(foldername) return self.select_folder(foldername)
def list_folders(self): def list_folders(self) -> List[str]:
""" return list of all existing folder names""" """ return list of all existing folder names"""
assert not self._idling assert not self._idling
folders = [] return [folder.name for folder in self.conn.folder.list()]
for meta, sep, foldername in self.conn.list_folders():
folders.append(foldername)
return folders
def delete(self, range, expunge=True): def delete(self, uid_list: str, expunge=True):
""" delete a range of messages (imap-syntax). """ delete a range of messages (imap-syntax).
If expunge is true, perform the expunge-operation If expunge is true, perform the expunge-operation
to make sure the messages are really gone and not to make sure the messages are really gone and not
just flagged as deleted. just flagged as deleted.
""" """
self.conn.set_flags(range, [DELETED]) self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
if expunge: if expunge:
self.conn.expunge() self.conn.expunge()
def get_all_messages(self): def get_all_messages(self) -> List[MailMessage]:
assert not self._idling assert not self._idling
return [mail for mail in self.conn.fetch()]
# Flush unsolicited responses. IMAPClient has problems def get_unread_messages(self) -> List[str]:
# dealing with them: https://github.com/mjs/imapclient/issues/334
# When this NOOP was introduced, next FETCH returned empty
# result instead of a single message, even though IMAP server
# can only return more untagged responses than required, not
# less.
self.conn.noop()
return self.conn.fetch(ALL, [FLAGS])
def get_unread_messages(self):
assert not self._idling assert not self._idling
res = self.conn.fetch(ALL, [FLAGS]) return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
return [uid for uid in res
if SEEN not in res[uid][FLAGS]]
def mark_all_read(self): def mark_all_read(self):
messages = self.get_unread_messages() messages = self.get_unread_messages()
if messages: if messages:
res = self.conn.set_flags(messages, [SEEN]) res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
print("marked seen:", messages, res) print("marked seen:", messages, res)
def get_unread_cnt(self): def get_unread_cnt(self) -> int:
return len(self.get_unread_messages()) return len(self.get_unread_messages())
def dump_imap_structures(self, dir, logfile): def dump_imap_structures(self, dir, logfile):
@@ -190,21 +170,20 @@ class DirectImap:
log("---------", imapfolder, len(messages), "messages ---------") log("---------", imapfolder, len(messages), "messages ---------")
# get message content without auto-marking it as seen # get message content without auto-marking it as seen
# fetching 'RFC822' would mark it as seen. # fetching 'RFC822' would mark it as seen.
requested = [b'BODY.PEEK[]', FLAGS] for msg in self.conn.fetch(mark_seen=False):
for uid, data in self.conn.fetch(messages, requested).items(): body = getattr(msg.obj, "text", None)
body_bytes = data[b'BODY[]'] if not body:
if not body_bytes: body = getattr(msg.obj, "html", None)
log("Message", uid, "has empty body") if not body:
log("Message", msg.uid, "has empty body")
continue continue
flags = data[FLAGS]
path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder) path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
path.mkdir(parents=True, exist_ok=True) path.mkdir(parents=True, exist_ok=True)
fn = path.joinpath(str(uid)) fn = path.joinpath(str(msg.uid))
fn.write_bytes(body_bytes) fn.write_bytes(body)
log("Message", uid, fn) log("Message", msg.uid, fn)
email_message = email.message_from_bytes(body_bytes) log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
if empty_folders: if empty_folders:
log("--------- EMPTY FOLDERS:", empty_folders) log("--------- EMPTY FOLDERS:", empty_folders)
@@ -214,59 +193,58 @@ class DirectImap:
def idle_start(self): def idle_start(self):
""" switch this connection to idle mode. non-blocking. """ """ switch this connection to idle mode. non-blocking. """
assert not self._idling assert not self._idling
res = self.conn.idle() res = self.conn.idle.start()
self._idling = True self._idling = True
return res return res
def idle_check(self, terminate=False): def idle_check(self, terminate=False, timeout=None) -> List[bytes]:
""" (blocking) wait for next idle message from server. """ """ (blocking) wait for next idle message from server. """
assert self._idling assert self._idling
self.account.log("imap-direct: calling idle_check") self.account.log("imap-direct: calling idle_check")
res = self.conn.idle_check() res = self.conn.idle.poll(timeout=timeout)
if terminate: if terminate:
self.idle_done() self.idle_done()
self.account.log("imap-direct: idle_check returned {!r}".format(res)) self.account.log("imap-direct: idle_check returned {!r}".format(res))
return res return res
def idle_wait_for_new_message(self, terminate=False): def idle_wait_for_new_message(self, terminate=False, timeout=None) -> bytes:
while 1: while 1:
for item in self.idle_check(): for item in self.idle_check(timeout=timeout):
if item[1] in (b'EXISTS', b'RECENT'): if b'EXISTS' in item or b'RECENT' in item:
if terminate: if terminate:
self.idle_done() self.idle_done()
return item return item
def idle_wait_for_seen(self, terminate=False): def idle_wait_for_seen(self, terminate=False, timeout=None) -> int:
""" Return first message with SEEN flag """ Return first message with SEEN flag from a running idle-stream.
from a running idle-stream REtiurn.
""" """
while 1: while 1:
for item in self.idle_check(): for item in self.idle_check(timeout=timeout):
if item[1] == FETCH: if FETCH in item:
if item[2][0] == FLAGS: self.account.log(str(item))
if SEEN in item[2][1]: if FLAGS in item and rb'\Seen' in item:
if terminate: if terminate:
self.idle_done() self.idle_done()
return item[0] return int(item.split(b' ')[1])
def idle_done(self): def idle_done(self):
""" send idle-done to server if we are currently in idle mode. """ """ send idle-done to server if we are currently in idle mode. """
if self._idling: if self._idling:
res = self.conn.idle_done() res = self.conn.idle.stop()
self._idling = False self._idling = False
return res return res
def append(self, folder, msg): def append(self, folder: str, msg: str):
"""Upload a message to *folder*. """Upload a message to *folder*.
Trailing whitespace or a linebreak at the beginning will be removed automatically. Trailing whitespace or a linebreak at the beginning will be removed automatically.
""" """
if msg.startswith("\n"): if msg.startswith("\n"):
msg = msg[1:] msg = msg[1:]
msg = '\n'.join([s.lstrip() for s in msg.splitlines()]) msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
self.conn.append(folder, msg) self.conn.append(bytes(msg, encoding='ascii'), folder)
def get_uid_by_message_id(self, message_id): def get_uid_by_message_id(self, message_id) -> str:
msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id]) msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
if len(msgs) == 0: if len(msgs) == 0:
raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?") raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
return msgs[0] return msgs[0]

32
python/tests/test_account.py
View File

@@ -11,6 +11,7 @@ from deltachat.hookspec import account_hookimpl
from deltachat.capi import ffi, lib from deltachat.capi import ffi, lib
from deltachat.cutil import iter_array from deltachat.cutil import iter_array
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from imap_tools import AND, U
@pytest.mark.parametrize("msgtext,res", [ @pytest.mark.parametrize("msgtext,res", [
@@ -1058,12 +1059,9 @@ class TestOnlineAccount:
# Accept the contact request. # Accept the contact request.
msg.chat.accept() msg.chat.accept()
ac2.mark_seen_messages([msg]) ac2.mark_seen_messages([msg])
ac2.direct_imap.idle_wait_for_seen(terminate=True) uid = ac2.direct_imap.idle_wait_for_seen(terminate=True)
fetch = list(ac2.direct_imap.conn.fetch("*", b'FLAGS').values()) assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1
flags = fetch[-1][b'FLAGS']
is_seen = b'\\Seen' in flags
assert is_seen
def test_multidevice_sync_seen(self, acfactory, lp): def test_multidevice_sync_seen(self, acfactory, lp):
"""Test that message marked as seen on one device is marked as seen on another.""" """Test that message marked as seen on one device is marked as seen on another."""
@@ -2550,10 +2548,9 @@ class TestOnlineAccount:
ac2._evtracker.get_info_contains("close/expunge succeeded") ac2._evtracker.get_info_contains("close/expunge succeeded")
lp.sec("imap2: test that only one message is left") lp.sec("ac2: test that only one message is left")
imap2 = ac2.direct_imap ac2.direct_imap.select_config_folder("inbox")
assert len(ac2.direct_imap.get_all_messages()) == 1
assert len(imap2.get_all_messages()) == 1
def test_configure_error_msgs(self, acfactory): def test_configure_error_msgs(self, acfactory):
ac1, configdict = acfactory.get_online_config() ac1, configdict = acfactory.get_online_config()
@@ -2772,10 +2769,7 @@ class TestOnlineAccount:
acfactory.wait_configure_and_start_io() acfactory.wait_configure_and_start_io()
assert_folders_configured(ac1) assert_folders_configured(ac1)
if mvbox_move: assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
ac1.direct_imap.select_config_folder("mvbox")
else:
ac1.direct_imap.select_folder("INBOX")
ac1.direct_imap.idle_start() ac1.direct_imap.idle_start()
lp.sec("send out message with bcc to ourselves") lp.sec("send out message with bcc to ourselves")
@@ -2784,22 +2778,24 @@ class TestOnlineAccount:
chat.send_text("message text") chat.send_text("message text")
assert_folders_configured(ac1) assert_folders_configured(ac1)
# now wait until the bcc_self message arrives lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
# Also test that bcc_self messages moved to the mvbox are marked as read.
assert ac1.direct_imap.idle_wait_for_seen() assert ac1.direct_imap.idle_wait_for_seen()
assert_folders_configured(ac1) assert_folders_configured(ac1)
lp.sec("create a cloned ac1 and fetch contact history during configure")
ac1_clone = acfactory.clone_online_account(ac1) ac1_clone = acfactory.clone_online_account(ac1)
ac1_clone.set_config("fetch_existing_msgs", "1") ac1_clone.set_config("fetch_existing_msgs", "1")
ac1_clone._configtracker.wait_finish() ac1_clone._configtracker.wait_finish()
ac1_clone.start_io() ac1_clone.start_io()
assert_folders_configured(ac1_clone) assert_folders_configured(ac1_clone)
lp.sec("check that ac2 contact was fetchted during configure")
ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED") ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
ac2_addr = ac2.get_config("addr") ac2_addr = ac2.get_config("addr")
assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts()) assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
assert_folders_configured(ac1_clone) assert_folders_configured(ac1_clone)
lp.sec("check that messages changed events arrive for the correct message")
msg = ac1_clone._evtracker.wait_next_messages_changed() msg = ac1_clone._evtracker.wait_next_messages_changed()
assert msg.text == "message text" assert msg.text == "message text"
assert_folders_configured(ac1) assert_folders_configured(ac1)
@@ -2858,15 +2854,15 @@ class TestOnlineAccount:
ac2 = acfactory.get_online_configuring_account() ac2 = acfactory.get_online_configuring_account()
acfactory.wait_configure(ac1) acfactory.wait_configure(ac1)
ac1.direct_imap.conn.delete_folder("DeltaChat") ac1.direct_imap.conn.folder.delete("DeltaChat")
assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 0 assert "DeltaChat" not in ac1.direct_imap.list_folders()
acfactory.wait_configure_and_start_io() acfactory.wait_configure_and_start_io()
ac2.create_chat(ac1).send_text("hello") ac2.create_chat(ac1).send_text("hello")
msg = ac1._evtracker.wait_next_incoming_message() msg = ac1._evtracker.wait_next_incoming_message()
assert msg.text == "hello" assert msg.text == "hello"
assert len(ac1.direct_imap.conn.list_folders(pattern="DeltaChat")) == 1 assert "DeltaChat" in ac1.direct_imap.list_folders()
class TestGroupStressTests: class TestGroupStressTests: