Compare commits

...

13 Commits

Author SHA1 Message Date
adbenitez
ba099ce46f fix set_config() 2020-02-23 18:55:56 -05:00
holger krekel
900a17fc00 another fix, thanks @adbenitez 2020-02-22 17:11:37 +01:00
holger krekel
78f36aaa0d another bug fix 2020-02-22 17:11:37 +01:00
holger krekel
e064e02794 fix eventlogger 2020-02-22 17:11:37 +01:00
holger krekel
e22e5045f1 add missing file, some streamlining 2020-02-22 17:11:37 +01:00
holger krekel
087f35482b factor out imex tracking 2020-02-22 17:11:37 +01:00
holger krekel
23ff5fea28 move towards pluggy 2020-02-22 17:11:37 +01:00
holger krekel
34347ccaf5 strike get_infostring 2020-02-22 17:11:37 +01:00
holger krekel
e704eb6cef move eventlogging to own module, start distinguishing ll events 2020-02-22 17:11:37 +01:00
holger krekel
bf63423fec strike footer and refine index page 2020-02-22 17:11:37 +01:00
holger krekel
f6d71ed8ef strike one Account parameter, always do eventlogging 2020-02-22 17:11:37 +01:00
Alexander Krotov
3c342339a1 imap: use parse_message_id from mimeparser 2020-02-22 16:41:07 +01:00
Alexander Krotov
33463856c5 Use mailparse::msgidparse to parse Message-IDs 2020-02-22 16:41:07 +01:00
13 changed files with 215 additions and 218 deletions

View File

@@ -15,3 +15,7 @@ div.globaltoc {
img.logo {
height: 120px;
}
div.footer {
display: none;
}

View File

@@ -55,7 +55,7 @@ master_doc = 'index'
# General information about the project.
project = u'deltachat'
copyright = u'2018, holger krekel and contributors'
copyright = u'2020, holger krekel and contributors'
# The language for content autogenerated by Sphinx. Refer to documentation

View File

@@ -1,15 +1,14 @@
deltachat python bindings
=========================
The ``deltachat`` Python package provides two bindings for the core Rust-library
of the https://delta.chat messaging ecosystem:
The ``deltachat`` Python package provides two layers of bindings for the
core Rust-library of the https://delta.chat messaging ecosystem:
- :doc:`api` is a high level interface to deltachat-core which aims
to be memory safe and thoroughly tested through continous tox/pytest runs.
- :doc:`capi` is a lowlevel CFFI-binding to the previous
`deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
replicates exactly the same C-level API).
- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
<https://github.com/deltachat/deltachat-core-rust>`_.
@@ -28,7 +27,6 @@ getting started
links
changelog
api
capi
lapi
..

View File

@@ -18,7 +18,7 @@ def main():
description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
long_description=long_description,
author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
install_requires=['cffi>=1.0.0', 'six'],
install_requires=['cffi>=1.0.0', 'pluggy'],
packages=setuptools.find_packages('src'),
package_dir={'': 'src'},
cffi_modules=['src/deltachat/_build.py:ffibuilder'],

View File

@@ -4,13 +4,9 @@ from __future__ import print_function
import atexit
import threading
import os
import re
import time
from array import array
try:
from queue import Queue, Empty
except ImportError:
from Queue import Queue, Empty
from queue import Queue
import deltachat
from . import const
@@ -19,6 +15,8 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
from .chat import Chat
from .message import Message
from .contact import Contact
from .eventlogger import EventLogger
from .hookspec import get_plugin_manager, hookimpl
class Account(object):
@@ -26,14 +24,13 @@ class Account(object):
by the underlying deltachat core library. All public Account methods are
meant to be memory-safe and return memory-safe objects.
"""
def __init__(self, db_path, logid=None, eventlogging=True, os_name=None, debug=True):
def __init__(self, db_path, logid=None, os_name=None, debug=True):
""" initialize account object.
:param db_path: a path to the account database. The database
will be created if it doesn't exist.
:param logid: an optional logging prefix that should be used with
the default internal logging.
:param eventlogging: if False no eventlogging and no context callback will be configured
:param os_name: this will be put to the X-Mailer header in outgoing messages
:param debug: turn on debug logging for events.
"""
@@ -41,19 +38,26 @@ class Account(object):
lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
_destroy_dc_context,
)
if eventlogging:
self._evlogger = EventLogger(self._dc_context, logid, debug)
deltachat.set_context_callback(self._dc_context, self._process_event)
self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
else:
self._threads = IOThreads(self._dc_context)
self._evlogger = EventLogger(self, logid, debug)
self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
# register event call back and initialize plugin system
def _ll_event(ctx, evt_name, data1, data2):
assert ctx == self._dc_context
self.pluggy.hook.process_low_level_event(
account=self, event_name=evt_name, data1=data1, data2=data2
)
self.pluggy = get_plugin_manager()
self.pluggy.register(self._evlogger)
deltachat.set_context_callback(self._dc_context, _ll_event)
# open database
if hasattr(db_path, "encode"):
db_path = db_path.encode("utf8")
if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
raise ValueError("Could not dc_open: {}".format(db_path))
self._configkeys = self.get_config("sys.config_keys").split()
self._imex_events = Queue()
atexit.register(self.shutdown)
# def __del__(self):
@@ -96,7 +100,7 @@ class Account(object):
"""
self._check_config_key(name)
name = name.encode("utf8")
if name == b"addr" and self.is_configured():
if name == "addr" and self.is_configured():
raise ValueError("can not change 'addr' after account is configured.")
if value is not None:
value = value.encode("utf8")
@@ -177,11 +181,6 @@ class Account(object):
raise ValueError("no flags set")
lib.dc_empty_server(self._dc_context, flags)
def get_infostring(self):
""" return info of the configured account. """
self.check_is_configured()
return from_dc_charpointer(lib.dc_get_info(self._dc_context))
def get_latest_backupfile(self, backupdir):
""" return the latest backup file in a given directory.
"""
@@ -382,27 +381,12 @@ class Account(object):
raise RuntimeError("found more than one new file")
return export_files[0]
def _imex_events_clear(self):
try:
while True:
self._imex_events.get_nowait()
except Empty:
pass
def _export(self, path, imex_cmd):
self._imex_events_clear()
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
if not self._threads.is_started():
lib.dc_perform_imap_jobs(self._dc_context)
files_written = []
while True:
ev = self._imex_events.get()
if isinstance(ev, str):
files_written.append(ev)
elif isinstance(ev, bool):
if not ev:
raise ValueError("export failed, exp-files: {}".format(files_written))
return files_written
with ImexTracker(self) as imex_tracker:
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
if not self._threads.is_started():
lib.dc_perform_imap_jobs(self._dc_context)
return imex_tracker.wait_finish()
def import_self_keys(self, path):
""" Import private keys found in the `path` directory.
@@ -420,12 +404,11 @@ class Account(object):
self._import(path, imex_cmd=12)
def _import(self, path, imex_cmd):
self._imex_events_clear()
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
if not self._threads.is_started():
lib.dc_perform_imap_jobs(self._dc_context)
if not self._imex_events.get():
raise ValueError("import from path '{}' failed".format(path))
with ImexTracker(self) as imex_tracker:
lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
if not self._threads.is_started():
lib.dc_perform_imap_jobs(self._dc_context)
imex_tracker.wait_finish()
def initiate_key_transfer(self):
"""return setup code after a Autocrypt setup message
@@ -528,24 +511,7 @@ class Account(object):
deltachat.clear_context_callback(self._dc_context)
del self._dc_context
atexit.unregister(self.shutdown)
def _process_event(self, ctx, evt_name, data1, data2):
assert ctx == self._dc_context
if hasattr(self, "_evlogger"):
self._evlogger(evt_name, data1, data2)
method = getattr(self, "on_" + evt_name.lower(), None)
if method is not None:
method(data1, data2)
return 0
def on_dc_event_imex_progress(self, data1, data2):
if data1 == 1000:
self._imex_events.put(True)
elif data1 == 0:
self._imex_events.put(False)
def on_dc_event_imex_file_written(self, data1, data2):
self._imex_events.put(data1)
self.pluggy.unregister(self._evlogger)
def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
"""set a new location. It effects all chats where we currently
@@ -562,6 +528,41 @@ class Account(object):
raise ValueError("no chat is streaming locations")
class ImexTracker:
def __init__(self, account):
self._imex_events = Queue()
self.account = account
def __enter__(self):
self.account.pluggy.register(self)
return self
def __exit__(self, *args):
self.account.pluggy.unregister(self)
@hookimpl
def process_low_level_event(self, account, event_name, data1, data2):
# there could be multiple accounts instantiated
if self.account is not account:
return
if event_name == "DC_EVENT_IMEX_PROGRESS":
self._imex_events.put(data1)
elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN":
self._imex_events.put(data1)
def wait_finish(self, progress_timeout=60):
""" Return list of written files, raise ValueError if ExportFailed. """
files_written = []
while True:
ev = self._imex_events.get(timeout=progress_timeout)
if isinstance(ev, str):
files_written.append(ev)
elif ev == 0:
raise ValueError("export failed, exp-files: {}".format(files_written))
elif ev == 1000:
return files_written
class IOThreads:
def __init__(self, dc_context, log_event=lambda *args: None):
self._dc_context = dc_context
@@ -642,80 +643,6 @@ class IOThreads:
self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
class EventLogger:
_loglock = threading.RLock()
def __init__(self, dc_context, logid=None, debug=True):
self._dc_context = dc_context
self._event_queue = Queue()
self._debug = debug
if logid is None:
logid = str(self._dc_context).strip(">").split()[-1]
self.logid = logid
self._timeout = None
self.init_time = time.time()
def __call__(self, evt_name, data1, data2):
self._log_event(evt_name, data1, data2)
self._event_queue.put((evt_name, data1, data2))
def set_timeout(self, timeout):
self._timeout = timeout
def consume_events(self, check_error=True):
while not self._event_queue.empty():
self.get()
def get(self, timeout=None, check_error=True):
timeout = timeout or self._timeout
ev = self._event_queue.get(timeout=timeout)
if check_error and ev[0] == "DC_EVENT_ERROR":
raise ValueError("{}({!r},{!r})".format(*ev))
return ev
def ensure_event_not_queued(self, event_name_regex):
__tracebackhide__ = True
rex = re.compile("(?:{}).*".format(event_name_regex))
while 1:
try:
ev = self._event_queue.get(False)
except Empty:
break
else:
assert not rex.match(ev[0]), "event found {}".format(ev)
def get_matching(self, event_name_regex, check_error=True, timeout=None):
self._log("-- waiting for event with regex: {} --".format(event_name_regex))
rex = re.compile("(?:{}).*".format(event_name_regex))
while 1:
ev = self.get(timeout=timeout, check_error=check_error)
if rex.match(ev[0]):
return ev
def get_info_matching(self, regex):
rex = re.compile("(?:{}).*".format(regex))
while 1:
ev = self.get_matching("DC_EVENT_INFO")
if rex.match(ev[2]):
return ev
def _log_event(self, evt_name, data1, data2):
# don't show events that are anyway empty impls now
if evt_name == "DC_EVENT_GET_STRING":
return
if self._debug:
evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
self._log(evpart)
def _log(self, msg):
t = threading.currentThread()
tname = getattr(t, "name", t)
if tname == "MainThread":
tname = "MAIN"
with self._loglock:
print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
# destructor for dc_context
dc_context_unref(dc_context)

View File

@@ -0,0 +1,81 @@
import threading
import re
import time
from queue import Queue, Empty
from .hookspec import hookimpl
class EventLogger:
_loglock = threading.RLock()
def __init__(self, account, logid=None, debug=True):
self.account = account
self._event_queue = Queue()
self._debug = debug
if logid is None:
logid = str(self.account._dc_context).strip(">").split()[-1]
self.logid = logid
self._timeout = None
self.init_time = time.time()
@hookimpl
def process_low_level_event(self, account, event_name, data1, data2):
if self.account == account:
self._log_event(event_name, data1, data2)
self._event_queue.put((event_name, data1, data2))
def set_timeout(self, timeout):
self._timeout = timeout
def consume_events(self, check_error=True):
while not self._event_queue.empty():
self.get(check_error=check_error)
def get(self, timeout=None, check_error=True):
timeout = timeout or self._timeout
ev = self._event_queue.get(timeout=timeout)
if check_error and ev[0] == "DC_EVENT_ERROR":
raise ValueError("{}({!r},{!r})".format(*ev))
return ev
def ensure_event_not_queued(self, event_name_regex):
__tracebackhide__ = True
rex = re.compile("(?:{}).*".format(event_name_regex))
while 1:
try:
ev = self._event_queue.get(False)
except Empty:
break
else:
assert not rex.match(ev[0]), "event found {}".format(ev)
def get_matching(self, event_name_regex, check_error=True, timeout=None):
self._log("-- waiting for event with regex: {} --".format(event_name_regex))
rex = re.compile("(?:{}).*".format(event_name_regex))
while 1:
ev = self.get(timeout=timeout, check_error=check_error)
if rex.match(ev[0]):
return ev
def get_info_matching(self, regex):
rex = re.compile("(?:{}).*".format(regex))
while 1:
ev = self.get_matching("DC_EVENT_INFO")
if rex.match(ev[2]):
return ev
def _log_event(self, evt_name, data1, data2):
# don't show events that are anyway empty impls now
if evt_name == "DC_EVENT_GET_STRING":
return
if self._debug:
evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
self._log(evpart)
def _log(self, msg):
t = threading.currentThread()
tname = getattr(t, "name", t)
if tname == "MainThread":
tname = "MAIN"
with self._loglock:
print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))

View File

@@ -0,0 +1,25 @@
""" Hooks for python bindings """
import pluggy
name = "deltachat"
hookspec = pluggy.HookspecMarker(name)
hookimpl = pluggy.HookimplMarker(name)
_plugin_manager = None
def get_plugin_manager():
global _plugin_manager
if _plugin_manager is None:
_plugin_manager = pluggy.PluginManager(name)
_plugin_manager.add_hookspecs(DeltaChatHookSpecs)
return _plugin_manager
class DeltaChatHookSpecs:
""" Plugin Hook specifications for Python bindings to Delta Chat CFFI. """
@hookspec
def process_low_level_event(self, account, event_name, data1, data2):
""" process a CFFI low level events for a given account. """

View File

@@ -1,5 +1,6 @@
from __future__ import print_function
import os
import sys
import py
import pytest
import requests
@@ -7,6 +8,7 @@ import time
from deltachat import Account
from deltachat import const
from deltachat.capi import lib
from _pytest.monkeypatch import MonkeyPatch
import tempfile
@@ -43,11 +45,14 @@ def pytest_report_header(config, startdir):
summary = []
t = tempfile.mktemp()
m = MonkeyPatch()
try:
ac = Account(t, eventlogging=False)
m.setattr(sys.stdout, "write", lambda x: len(x))
ac = Account(t)
info = ac.get_info()
ac.shutdown()
finally:
m.undo()
os.remove(t)
summary.extend(['Deltachat core={} sqlite={}'.format(
info['deltachat_core_version'],

View File

@@ -65,11 +65,6 @@ class TestOfflineAccountBasic:
with pytest.raises(ValueError):
ac1.get_self_contact()
def test_get_info(self, acfactory):
ac1 = acfactory.get_configured_offline_account()
out = ac1.get_infostring()
assert "number_of_chats=0" in out
def test_selfcontact_configured(self, acfactory):
ac1 = acfactory.get_configured_offline_account()
me = ac1.get_self_contact()

View File

@@ -2,7 +2,6 @@ from __future__ import print_function
from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
from deltachat.capi import ffi
from deltachat.capi import lib
from deltachat.account import EventLogger
def test_empty_context():
@@ -18,21 +17,13 @@ def test_callback_None2int():
def test_dc_close_events(tmpdir):
ctx = ffi.gc(
capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
lib.dc_context_unref,
)
evlog = EventLogger(ctx)
evlog.set_timeout(5)
set_context_callback(
ctx,
lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
)
from deltachat.account import Account
p = tmpdir.join("hello.db")
lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
capi.lib.dc_close(ctx)
ac1 = Account(p.strpath)
ac1.shutdown()
def find(info_string):
evlog = ac1._evlogger
while 1:
ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
data2 = ev[2]

View File

@@ -1464,7 +1464,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
return false;
}
if let Ok(ids) = mailparse::addrparse(mid_list) {
if let Ok(ids) = mailparse::msgidparse(mid_list) {
for id in ids.iter() {
if is_known_rfc724_mid(context, id) {
return true;
@@ -1476,8 +1476,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
}
/// Check if a message is a reply to a known message (messenger or non-messenger).
fn is_known_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> bool {
let addr = extract_single_from_addr(rfc724_mid);
fn is_known_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
context
.sql
.exists(
@@ -1485,7 +1484,7 @@ fn is_known_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> b
LEFT JOIN chats c ON m.chat_id=c.id \
WHERE m.rfc724_mid=? \
AND m.chat_id>9 AND c.blocked=0;",
params![addr],
params![rfc724_mid],
)
.unwrap_or_default()
}
@@ -1512,7 +1511,7 @@ fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) -
}
pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
if let Ok(ids) = mailparse::addrparse(mid_list) {
if let Ok(ids) = mailparse::msgidparse(mid_list) {
for id in ids.iter() {
if is_msgrmsg_rfc724_mid(context, id) {
return true;
@@ -1522,21 +1521,13 @@ pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -
false
}
fn extract_single_from_addr(addr: &mailparse::MailAddr) -> &String {
match addr {
mailparse::MailAddr::Group(infos) => &infos.addrs[0].addr,
mailparse::MailAddr::Single(info) => &info.addr,
}
}
/// Check if a message is a reply to any messenger message.
fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> bool {
let addr = extract_single_from_addr(rfc724_mid);
fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
context
.sql
.exists(
"SELECT id FROM msgs WHERE rfc724_mid=? AND msgrmsg!=0 AND chat_id>9;",
params![addr],
params![rfc724_mid],
)
.unwrap_or_default()
}

View File

@@ -1306,17 +1306,6 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
}
}
fn parse_message_id(value: &str) -> crate::error::Result<String> {
let addrs = mailparse::addrparse(value)
.map_err(|err| format_err!("failed to parse message id {:?}", err))?;
if let Some(info) = addrs.extract_single_info() {
return Ok(info.addr);
}
bail!("could not parse message_id: {}", value);
}
fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>> {
let header_bytes = match prefetch_msg.header() {
Some(header_bytes) => header_bytes,
@@ -1328,7 +1317,7 @@ fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>>
fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result<String> {
if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId)? {
Ok(parse_message_id(&message_id)?)
Ok(crate::mimeparser::parse_message_id(&message_id)?)
} else {
Err(Error::Other("prefetch: No message ID found".to_string()))
}
@@ -1384,20 +1373,3 @@ fn prefetch_should_download(
let show = show && !blocked_contact;
Ok(show)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_message_id() {
assert_eq!(
parse_message_id("Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
);
assert_eq!(
parse_message_id("<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
);
}
}

View File

@@ -741,7 +741,7 @@ impl MimeMessage {
pub fn get_rfc724_mid(&self) -> Option<String> {
self.get(HeaderDef::MessageId)
.and_then(|msgid| parse_message_id(msgid))
.and_then(|msgid| parse_message_id(msgid).ok())
}
fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
@@ -779,14 +779,16 @@ impl MimeMessage {
.get_header_value(HeaderDef::OriginalMessageId)
.ok()
.flatten()
.and_then(|v| parse_message_id(&v))
.and_then(|v| parse_message_id(&v).ok())
{
let additional_message_ids = report_fields
.get_header_value(HeaderDef::AdditionalMessageIds)
.ok()
.flatten()
.map_or_else(Vec::new, |v| {
v.split(' ').filter_map(parse_message_id).collect()
v.split(' ')
.filter_map(|s| parse_message_id(s).ok())
.collect()
});
return Ok(Some(Report {
@@ -909,14 +911,20 @@ pub(crate) struct Report {
additional_message_ids: Vec<String>,
}
fn parse_message_id(field: &str) -> Option<String> {
if let Ok(addrs) = mailparse::addrparse(field) {
// Assume the message id is a single id in the form of <id>
if let mailparse::MailAddr::Single(mailparse::SingleInfo { ref addr, .. }) = addrs[0] {
return Some(addr.clone());
pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
let ids = mailparse::msgidparse(value)
.map_err(|err| format_err!("failed to parse message id {:?}", err))?;
if ids.len() == 1 {
let id = &ids[0];
if id.starts_with('<') && id.ends_with('>') {
Ok(id.chars().skip(1).take(id.len() - 2).collect())
} else {
bail!("message-ID {} is not enclosed in < and >", value);
}
} else {
bail!("could not parse message_id: {}", value);
}
None
}
fn is_known(key: &str) -> bool {