Merge branch 'master' into flub-py-glue-fixes

python/CHANGELOG

@@ -4,6 +4,12 @@
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 
+- integrate latest DCC core-rust with dc_close() fixes
+
+- provide a account.shutdown() method  and improve termination
+  logic also in tests. also fixes output-clubbering during 
+  test runs. 
+
 
 0.600.0
 ---------

python/install_py_bindings.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-cargo build -p deltachat_ffi --release
-rm -rf build/ src/deltachat/*.so
-DCC_RS_DEV=`pwd`/.. pip install -e .

python/install_python_bindings.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+"""
+ setup a python binding development in-place install with cargo debug symbols.
+"""
+
+import os
+import subprocess
+
+if __name__ == "__main__":
+    os.environ["DCC_RS_TARGET"] = target = "release"
+
+    toml = os.path.join(os.getcwd(), "..", "Cargo.toml")
+    assert os.path.exists(toml)
+    with open(toml) as f:
+        s = orig = f.read()
+    s += "\n"
+    s += "[profile.release]\n"
+    s += "debug = true\n"
+    with open(toml, "w") as f:
+        f.write(s)
+    print("temporarily modifying Cargo.toml to provide release build with debug symbols ")
+    try:
+        subprocess.check_call([
+            "cargo", "build", "-p", "deltachat_ffi", "--" + target
+        ])
+    finally:
+        with open(toml, "w") as f:
+            f.write(orig)
+        print("\nreseted Cargo.toml to previous original state")
+
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
+
+    subprocess.check_call([
+        "pip", "install", "-e", "."
+    ])

python/src/deltachat/__init__.py

@@ -43,6 +43,9 @@ def py_dc_callback(ctx, evt, data1, data2):
 
     try:
         ret = callback(ctx, evt_name, data1, data2)
+        if ret is None:
+            ret = 0
+        assert isinstance(ret, int), repr(ret)
         if event_sig_types & 4:
             return ffi.cast('uintptr_t', ret)
         elif event_sig_types & 8:
@@ -58,7 +61,10 @@ def set_context_callback(dc_context, func):
 
 
 def clear_context_callback(dc_context):
-    _DC_CALLBACK_MAP.pop(dc_context, None)
+    try:
+        _DC_CALLBACK_MAP.pop(dc_context, None)
+    except AttributeError:
+        pass
 
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):

python/src/deltachat/_build.py

@@ -25,6 +25,7 @@ def ffibuilder():
         else:
             raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
         objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')]
+        assert os.path.exists(objs[0]), objs
         incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:
         libs = ['deltachat']

python/src/deltachat/account.py

@@ -23,28 +23,36 @@ class Account(object):
     by the underlying deltachat c-library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None):
+    def __init__(self, db_path, logid=None, eventlogging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
         :param logid: an optional logging prefix that should be used with
                       the default internal logging.
+        :param eventlogging: if False no eventlogging and no context callback will be configured
         """
         self._dc_context = ffi.gc(
             lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
             _destroy_dc_context,
         )
+        if eventlogging:
+            self._evlogger = EventLogger(self._dc_context, logid)
+            deltachat.set_context_callback(self._dc_context, self._process_event)
+            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
+        else:
+            self._threads = IOThreads(self._dc_context)
+
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
-        self._evlogger = EventLogger(self._dc_context, logid)
-        deltachat.set_context_callback(self._dc_context, self._process_event)
-        self._threads = IOThreads(self._dc_context)
         self._configkeys = self.get_config("sys.config_keys").split()
         self._imex_completed = threading.Event()
 
+    def __del__(self):
+        self.shutdown()
+
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -333,12 +341,22 @@ class Account(object):
         lib.dc_stop_ongoing_process(self._dc_context)
         self._threads.stop(wait=wait)
 
+    def shutdown(self, wait=True):
+        """ stop threads and close and remove underlying dc_context and callbacks. """
+        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
+            self.stop_threads(wait=False)  # to interrupt idle and tell python threads to stop
+            lib.dc_close(self._dc_context)
+            self.stop_threads(wait=wait)  # to wait for threads
+            deltachat.clear_context_callback(self._dc_context)
+            del self._dc_context
+
     def _process_event(self, ctx, evt_name, data1, data2):
         assert ctx == self._dc_context
-        self._evlogger(evt_name, data1, data2)
-        method = getattr(self, "on_" + evt_name.lower(), None)
-        if method is not None:
-            method(data1, data2)
+        if hasattr(self, "_evlogger"):
+            self._evlogger(evt_name, data1, data2)
+            method = getattr(self, "on_" + evt_name.lower(), None)
+            if method is not None:
+                method(data1, data2)
         return 0
 
     def on_dc_event_imex_progress(self, data1, data2):
@@ -347,10 +365,11 @@ class Account(object):
 
 
 class IOThreads:
-    def __init__(self, dc_context):
+    def __init__(self, dc_context, log_event=lambda *args: None):
         self._dc_context = dc_context
         self._thread_quitflag = False
         self._name2thread = {}
+        self._log_event = log_event
 
     def is_started(self):
         return len(self._name2thread) > 0
@@ -376,15 +395,19 @@ class IOThreads:
                 thread.join()
 
     def imap_thread_run(self):
+        self._log_event("py-bindings-info", 0, "IMAP THREAD START")
         while not self._thread_quitflag:
             lib.dc_perform_imap_jobs(self._dc_context)
             lib.dc_perform_imap_fetch(self._dc_context)
             lib.dc_perform_imap_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "IMAP THREAD FINISHED")
 
     def smtp_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
         while not self._thread_quitflag:
             lib.dc_perform_smtp_jobs(self._dc_context)
             lib.dc_perform_smtp_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
 
 
 class EventLogger:
@@ -414,7 +437,7 @@ class EventLogger:
             raise ValueError("{}({!r},{!r})".format(*ev))
         return ev
 
-    def get_matching(self, event_name_regex):
+    def get_matching(self, event_name_regex, check_error=True):
         self._log("-- waiting for event with regex: {} --".format(event_name_regex))
         rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:

python/tests/conftest.py

@@ -24,12 +24,23 @@ def pytest_configure(config):
             config.option.liveconfig = cfg
 
 
+@pytest.hookimpl(trylast=True)
+def pytest_runtest_call(item):
+    # perform early finalization because we otherwise get cloberred
+    # output from concurrent threads printing between execution
+    # of the test function and the teardown phase of that test function
+    if "acfactory" in item.funcargs:
+        print("*"*30, "finalizing", "*"*30)
+        acfactory = item.funcargs["acfactory"]
+        acfactory.finalize()
+
+
 def pytest_report_header(config, startdir):
     t = tempfile.mktemp()
     try:
-        ac = Account(t)
+        ac = Account(t, eventlogging=False)
         info = ac.get_info()
-        del ac
+        ac.shutdown()
     finally:
         os.remove(t)
     summary = ['Deltachat core={} sqlite={}'.format(
@@ -64,7 +75,6 @@ def acfactory(pytestconfig, tmpdir, request):
             self.live_count = 0
             self.offline_count = 0
             self._finalizers = []
-            request.addfinalizer(self.finalize)
             self.init_time = time.time()
 
         def finalize(self):
@@ -90,6 +100,7 @@ def acfactory(pytestconfig, tmpdir, request):
             ac = Account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
             ac._evlogger.init_time = self.init_time
             ac._evlogger.set_timeout(2)
+            self._finalizers.append(ac.shutdown)
             return ac
 
         def get_configured_offline_account(self):
@@ -115,7 +126,7 @@ def acfactory(pytestconfig, tmpdir, request):
             ac._evlogger.set_timeout(30)
             ac.configure(**configdict)
             ac.start_threads()
-            self._finalizers.append(lambda: ac.stop_threads(wait=False))
+            self._finalizers.append(ac.shutdown)
             return ac
 
         def clone_online_account(self, account):
@@ -126,7 +137,7 @@ def acfactory(pytestconfig, tmpdir, request):
             ac._evlogger.set_timeout(30)
             ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
             ac.start_threads()
-            self._finalizers.append(lambda: ac.stop_threads(wait=False))
+            self._finalizers.append(ac.shutdown)
             return ac
 
     return AccountMaker()

python/tests/test_account.py

@@ -1,12 +1,18 @@
 from __future__ import print_function
 import pytest
 import os
-from deltachat import const
+from deltachat import const, Account
 from datetime import datetime, timedelta
 from conftest import wait_configuration_progress, wait_successful_IMAP_SMTP_connection
 
 
 class TestOfflineAccount:
+    def test_wrong_db(self, tmpdir):
+        p = tmpdir.join("hello.db")
+        p.write("123")
+        with pytest.raises(ValueError):
+            Account(p.strpath)
+
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
         d = ac1.get_info()

python/tests/test_increation.py

@@ -32,7 +32,11 @@ class TestInCreation:
         chat2.add_contact(c2)
         wait_msgs_changed(ac1, 0, 0)  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
+        # XXX there might be two EVENT_MSGS_CHANGED and only one of them
+        # is the one caused by forwarding
         forwarded_id = wait_msgs_changed(ac1, chat2.id)
+        if forwarded_id == 0:
+            forwarded_id = wait_msgs_changed(ac1, chat2.id)
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.get_state().is_out_preparing()
 

python/tests/test_lowlevel.py

@@ -1,6 +1,8 @@
 from __future__ import print_function
-import pytest
-from deltachat import capi, Account, const
+from deltachat import capi, const, set_context_callback, clear_context_callback
+from deltachat.capi import ffi
+from deltachat.capi import lib
+from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -8,10 +10,45 @@ def test_empty_context():
     capi.lib.dc_close(ctx)
 
 
+def test_callback_None2int():
+    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
+    set_context_callback(ctx, lambda *args: None)
+    capi.lib.dc_close(ctx)
+    clear_context_callback(ctx)
+
+
+def test_dc_close_events():
+    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
+    evlog = EventLogger(ctx)
+    evlog.set_timeout(5)
+    set_context_callback(ctx, lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2))
+    capi.lib.dc_close(ctx)
+
+    def find(info_string):
+        while 1:
+            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
+            data2 = ev[2]
+            if info_string in data2:
+                return
+            else:
+                print("skipping event", *ev)
+
+    find("disconnecting INBOX-watch")
+    find("disconnecting sentbox-thread")
+    find("disconnecting mvbox-thread")
+    find("disconnecting SMTP")
+    find("Database closed")
+
+
 def test_wrong_db(tmpdir):
-    tmpdir.join("hello.db").write("123")
-    with pytest.raises(ValueError):
-        Account(db_path=tmpdir.strpath)
+    dc_context = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    p = tmpdir.join("hello.db")
+    # write an invalid database file
+    p.write("x123" * 10)
+    assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_event_defines():

python/tox.ini

@@ -54,6 +54,7 @@ commands =
 python_files = tests/test_*.py 
 norecursedirs = .tox 
 xfail_strict=true
+timeout = 60
 
 [flake8]
 max-line-length = 120