refactor docs and ffi/high level event handling to pass all tests again

python/doc/capi.rst

@@ -1,7 +0,0 @@
-
-C deltachat interface
-=====================
-
-See :doc:`lapi` for accessing many of the below functions
-through the ``deltachat.capi.lib`` namespace.
-

python/doc/examples.rst

@@ -4,8 +4,8 @@ examples
 ========
 
 
-Playing around on the commandline
+Sending a Chat message from the command line
-----------------------------------
+---------------------------------------------
 
 Once you have :doc:`installed deltachat bindings <install>`
 you can start playing from the python interpreter commandline.
@@ -14,10 +14,10 @@ For example you can type ``python`` and then::
     # instantiate and configure deltachat account
     import deltachat
     ac = deltachat.Account("/tmp/db")
-    ac.set_config("addr", "test2@hq5.merlinux.eu")
+    ac.set_config("addr", "address@example.org")
     ac.set_config("mail_pwd", "some password")
 
-    # start the IO threads and perform configuration
+    # start IO threads and perform configuration
     ac.start()
 
     # create a contact and send a message
@@ -25,6 +25,23 @@ For example you can type ``python`` and then::
     chat = ac.create_chat_by_contact(contact)
     chat.send_text("hi from the python interpreter command line")
 
+    # shutdown IO threads
+    ac.shutdown()
+
+
+Checkout our :doc:`api` for the various high-level things you can do
+to send/receive messages, create contacts and chats.
+
+
+Receiving a Chat message from the command line
+----------------------------------------------
+
+Instantiate an account and register a plugin to process
+incoming messages:
+
+.. include:: ../examples/echo_and_quit.py
+    :literal:
+
 Checkout our :doc:`api` for the various high-level things you can do
 to send/receive messages, create contacts and chats.
 

python/doc/index.rst

@@ -29,6 +29,7 @@ getting started
    changelog
    api
    lapi
+   plugins
 
 ..
     Indices and tables

python/examples/echo_and_quit.py

@@ -0,0 +1,37 @@
+
+# instantiate and configure deltachat account
+import deltachat
+ac = deltachat.Account("/tmp/db")
+
+# to see low-level events in the console uncomment the following line
+# ac.add_account_plugin(deltachat.eventlogger.FFIEventLogger(ac, ""))
+
+if not ac.is_configured():
+    ac.set_config("addr", "tmpy.94mtm@testrun.org")
+    ac.set_config("mail_pw", "5CbD6VnjD/li")
+    ac.set_config("mvbox_watch", "0")
+    ac.set_config("sentbox_watch", "0")
+
+class MyPlugin:
+    @deltachat.hookspec.account_hookimpl
+    def process_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            print("shutting down")
+            ac.shutdown()
+        else:
+            ch = ac.create_chat_by_contact(message.get_sender_contact())
+            ch.send_text("echoing {}".format(message.text))
+
+    @deltachat.hookspec.account_hookimpl
+    def process_message_delivered(self, message):
+        print("process_message_delivered", message)
+
+ac.add_account_plugin(MyPlugin())
+
+# start IO threads and perform configuration
+ac.start()
+
+print("waiting for /quit or other message on {}".format(ac.get_config("addr")))
+
+ac.wait_shutdown()

python/src/deltachat/account.py

@@ -15,6 +15,7 @@ from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker
 from . import hookspec, iothreads
+from queue import Queue
 
 
 class MissingCredentials(ValueError):
@@ -48,6 +49,9 @@ class Account(object):
         hook.account_init(account=self, db_path=db_path)
 
         self._threads = iothreads.IOThreads(self)
+        self._hook_event_queue = Queue()
+        self._in_use_iter_events = False
+        self._shutdown_event = Event()
 
         # open database
         if hasattr(db_path, "encode"):
@@ -56,30 +60,13 @@ class Account(object):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
         atexit.register(self.shutdown)
-        self._shutdown_event = Event()
 
     @hookspec.account_hookimpl
     def process_ffi_event(self, ffi_event):
-        name = ffi_event.name
+        name, kwargs = self._map_ffi_event(ffi_event)
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+        if name is not None:
-            data1 = ffi_event.data1
+            ev = HookEvent(self, name=name, kwargs=kwargs)
-            if data1 == 0 or data1 == 1000:
+            self._hook_event_queue.put(ev)
-                success = data1 == 1000
-                self._pm.hook.configure_completed(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = self.get_message_by_id(ffi_event.data2)
-            self._pm.hook.process_incoming_message(message=msg)
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = self.get_message_by_id(ffi_event.data2)
-                self._pm.hook.process_incoming_message(message=msg)
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = self.get_message_by_id(ffi_event.data2)
-            self._pm.hook.process_message_delivered(message=msg)
-        elif name == "DC_EVENT_MEMBER_ADDED":
-            chat = self.get_chat_by_id(ffi_event.data1)
-            contact = self.get_contact_by_id(ffi_event.data2)
-            self._pm.hook.member_added(chat=chat, contact=contact)
 
     # def __del__(self):
     #    self.shutdown()
@@ -547,7 +534,7 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def start(self):
+    def start(self, callback_thread=True):
         """ start this account (activate imap/smtp threads etc.)
         and return immediately.
 
@@ -565,7 +552,7 @@ class Account(object):
             if not self.get_config("addr") or not self.get_config("mail_pw"):
                 raise MissingCredentials("addr or mail_pwd not set in config")
             lib.dc_configure(self._dc_context)
-        self._threads.start()
+        self._threads.start(callback_thread=callback_thread)
 
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
@@ -582,12 +569,51 @@ class Account(object):
             self.stop_ongoing()
             self._threads.stop(wait=False)
         lib.dc_close(dc_context)
+        self._hook_event_queue.put(None)
         self._threads.stop(wait=wait)  # to wait for threads
         self._dc_context = None
         atexit.unregister(self.shutdown)
+        self._shutdown_event.set()
         hook = hookspec.Global._get_plugin_manager().hook
         hook.account_after_shutdown(account=self, dc_context=dc_context)
-        self._shutdown_event.set()
+
+    def iter_events(self, timeout=None):
+        """ yield hook events until shutdown.
+
+        It is not allowed to call iter_events() from multiple threads.
+        """
+        if self._in_use_iter_events:
+            raise RuntimeError("can only call iter_events() from one thread")
+        self._in_use_iter_events = True
+        while 1:
+            event = self._hook_event_queue.get(timeout=timeout)
+            if event is None:
+                break
+            yield event
+
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                return "configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "process_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = self.get_message_by_id(ffi_event.data2)
+                if msg.is_in_fresh():
+                    return "process_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "process_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_MEMBER_ADDED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "member_added", dict(chat=chat, contact=contact)
+        return None, {}
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -614,3 +640,17 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
+
+
+class HookEvent:
+    def __init__(self, account, name, kwargs):
+        assert hasattr(account._pm.hook, name), name
+        self.account = account
+        self.name = name
+        self.kwargs = kwargs
+
+    def call_hook(self):
+        hook = getattr(self.account._pm.hook, self.name, None)
+        if hook is None:
+            raise ValueError("event_name {} unknown".format(self.name))
+        return hook(**self.kwargs)

python/src/deltachat/hookspec.py

@@ -15,7 +15,8 @@ global_hookimpl = pluggy.HookimplMarker(_global_name)
 class PerAccount:
     """ per-Account-instance hook specifications.
 
-    If you write a plugin you need to implement one of the following hooks.
+    Except for process_ffi_event all hooks are executed
+    in the thread which calls Account.wait_shutdown().
     """
     @classmethod
     def _make_plugin_manager(cls):
@@ -29,6 +30,10 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec

python/src/deltachat/iothreads.py

@@ -17,13 +17,17 @@ class IOThreads:
     def is_started(self):
         return len(self._name2thread) > 0
 
-    def start(self):
+    def start(self, callback_thread):
         assert not self.is_started()
         self._start_one_thread("inbox", self.imap_thread_run)
         self._start_one_thread("smtp", self.smtp_thread_run)
 
+        if callback_thread:
+            self._start_one_thread("cb", self.cb_thread_run)
+
         if int(self.account.get_config("mvbox_watch")):
             self._start_one_thread("mvbox", self.mvbox_thread_run)
+
         if int(self.account.get_config("sentbox_watch")):
             self._start_one_thread("sentbox", self.sentbox_thread_run)
 
@@ -53,7 +57,18 @@ class IOThreads:
             lib.dc_interrupt_sentbox_idle(self._dc_context)
         if wait:
             for name, thread in self._name2thread.items():
-                thread.join()
+                if thread != threading.currentThread():
+                    thread.join()
+
+    def cb_thread_run(self):
+        with self.log_execution("CALLBACK THREAD START"):
+            it = self.account.iter_events()
+            while not self._thread_quitflag:
+                try:
+                    ev = next(it)
+                except StopIteration:
+                    break
+                ev.call_hook()
 
     def imap_thread_run(self):
         with self.log_execution("INBOX THREAD START"):

python/src/deltachat/testplugin.py

@@ -3,12 +3,10 @@ import os
 import sys
 import pytest
 import requests
-from contextlib import contextmanager
 import time
 from . import Account, const
 from .tracker import ConfigureTracker
 from .capi import lib
-from .hookspec import account_hookimpl
 from .eventlogger import FFIEventLogger, FFIEventTracker
 from _pytest.monkeypatch import MonkeyPatch
 import tempfile
@@ -63,9 +61,9 @@ def pytest_report_header(config, startdir):
 
     cfg = config.option.liveconfig
     if cfg:
-        if "#" in cfg:
+        if "?" in cfg:
-            url, token = cfg.split("#", 1)
+            url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
+            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
         else:
             summary.append('Liveconfig file: {}'.format(cfg))
     return summary
@@ -263,26 +261,3 @@ def lp():
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
     return Printer()
-
-
-@pytest.fixture
-def make_plugin_recorder():
-    @contextmanager
-    def make_plugin_recorder(account):
-        class HookImpl:
-            def __init__(self):
-                self.calls_member_added = []
-
-            @account_hookimpl
-            def member_added(self, chat, contact):
-                self.calls_member_added.append(dict(chat=chat, contact=contact))
-
-            def get_first(self, name):
-                val = getattr(self, "calls_" + name, None)
-                if val is not None:
-                    return val.pop(0)
-
-        with account.temp_plugin(HookImpl()) as plugin:
-            yield plugin
-
-    return make_plugin_recorder

python/tests/test_account.py

@@ -170,18 +170,19 @@ class TestOfflineChat:
         else:
             pytest.fail("could not find chat")
 
-    def test_add_member_event(self, ac1, make_plugin_recorder):
+    def test_add_member_event(self, ac1):
         chat = ac1.create_group_chat(name="title1")
         assert chat.is_group()
         # promote the chat
         chat.send_text("hello")
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
 
-        with make_plugin_recorder(ac1) as rec:
+        chat.add_contact(contact1)
-            chat.add_contact(contact1)
+        for ev in ac1.iter_events(timeout=1):
-            kwargs = rec.get_first("member_added")
+            if ev.name == "member_added":
-            assert kwargs["chat"] == chat
+                assert ev.kwargs["chat"] == chat
-            assert kwargs["contact"] == contact1
+                assert ev.kwargs["contact"] == contact1
+                break
 
     def test_group_chat_creation(self, ac1):
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
@@ -461,6 +462,11 @@ class TestOnlineAccount:
             ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr")))
         return chat
 
+    def test_double_iter_events(self, acfactory):
+        ac1 = acfactory.get_one_online_account()
+        with pytest.raises(RuntimeError):
+            next(ac1.iter_events())
+
     @pytest.mark.ignored
     def test_configure_generate_key(self, acfactory, lp):
         # A slow test which will generate new keys.