fix buffer handling so that the group-tracking bot example passes

python/examples/group_tracking.py

@@ -12,6 +12,10 @@ class GroupTrackingPlugin:
         for member in message.chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
+    @deltachat.hookspec.account_hookimpl
+    def configure_completed(self, success):
+        print("*** configure_completed:", success)
+
     @deltachat.hookspec.account_hookimpl
     def member_added(self, chat, contact):
         print("*** member_added", contact.addr, "from", chat)

python/examples/test_examples.py

@@ -3,6 +3,7 @@ import pytest
 import py
 import echo_and_quit
 import group_tracking
+from deltachat.eventlogger import FFIEventLogger
 
 
 @pytest.fixture(scope='session')
@@ -30,19 +31,39 @@ def test_echo_quit_plugin(acfactory):
     botproc.wait()
 
 
-@pytest.mark.skip(reason="botproc-matching not implementing")
+def test_group_tracking_plugin(acfactory, lp):
-def test_group_tracking_plugin(acfactory):
+    lp.sec("creating one group-tracking bot and two temp accounts")
     botproc = acfactory.run_bot_process(group_tracking)
 
-    ac1 = acfactory.get_one_online_account()
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_group_chat("bot test group")
-    ch1.add_contact(bot_contact)
-    ch1.send_text("hello")
-    ch1.add_contact(ac1.create_contact("x@example.org"))
 
     botproc.fnmatch_lines("""
-        *member_added x@example.org*
+        *configure_completed: True*
     """)
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
 
-    botproc.kill()
+    lp.sec("creating bot test group with all three")
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch = ac1.create_group_chat("bot test group")
+    ch.add_contact(bot_contact)
+    ch.send_text("hello")
+
+    botproc.fnmatch_lines("""
+        *member_added {}*
+    """.format(ac1.get_config("addr")))
+
+    lp.sec("adding third member {}".format(ac2.get_config("addr")))
+    contact3 = ac1.create_contact(ac2.get_config("addr"))
+    ch.add_contact(contact3)
+
+    lp.sec("now looking at what the bot received")
+    botproc.fnmatch_lines("""
+        *member_added {}*
+    """.format(contact3.addr))
+
+    lp.sec("contact successfully added, now removing")
+    ch.remove_contact(contact3)
+    botproc.fnmatch_lines("""
+        *member_removed {}*
+    """.format(contact3.addr))

python/src/deltachat/__init__.py

@@ -115,7 +115,7 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = eventlogger.FFIEventLogger(ac, "echo")
+        log = eventlogger.FFIEventLogger(ac, "bot")
         ac.add_account_plugin(log)
 
     if not ac.is_configured():
@@ -124,6 +124,7 @@ def run_cmdline(argv=None, account_plugins=None):
         )
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
+        ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
 

python/src/deltachat/eventlogger.py

@@ -71,7 +71,7 @@ class FFIEventLogger:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
         with self._loglock:
-            print(s)
+            print(s, flush=True)
 
 
 class FFIEventTracker:

python/src/deltachat/testplugin.py

@@ -2,6 +2,9 @@ from __future__ import print_function
 import os
 import sys
 import subprocess
+import queue
+import threading
+import fnmatch
 import pytest
 import requests
 import time
@@ -10,6 +13,8 @@ from .tracker import ConfigureTracker
 from .capi import lib
 from .eventlogger import FFIEventLogger, FFIEventTracker
 from _pytest.monkeypatch import MonkeyPatch
+from _pytest._code import Source
+
 import tempfile
 
 
@@ -138,11 +143,12 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
                 fin = self._finalizers.pop()
                 fin()
 
-        def make_account(self, path, logid):
+        def make_account(self, path, logid, quiet=False):
             ac = Account(path)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
             ac._configtracker = ac.add_account_plugin(ConfigureTracker())
-            ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+            if not quiet:
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
             self._finalizers.append(ac.shutdown)
             return ac
 
@@ -177,7 +183,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
             lib.dc_set_config(ac._dc_context, b"configured", b"1")
             return ac
 
-        def get_online_config(self, pre_generated_key=True):
+        def get_online_config(self, pre_generated_key=True, quiet=False):
             if not session_liveconfig:
                 pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
             configdict = session_liveconfig.get(self.live_count)
@@ -190,7 +196,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
             configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
             ac._evtracker.init_time = self.init_time
@@ -198,9 +204,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
-                                           pre_generated_key=True, config={}):
+                                           pre_generated_key=True, quiet=False, config={}):
             ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key)
+                pre_generated_key=pre_generated_key, quiet=quiet)
             configdict.update(config)
             configdict["mvbox_watch"] = str(int(mvbox))
             configdict["mvbox_move"] = str(int(move))
@@ -217,9 +223,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
             ac1._configtracker.wait_finish()
             return ac1
 
-        def get_two_online_accounts(self, move=False):
+        def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True)
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
-            ac2 = self.get_online_configuring_account()
+            ac2 = self.get_online_configuring_account(quiet=quiet)
             ac1._configtracker.wait_finish()
             ac2._configtracker.wait_finish()
             return ac1, ac2
@@ -247,14 +253,24 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
 
             bot_ac, bot_cfg = self.get_online_config()
 
-            popen = subprocess.Popen([
+            args = [
                 sys.executable,
                 fn,
                 "--show-ffi",
                 "--db", bot_ac.db_path,
                 "--email", bot_cfg["addr"],
                 "--password", bot_cfg["mail_pw"],
-            ])
+            ]
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=1,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
             bot = BotProcess(popen, bot_cfg)
             self._finalizers.append(bot.kill)
             return bot
@@ -269,12 +285,45 @@ class BotProcess:
         self.popen = popen
         self.addr = bot_cfg["addr"]
 
+        # we read stdout as quickly as we can in a thread and make
+        # the (unicode) lines available for readers through a queue.
+        self.stdout_queue = queue.Queue()
+        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
+        t.setDaemon(1)
+        t.start()
+
+    def _run_stdout_thread(self):
+        try:
+            while 1:
+                line = self.popen.stdout.readline()
+                if not line:
+                    break
+                line = line.strip()
+                print("QUEUING:", repr(line))
+                self.stdout_queue.put(line)
+        finally:
+            self.stdout_queue.put(None)
+
     def kill(self):
         self.popen.kill()
 
     def wait(self, timeout=30):
         self.popen.wait(timeout=timeout)
 
+    def fnmatch_lines(self, pattern_lines):
+        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
+        for next_pattern in patterns:
+            print("+++FNMATCH:", next_pattern)
+            while 1:
+                line = self.stdout_queue.get(timeout=15)
+                if line is None:
+                    raise IOError("BOT stdout-thread terminated")
+                if fnmatch.fnmatch(line, next_pattern):
+                    print("+++MATCHED:", line)
+                    break
+                else:
+                    print("+++IGN:", line)
+
 
 @pytest.fixture
 def tmp_db_path(tmpdir):