Peek reipients, fetch existing messages

Read all of an e-mail accounts messages and extract all To/CC addresses
if the From was from our own account.
Then, fetch existing messages from the server and show them.

Also, I fixed two other things:
- just by chance my test failed because of an completely unrelated bug.
The bug: bcc_self messages were not marked as read if mvbox_move was set
to true.
- add some color to the test output (minor change)

python/src/deltachat/account.py

@@ -546,6 +546,9 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
+        If you are using this from a test, you may want to call
+        wait_all_initial_fetches() afterwards.
+
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 

python/src/deltachat/direct_imap.py

@@ -24,12 +24,13 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
+    if not hasattr(account, "direct_imap"):
+        imap = DirectImap(account)
+        if imap.select_config_folder("mvbox"):
+            imap.delete(ALL, expunge=True)
+        assert imap.select_config_folder("inbox")
         imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
+        setattr(account, "direct_imap", imap)
 
 
 @deltachat.global_hookimpl

python/src/deltachat/events.py

@@ -1,6 +1,7 @@
 import threading
 import time
 import re
+import os
 from queue import Queue, Empty
 
 import deltachat
@@ -48,6 +49,15 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+
+        if os.name == "posix":
+            WARN = '\033[93m'
+            ERROR = '\033[91m'
+            ENDC = '\033[0m'
+            if message.startswith("DC_EVENT_WARNING"):
+                s = WARN + s + ENDC
+            if message.startswith("DC_EVENT_ERROR"):
+                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -111,6 +121,15 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
+    def wait_all_initial_fetches(self):
+        """Has to be called after start_io() to wait for fetch_existing_msgs to run
+        so that new messages are not mistaken for old ones:
+        - ac1 and ac2 are created
+        - ac1 sends a message to ac2
+        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
+        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        self.get_info_contains("Done fetching existing messages")
+
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")

python/src/deltachat/testplugin.py

@@ -342,10 +342,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
             ac._configtracker = ac.configure()
             return ac
 
         def wait_configure_and_start_io(self):
+            started_accounts = []
             for acc in self._accounts:
                 if hasattr(acc, "_configtracker"):
                     acc._configtracker.wait_finish()
@@ -353,8 +358,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc.set_config("bcc_self", "0")
                 if acc.is_configured() and not acc.is_started():
                     acc.start_io()
+                    started_accounts.append(acc)
                 print("{}: {} account was successfully setup".format(
                     acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
 
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__

python/tests/test_account.py

@@ -1644,6 +1644,7 @@ class TestOnlineAccount:
         chat41 = ac4.create_chat(ac1)
         chat42 = ac4.create_chat(ac2)
         ac4.start_io()
+        ac4._evtracker.wait_all_initial_fetches()
 
         lp.sec("ac1: creating group chat with 2 other members")
         chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
@@ -1923,6 +1924,43 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
+    @pytest.mark.parametrize("mvbox_move", [False, True])
+    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+        """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
+        This way, we can already offer them some email addresses they can write to.
+
+        Also test that existing emails are fetched during onboarding.
+
+        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("send out message with bcc to ourselves")
+        if mvbox_move:
+            ac1.direct_imap.select_config_folder("mvbox")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        chat.send_text("message text")
+
+        # now wait until the bcc_self message arrives
+        # Also test that bcc_self messages moved to the mvbox are marked as read.
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone._configtracker.wait_finish()
+        ac1_clone.start_io()
+
+        ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
+        ac2_addr = ac2.get_config("addr")
+        assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+
+        msg = ac1_clone._evtracker.wait_next_messages_changed()
+        assert msg.text == "message text"
+
 
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):