Connectivity view (instead of spamming the user with error_network when sth fails) (#2319)

See https://support.delta.chat/t/discussion-how-to-show-error-states/1363/10 <!-- comment -->

It turns out that it's pretty easy to distinguish between lots of states (currently Error/NotConnected, Connecting…, Getting new messages… and Connected). What's not that easy is distinguishing between an actual error and no network, because if the server just doesn't respond, it could mean that we don't have network or that we are trying ipv6, but only ipv4 works.

**WRT debouncing:**

Sending of EVENT_CONNECTIVITY_CHANGED is not debounced, but emitted every time one of the 3 threads (Inbox, Mvbox and Sentbox) has a network error, starts fetching data, or is done fetching data.
This means that it is emitted:
- 9 times when dc_maybe_network() is called or we get network connection
- 12 times when we lose network connection

Some measurements: dc_get_connectivity() takes a little more than 1ms (in my measurements back in March), dc_get_connectivity_html() takes 10-20ms. This means that it's no immmediate problem to call them very often, might increase battery drain though. For the UI it may be a lot of work to update the title everytime; at least Android is smart enough to update the title only once.

Possible problems (we don't have to worry about them now I think):
- Due to the scan_folders feature, if the user has lots of folders, the state could be "Connecting..." for quite a long time, generally DC seemed a little unresponsive to me because it took so long for "Connecting..." to go away. Telegram has a state "Updating..." that sometimes comes after "Connecting...".

To be done in other PRs:
- Better handle the case that the password was changed on the server and authenticating fails, see https://github.com/deltachat/deltachat-core-rust/issues/1923 and https://github.com/deltachat/deltachat-core-rust/issues/1768
- maybe event debouncing  (except for "Connected" connectivity events)

fix https://github.com/deltachat/deltachat-android/issues/1760

python/src/deltachat/_build.py

@@ -150,6 +150,7 @@ def extract_defines(flags):
                 | DC_PROVIDER
                 | DC_KEY_GEN
                 | DC_IMEX
+                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains

python/src/deltachat/account.py

@@ -574,6 +574,15 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
+    def get_connectivity(self):
+        return lib.dc_get_connectivity(self._dc_context)
+
+    def get_connectivity_html(self):
+        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
+
+    def all_work_done(self):
+        return lib.dc_all_work_done(self._dc_context)
+
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 

python/src/deltachat/events.py

@@ -111,6 +111,33 @@ class FFIEventTracker:
             if m is not None:
                 return m.groups()
 
+    def wait_for_connectivity(self, connectivity):
+        """Wait for the specified connectivity.
+        This only works reliably if the connectivity doesn't change
+        again too quickly, otherwise we might miss it."""
+        while 1:
+            if self.account.get_connectivity() == connectivity:
+                return
+            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+    def wait_for_connectivity_change(self, previous, expected_next):
+        """Wait until the connectivity changes to `expected_next`.
+        Fails the test if it changes to something else."""
+        while 1:
+            current = self.account.get_connectivity()
+            if current == expected_next:
+                return
+            elif current != previous:
+                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
+
+            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+    def wait_for_all_work_done(self):
+        while 1:
+            if self.account.all_work_done():
+                return
+            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))

python/tests/test_account.py

@@ -2042,6 +2042,84 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
+    @pytest.mark.parametrize("inbox_watch", ["0", "1"])
+    def test_connectivity(self, acfactory, lp, inbox_watch):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("inbox_watch", inbox_watch)
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTED)
+
+        lp.sec("Test stop_io() and start_io()")
+        ac1.stop_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+        ac1.start_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
+        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTING, const.DC_CONNECTIVITY_CONNECTED)
+
+        lp.sec("Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " +
+               "all messages are fetched")
+
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        ac2.create_chat(ac1).send_text("Hi")
+
+        ac1.direct_imap.idle_check(terminate=False)
+        ac1.maybe_network()
+
+        ac1._evtracker.wait_for_all_work_done()
+        msgs = ac1.create_chat(ac2).get_messages()
+        assert len(msgs) == 1
+        assert msgs[0].text == "Hi"
+
+        lp.sec("Test that the connectivity changes to WORKING while new messages are fetched")
+
+        ac2.create_chat(ac1).send_text("Hi 2")
+
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.maybe_network()
+        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
+        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
+
+        msgs = ac1.create_chat(ac2).get_messages()
+        assert len(msgs) == 2
+        assert msgs[1].text == "Hi 2"
+
+        lp.sec("Test that the connectivity doesn't flicker to WORKING if there are no new messages")
+
+        ac1.maybe_network()
+        while 1:
+            assert ac1.get_connectivity() == const.DC_CONNECTIVITY_CONNECTED
+            if ac1.all_work_done():
+                break
+            ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+        lp.sec("Test that the connectivity doesn't flicker to WORKING if the sender of the message is blocked")
+        ac1.create_contact(ac2).block()
+
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        ac2.create_chat(ac1).send_text("Hi")
+
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.maybe_network()
+
+        while 1:
+            assert ac1.get_connectivity() == const.DC_CONNECTIVITY_CONNECTED
+            if ac1.all_work_done():
+                break
+            ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+        lp.sec("Test that the connectivity is NOT_CONNECTED if the password is wrong")
+
+        ac1.set_config("configured_mail_pw", "abc")
+        ac1.stop_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
+        ac1.start_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
+
     def test_fetch_deleted_msg(self, acfactory, lp):
         """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
         hundreds of times, because uid_next was not updated.
@@ -2758,7 +2836,6 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_user(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2766,7 +2843,6 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_domain(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2774,4 +2850,3 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")