feat: support underscore-prefixed domains with self-signed TLS certificates

Allow Delta Chat core to work with chatmail servers running on
underscore-prefixed domains (e.g. _alice.localchat) which use
self-signed TLS certificates. This is mirroring related work
on chatmail relays: https://github.com/chatmail/relay/pull/855
Underscore domains with self-signed TLS certs can be used by LXC test
containers where obtaining real certificates is not practical.

When the domain starts with '_', certificate verification is
automatically relaxed for IMAP/SMTP connections, dcaccount QR
code handling, and iroh relay endpoints. The Python test suite
is adapted to also work against such underscore-domain servers,
including cross-core tests with older Delta Chat versions.

Note: this PR does not support HTTPS requests with underscore
domains. They are not currently needed for working with LXC test
containers.

14 files changed, +102/-31 lines (excluding Cargo.lock).
Cargo.lock: +606/-11 lines from enabling iroh features
needed for connecting to iroh relay endpoint on underscore domains.
The added dependencies are unfortunate but best considered
when finally upgrading to iroh 1.0 (tm).

deltachat-rpc-client/tests/conftest.py

@@ -37,7 +37,11 @@ class DirectImap:
         host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
+        ssl_context = ssl.create_default_context()
+        if host.startswith("_"):
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+        self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

deltachat-rpc-client/tests/test_calls.py

@@ -77,7 +77,7 @@ def test_ice_servers(acfactory) -> None:
     alice = acfactory.get_online_account()
 
     ice_servers = alice.ice_servers()
-    assert len(ice_servers) == 1
+    assert len(ice_servers) >= 1
 
 
 def test_no_contact_request_call(acfactory) -> None:

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -109,7 +109,7 @@ def test_delivery_status_failed(acfactory: ACFactory) -> None:
     assert failing_message.get_snapshot().state == const.MessageState.OUT_FAILED
 
 
-def test_download_on_demand(acfactory: ACFactory) -> None:
+def test_download_on_demand(acfactory: ACFactory, data) -> None:
     """
     Test if download on demand emits chatlist update events.
     This is only needed for last message in chat, but finding that out is too expensive, so it's always emitted
@@ -127,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
     msg.get_snapshot().chat.accept()
     bob.get_chat_by_id(chat_id).send_message(
         "Hello World, this message is bigger than 5 bytes",
-        file="../test-data/image/screenshot.jpg",
+        file=data.get_path("image/screenshot.jpg"),
     )
 
     message = alice.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_cross_core.py

@@ -36,7 +36,7 @@ def test_qr_setup_contact(alice_and_remote_bob, version) -> None:
 
 def test_send_and_receive_message(alice_and_remote_bob) -> None:
     """Test other-core Bob profile can send a message to Alice on current core."""
-    alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.20.0")
+    alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.24.0")
 
     remote_eval("bob_contact_alice.create_chat().send_text('hello')")
 
@@ -46,7 +46,7 @@ def test_send_and_receive_message(alice_and_remote_bob) -> None:
 
 def test_second_device(acfactory, alice_and_remote_bob) -> None:
     """Test setting up current version as a second device for old version."""
-    _alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.20.0")
+    _alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.24.0")
 
     remote_eval("locals().setdefault('future', bob._rpc.provide_backup.future(bob.id))")
     qr = remote_eval("bob._rpc.get_backup_qr(bob.id)")

deltachat-rpc-client/tests/test_multitransport.py

@@ -120,7 +120,7 @@ def test_change_address(acfactory) -> None:
     assert sender_addr2 == new_alice_addr
 
 
-def test_download_on_demand(acfactory) -> None:
+def test_download_on_demand(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
     alice.set_config("download_limit", "1")
 
@@ -131,7 +131,7 @@ def test_download_on_demand(acfactory) -> None:
 
     alice.create_chat(bob)
     chat_bob_alice = bob.create_chat(alice)
-    chat_bob_alice.send_message(file="../test-data/image/screenshot.jpg")
+    chat_bob_alice.send_message(file=data.get_path("image/screenshot.jpg"))
     msg = alice.wait_for_incoming_msg()
     snapshot = msg.get_snapshot()
     assert snapshot.download_state == DownloadState.AVAILABLE

deltachat-rpc-client/tests/test_securejoin.py

@@ -141,10 +141,9 @@ def test_qr_securejoin_broadcast(acfactory, all_devices_online):
 
     def wait_for_broadcast_messages(ac):
         snapshot1 = ac.wait_for_incoming_msg().get_snapshot()
-        assert snapshot1.text == "You joined the channel."
-
         snapshot2 = ac.wait_for_incoming_msg().get_snapshot()
-        assert snapshot2.text == "Hello everyone!"
+        texts = {snapshot1.text, snapshot2.text}
+        assert texts == {"You joined the channel.", "Hello everyone!"}
 
         chat = get_broadcast(ac)
         assert snapshot1.chat_id == chat.id

deltachat-rpc-client/tests/test_something.py

@@ -97,8 +97,11 @@ def test_lowercase_address(acfactory) -> None:
 
 def test_configure_ip(acfactory) -> None:
     addr, password = acfactory.get_credentials()
+    domain = addr.rsplit("@")[-1]
+    if domain.startswith("_"):
+        pytest.skip("Underscore domains accept invalid certificates")
     account = acfactory.get_unconfigured_account()
-    ip_address = socket.gethostbyname(addr.rsplit("@")[-1])
+    ip_address = socket.gethostbyname(domain)
 
     with pytest.raises(JsonRpcError):
         account.add_or_update_transport(
@@ -1012,7 +1015,12 @@ def test_configured_imap_certificate_checks(acfactory):
     alice = acfactory.new_configured_account()
 
     # Certificate checks should be configured (not None)
-    assert "cert_strict" in alice.get_info().used_transport_settings
+    info = alice.get_info()
+    domain = alice.get_config("addr").split("@")[-1]
+    if domain.startswith("_"):
+        assert "cert_accept_invalid_certificates" in info.used_transport_settings
+    else:
+        assert "cert_strict" in info.used_transport_settings
 
     # "cert_old_automatic" is the value old Delta Chat core versions used
     # to mean user entered "imap_certificate_checks=0" (Automatic)
@@ -1361,7 +1369,7 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log):
     assert msg.get_snapshot().chat.num_contacts() == 2
 
 
-def test_large_message(acfactory) -> None:
+def test_large_message(acfactory, data) -> None:
     """
     Test sending large message without download limit set,
     so it is sent with pre-message but downloaded without user interaction.
@@ -1371,7 +1379,7 @@ def test_large_message(acfactory) -> None:
     alice_chat_bob = alice.create_chat(bob)
     alice_chat_bob.send_message(
         "Hello World, this message is bigger than 5 bytes",
-        file="../test-data/image/screenshot.jpg",
+        file=data.get_path("image/screenshot.jpg"),
     )
 
     msg = bob.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_webxdc.py

@@ -1,9 +1,9 @@
-def test_webxdc(acfactory) -> None:
+def test_webxdc(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+    alice_chat_bob.send_message(text="Let's play chess!", file=data.get_path("webxdc/chess.xdc"))
 
     event = bob.wait_for_incoming_msg_event()
     bob_chat_alice = bob.get_chat_by_id(event.chat_id)
@@ -41,12 +41,12 @@ def test_webxdc(acfactory) -> None:
     ]
 
 
-def test_webxdc_insert_lots_of_updates(acfactory) -> None:
+def test_webxdc_insert_lots_of_updates(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
-    message = alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+    message = alice_chat_bob.send_message(text="Let's play chess!", file=data.get_path("webxdc/chess.xdc"))
 
     for i in range(2000):
         message.send_webxdc_status_update({"payload": str(i)}, "description")