feat: support underscore-prefixed domains with self-signed TLS certificates

Allow Delta Chat core to work with chatmail servers running on
underscore-prefixed domains (e.g. _alice.localchat) which use
self-signed TLS certificates. This is mirroring related work
on chatmail relays: https://github.com/chatmail/relay/pull/855
Underscore domains with self-signed TLS certs can be used by LXC test
containers where obtaining real certificates is not practical.

When the domain starts with '_', certificate verification is
automatically relaxed for IMAP/SMTP connections, dcaccount QR
code handling, and iroh relay endpoints. The Python test suite
is adapted to also work against such underscore-domain servers,
including cross-core tests with older Delta Chat versions.

Note: this PR does not support HTTPS requests with underscore
domains. They are not currently needed for working with LXC test
containers.

14 files changed, +102/-31 lines (excluding Cargo.lock).
Cargo.lock: +606/-11 lines from enabling iroh features
needed for connecting to iroh relay endpoint on underscore domains.
The added dependencies are unfortunate but best considered
when finally upgrading to iroh 1.0 (tm).

Cargo.toml

@@ -67,7 +67,7 @@ hyper = "1"
 hyper-util = "0.1.16"
 image = { version = "0.25.6", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 iroh-gossip = { version = "0.35", default-features = false, features = ["net"] }
-iroh = { version = "0.35", default-features = false }
+iroh = { version = "0.35", default-features = false, features = ["test-utils", "metrics"] }
 kamadak-exif = "0.6.1"
 libc = { workspace = true }
 mail-builder = { version = "0.4.4", default-features = false }

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -9,6 +9,7 @@ import platform
 import random
 import subprocess
 import sys
+import urllib.parse
 from typing import AsyncGenerator, Optional
 
 import execnet
@@ -283,8 +284,16 @@ def alice_and_remote_bob(tmp_path, acfactory, get_core_python_env):
         channel = gw.remote_exec(remote_bob_loop)
         cm = os.environ.get("CHATMAIL_DOMAIN")
 
+        # Build a dclogin QR code for the remote bob account.
+        # Using dclogin scheme with ic=3&sc=3 allows old cores
+        # to accept invalid certificates for underscore-prefixed domains.
+        addr, password = acfactory.get_credentials()
+        dclogin_qr = f"dclogin://{urllib.parse.quote(addr, safe='@')}?p={urllib.parse.quote(password)}&v=1"
+        if cm and cm.startswith("_"):
+            dclogin_qr += "&ic=3&sc=3"
+
         # trigger getting an online account on bob's side
-        channel.send((accounts_dir, str(rpc_server_path), cm))
+        channel.send((accounts_dir, str(rpc_server_path), dclogin_qr))
 
         # meanwhile get a local alice account
         alice = acfactory.get_online_account()
@@ -316,10 +325,8 @@ def remote_bob_loop(channel):
     import os
 
     from deltachat_rpc_client import DeltaChat, Rpc
-    from deltachat_rpc_client.pytestplugin import ACFactory
 
-    accounts_dir, rpc_server_path, chatmail_domain = channel.receive()
-    os.environ["CHATMAIL_DOMAIN"] = chatmail_domain
+    accounts_dir, rpc_server_path, dclogin_qr = channel.receive()
 
     # older core versions don't support specifying rpc_server_path
     # so we can't just pass `rpc_server_path` argument to Rpc constructor
@@ -330,8 +337,14 @@ def remote_bob_loop(channel):
     with rpc:
         dc = DeltaChat(rpc)
         channel.send(dc.rpc.get_system_info()["deltachat_core_version"])
-        acfactory = ACFactory(dc)
-        bob = acfactory.get_online_account()
+
+        # Configure account using dclogin scheme directly,
+        # avoiding the old ACFactory which doesn't handle
+        # underscore-prefixed domains' TLS on older cores.
+        bob = dc.add_account()
+        bob.set_config_from_qr(dclogin_qr)
+        bob.bring_online()
+
         alice_vcard = channel.receive()
         [alice_contact] = bob.import_vcard(alice_vcard)
         ns = {"bob": bob, "bob_contact_alice": alice_contact}
@@ -345,3 +358,4 @@ def remote_bob_loop(channel):
             except Exception:
                 # some unserializable result
                 channel.send(None)
+

deltachat-rpc-client/tests/conftest.py

@@ -37,7 +37,11 @@ class DirectImap:
         host = user.rsplit("@")[-1]
         pw = self.account.get_config("mail_pw")
 
-        self.conn = MailBox(host, port, ssl_context=ssl.create_default_context())
+        ssl_context = ssl.create_default_context()
+        if host.startswith("_"):
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+        self.conn = MailBox(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")

deltachat-rpc-client/tests/test_calls.py

@@ -77,7 +77,7 @@ def test_ice_servers(acfactory) -> None:
     alice = acfactory.get_online_account()
 
     ice_servers = alice.ice_servers()
-    assert len(ice_servers) == 1
+    assert len(ice_servers) >= 1
 
 
 def test_no_contact_request_call(acfactory) -> None:

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -109,7 +109,7 @@ def test_delivery_status_failed(acfactory: ACFactory) -> None:
     assert failing_message.get_snapshot().state == const.MessageState.OUT_FAILED
 
 
-def test_download_on_demand(acfactory: ACFactory) -> None:
+def test_download_on_demand(acfactory: ACFactory, data) -> None:
     """
     Test if download on demand emits chatlist update events.
     This is only needed for last message in chat, but finding that out is too expensive, so it's always emitted
@@ -127,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
     msg.get_snapshot().chat.accept()
     bob.get_chat_by_id(chat_id).send_message(
         "Hello World, this message is bigger than 5 bytes",
-        file="../test-data/image/screenshot.jpg",
+        file=data.get_path("image/screenshot.jpg"),
     )
 
     message = alice.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_cross_core.py

@@ -36,7 +36,7 @@ def test_qr_setup_contact(alice_and_remote_bob, version) -> None:
 
 def test_send_and_receive_message(alice_and_remote_bob) -> None:
     """Test other-core Bob profile can send a message to Alice on current core."""
-    alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.20.0")
+    alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.24.0")
 
     remote_eval("bob_contact_alice.create_chat().send_text('hello')")
 
@@ -46,7 +46,7 @@ def test_send_and_receive_message(alice_and_remote_bob) -> None:
 
 def test_second_device(acfactory, alice_and_remote_bob) -> None:
     """Test setting up current version as a second device for old version."""
-    _alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.20.0")
+    _alice, alice_contact_bob, remote_eval = alice_and_remote_bob("2.24.0")
 
     remote_eval("locals().setdefault('future', bob._rpc.provide_backup.future(bob.id))")
     qr = remote_eval("bob._rpc.get_backup_qr(bob.id)")

deltachat-rpc-client/tests/test_multitransport.py

@@ -120,7 +120,7 @@ def test_change_address(acfactory) -> None:
     assert sender_addr2 == new_alice_addr
 
 
-def test_download_on_demand(acfactory) -> None:
+def test_download_on_demand(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
     alice.set_config("download_limit", "1")
 
@@ -131,7 +131,7 @@ def test_download_on_demand(acfactory) -> None:
 
     alice.create_chat(bob)
     chat_bob_alice = bob.create_chat(alice)
-    chat_bob_alice.send_message(file="../test-data/image/screenshot.jpg")
+    chat_bob_alice.send_message(file=data.get_path("image/screenshot.jpg"))
     msg = alice.wait_for_incoming_msg()
     snapshot = msg.get_snapshot()
     assert snapshot.download_state == DownloadState.AVAILABLE

deltachat-rpc-client/tests/test_securejoin.py

@@ -141,10 +141,9 @@ def test_qr_securejoin_broadcast(acfactory, all_devices_online):
 
     def wait_for_broadcast_messages(ac):
         snapshot1 = ac.wait_for_incoming_msg().get_snapshot()
-        assert snapshot1.text == "You joined the channel."
-
         snapshot2 = ac.wait_for_incoming_msg().get_snapshot()
-        assert snapshot2.text == "Hello everyone!"
+        texts = {snapshot1.text, snapshot2.text}
+        assert texts == {"You joined the channel.", "Hello everyone!"}
 
         chat = get_broadcast(ac)
         assert snapshot1.chat_id == chat.id

deltachat-rpc-client/tests/test_something.py

@@ -97,8 +97,11 @@ def test_lowercase_address(acfactory) -> None:
 
 def test_configure_ip(acfactory) -> None:
     addr, password = acfactory.get_credentials()
+    domain = addr.rsplit("@")[-1]
+    if domain.startswith("_"):
+        pytest.skip("Underscore domains accept invalid certificates")
     account = acfactory.get_unconfigured_account()
-    ip_address = socket.gethostbyname(addr.rsplit("@")[-1])
+    ip_address = socket.gethostbyname(domain)
 
     with pytest.raises(JsonRpcError):
         account.add_or_update_transport(
@@ -1012,7 +1015,12 @@ def test_configured_imap_certificate_checks(acfactory):
     alice = acfactory.new_configured_account()
 
     # Certificate checks should be configured (not None)
-    assert "cert_strict" in alice.get_info().used_transport_settings
+    info = alice.get_info()
+    domain = alice.get_config("addr").split("@")[-1]
+    if domain.startswith("_"):
+        assert "cert_accept_invalid_certificates" in info.used_transport_settings
+    else:
+        assert "cert_strict" in info.used_transport_settings
 
     # "cert_old_automatic" is the value old Delta Chat core versions used
     # to mean user entered "imap_certificate_checks=0" (Automatic)
@@ -1361,7 +1369,7 @@ def test_synchronize_member_list_on_group_rejoin(acfactory, log):
     assert msg.get_snapshot().chat.num_contacts() == 2
 
 
-def test_large_message(acfactory) -> None:
+def test_large_message(acfactory, data) -> None:
     """
     Test sending large message without download limit set,
     so it is sent with pre-message but downloaded without user interaction.
@@ -1371,7 +1379,7 @@ def test_large_message(acfactory) -> None:
     alice_chat_bob = alice.create_chat(bob)
     alice_chat_bob.send_message(
         "Hello World, this message is bigger than 5 bytes",
-        file="../test-data/image/screenshot.jpg",
+        file=data.get_path("image/screenshot.jpg"),
     )
 
     msg = bob.wait_for_incoming_msg()

deltachat-rpc-client/tests/test_webxdc.py

@@ -1,9 +1,9 @@
-def test_webxdc(acfactory) -> None:
+def test_webxdc(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+    alice_chat_bob.send_message(text="Let's play chess!", file=data.get_path("webxdc/chess.xdc"))
 
     event = bob.wait_for_incoming_msg_event()
     bob_chat_alice = bob.get_chat_by_id(event.chat_id)
@@ -41,12 +41,12 @@ def test_webxdc(acfactory) -> None:
     ]
 
 
-def test_webxdc_insert_lots_of_updates(acfactory) -> None:
+def test_webxdc_insert_lots_of_updates(acfactory, data) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     alice_contact_bob = alice.create_contact(bob, "Bob")
     alice_chat_bob = alice_contact_bob.create_chat()
-    message = alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+    message = alice_chat_bob.send_message(text="Let's play chess!", file=data.get_path("webxdc/chess.xdc"))
 
     for i in range(2000):
         message.send_webxdc_status_update({"payload": str(i)}, "description")

src/configure.rs

@@ -581,7 +581,13 @@ async fn get_configured_param(
         smtp_password,
         provider,
         certificate_checks: match param.certificate_checks {
-            EnteredCertificateChecks::Automatic => ConfiguredCertificateChecks::Automatic,
+            EnteredCertificateChecks::Automatic => {
+                if param_domain.starts_with('_') {
+                    ConfiguredCertificateChecks::AcceptInvalidCertificates
+                } else {
+                    ConfiguredCertificateChecks::Automatic
+                }
+            }
             EnteredCertificateChecks::Strict => ConfiguredCertificateChecks::Strict,
             EnteredCertificateChecks::AcceptInvalidCertificates
             | EnteredCertificateChecks::AcceptInvalidCertificates2 => {

src/peer_channels.rs

@@ -238,18 +238,21 @@ impl Context {
         let secret_key = SecretKey::generate(rand_old::rngs::OsRng);
         let public_key = secret_key.public();
 
-        let relay_mode = if let Some(relay_url) = self
+        let (relay_mode, skip_relay_tls) = if let Some(relay_url) = self
             .metadata
             .read()
             .await
             .as_ref()
             .and_then(|conf| conf.iroh_relay.clone())
         {
-            RelayMode::Custom(RelayUrl::from(relay_url).into())
+            // Underscore-prefixed domains use self-signed TLS certificates,
+            // so we need to skip relay certificate verification for them.
+            let skip = relay_url.host_str().map_or(false, |h| h.starts_with('_'));
+            (RelayMode::Custom(RelayUrl::from(relay_url).into()), skip)
         } else {
             // FIXME: this should be RelayMode::Disabled instead.
             // Currently using default relays because otherwise Rust tests fail.
-            RelayMode::Default
+            (RelayMode::Default, false)
         };
 
         let endpoint = Endpoint::builder()
@@ -257,6 +260,7 @@ impl Context {
             .secret_key(secret_key)
             .alpns(vec![GOSSIP_ALPN.to_vec()])
             .relay_mode(relay_mode)
+            .insecure_skip_relay_cert_verify(skip_relay_tls)
             .bind()
             .await?;
 

src/qr.rs

@@ -817,6 +817,11 @@ pub(crate) async fn login_param_from_account_qr(
         .context("Invalid DCACCOUNT scheme")?;
 
     if !payload.starts_with(HTTPS_SCHEME) {
+        let certificate_checks = if payload.starts_with('_') {
+            EnteredCertificateChecks::AcceptInvalidCertificates
+        } else {
+            EnteredCertificateChecks::Strict
+        };
         let rng = &mut rand::rngs::OsRng.unwrap_err();
         let username = Alphanumeric.sample_string(rng, 9);
         let addr = username + "@" + payload;
@@ -829,7 +834,7 @@ pub(crate) async fn login_param_from_account_qr(
                 ..Default::default()
             },
             smtp: Default::default(),
-            certificate_checks: EnteredCertificateChecks::Strict,
+            certificate_checks,
             oauth2: false,
         };
         return Ok(param);

src/qr/qr_tests.rs

@@ -754,6 +754,35 @@ async fn test_decode_empty_account() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_decode_account_underscore_domain() -> Result<()> {
+    let ctx = TestContext::new().await;
+
+    // Underscore domain is kept as-is.
+    let qr = check_qr(&ctx.ctx, "dcaccount:_example.org").await?;
+    assert_eq!(
+        qr,
+        Qr::Account {
+            domain: "_example.org".to_string()
+        }
+    );
+
+    // Verify login params use AcceptInvalidCertificates for underscore domain.
+    let param = login_param_from_account_qr(&ctx.ctx, "dcaccount:_example.org").await?;
+    assert!(param.addr.ends_with("@_example.org"));
+    assert_eq!(
+        param.certificate_checks,
+        EnteredCertificateChecks::AcceptInvalidCertificates
+    );
+
+    // Regular domain still uses Strict.
+    let param = login_param_from_account_qr(&ctx.ctx, "dcaccount:example.org").await?;
+    assert!(param.addr.ends_with("@example.org"));
+    assert_eq!(param.certificate_checks, EnteredCertificateChecks::Strict);
+
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_decode_tg_socks_proxy() -> Result<()> {
     let t = TestContext::new().await;