--wip-- [skip ci]

Enabling brotli does not add new dependencies,
we have it already.
Documentation at <https://docs.rs/rustls/0.23.42/rustls/compress/>
recommends enabling at least brotli.

zlib feature is not enabled as it pulls in duplicate zlib-rs 0.6.6
zlib was even never supported in Chromium (while brotli is).
It might still be interesting to enable it in the future
because it seems to be the only option supported by OpenSSL
and there may be servers that support zlib but not brotli.

RFC 8879 specifies zstd as well, but there is no feature to enable it yet.

This also brings back explicit TLS 1.2 feature that was reverted.
We don't want TLS 1.2 support to be accidentally dropped
if no dependencies explicitly require it anymore.

.github/workflows/deltachat-rpc-server.yml

@@ -514,7 +514,7 @@ jobs:
 
       # Configure Node.js for publishing.
       # Check <https://docs.npmjs.com/trusted-publishers> for the version requirements.
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@v7
         with:
           node-version: 24
           registry-url: "https://registry.npmjs.org"

.github/workflows/jsonrpc-client-npm-package.yml

@@ -24,7 +24,7 @@ jobs:
 
       # Configure Node.js for publishing.
       # Check <https://docs.npmjs.com/trusted-publishers> for the version requirements.
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@v7
         with:
           node-version: 24
           registry-url: "https://registry.npmjs.org"

.github/workflows/jsonrpc.yml

@@ -21,7 +21,7 @@ jobs:
           show-progress: false
           persist-credentials: false
       - name: Use Node.js 24
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: 24
       - name: Add Rust cache

.github/workflows/upload-docs.yml

@@ -90,7 +90,7 @@ jobs:
           persist-credentials: false
           fetch-depth: 0 # Fetch history to calculate VCS version number.
       - name: Use Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: 24
       - name: npm install

Cargo.lock

@@ -1317,7 +1317,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "2.56.0"
+version = "2.57.0-dev"
 dependencies = [
  "anyhow",
  "astral-tokio-tar",
@@ -1425,7 +1425,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "2.56.0"
+version = "2.57.0-dev"
 dependencies = [
  "anyhow",
  "async-channel 2.5.0",
@@ -1446,7 +1446,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "2.56.0"
+version = "2.57.0-dev"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1462,7 +1462,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "2.56.0"
+version = "2.57.0-dev"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1491,7 +1491,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "2.56.0"
+version = "2.57.0-dev"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -5222,6 +5222,8 @@ version = "0.23.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
 dependencies = [
+ "brotli",
+ "brotli-decompressor",
  "log",
  "once_cell",
  "ring",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "2.56.0"
+version = "2.57.0-dev"
 edition = "2024"
 license = "MPL-2.0"
 rust-version = "1.89"
@@ -100,7 +100,7 @@ tagger = "4.3.4"
 textwrap = "0.16.2"
 thiserror = { workspace = true }
 tokio-io-timeout = "1.2.1"
-tokio-rustls = { version = "0.26.2", default-features = false }
+tokio-rustls = { version = "0.26.2", default-features = false, features = ["tls12", "brotli"] }
 tokio-stream = { version = "0.1.17", features = ["fs"] }
 astral-tokio-tar = { version = "0.6.3", default-features = false }
 tokio-util = { workspace = true }

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "2.56.0"
+version = "2.57.0-dev"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "2.56.0"
+version = "2.57.0-dev"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 license = "MPL-2.0"

deltachat-jsonrpc/src/api.rs

@@ -593,7 +593,9 @@ impl CommandApi {
     /// UI implementations must use [`Self::set_transport_unpublished`] instead.
     async fn delete_transport(&self, account_id: u32, addr: String) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        ctx.delete_transport(&addr).await
+        ctx.delete_transport(&addr)
+            .await
+            .context("delete_transport")
     }
 
     /// Change whether the transport is unpublished.

deltachat-jsonrpc/typescript/package.json

@@ -54,5 +54,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "2.56.0"
+  "version": "2.57.0-dev"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "2.56.0"
+version = "2.57.0-dev"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/chatmail/core"

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "2.56.0"
+version = "2.57.0-dev"
 license = "MPL-2.0"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "2.56.0"
+version = "2.57.0-dev"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "2.56.0"
+  "version": "2.57.0-dev"
 }

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "2.56.0"
+version = "2.57.0-dev"
 license = "MPL-2.0"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"

src/automatic_transport_management.rs

@@ -0,0 +1,142 @@
+use std::pin::Pin;
+
+use anyhow::Result;
+use deltachat_contact_tools::addr_normalize;
+use rand::distr::{Alphanumeric, SampleString};
+use rand::seq::IndexedRandom;
+
+use crate::config::Config;
+use crate::log::LogExt as _;
+use crate::login_param::{EnteredCertificateChecks, EnteredImapLoginParam};
+use crate::{configure::EnteredLoginParam, context::Context, tools::time};
+
+/// The target number of transports we try to reach.
+const NUM_TRANSPORTS_TARGET: usize = 3;
+/// How often we want to try adding new transports.
+const AUTOMATIC_ADDITION_DEBOUNCE_SECONDS: i64 = 60 * 60; // one hour
+/// How long we ignore a transport candidate after failing to create an account there:
+const BACKOFF_PERIOD_FOR_NOT_WORKING_TRANSPORT: i64 = 60 * 60 * 24 * 7; // one week
+
+pub(crate) fn maybe_add_additional_transports(
+    context: Context,
+) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+    Box::pin(async move {
+        maybe_add_additional_transports_inner(&context)
+            .await
+            .log_err(&context)
+            .ok();
+    })
+}
+
+async fn maybe_add_additional_transports_inner(context: &Context) -> Result<()> {
+    let now = time();
+    let mut transport_added = false;
+    info!(context, "dbg maybe_add_additional_transports");
+
+    let Ok(_housekeeping_lock) = context.housekeeping_mutex.try_lock() else {
+        // Housekeeping or automatic relay management is already running in another thread, do nothing.
+        info!(context, "dbg skipping because of taken mutex");
+        return Ok(());
+    };
+    if context
+        .get_config_i64(Config::LastAutomaticTransportManagement)
+        .await?
+        > now.saturating_sub(AUTOMATIC_ADDITION_DEBOUNCE_SECONDS)
+    {
+        info!(context, "dbg already ran recently");
+        return Ok(());
+    }
+    // TODO uncomment this after I'm done with testing:
+    if context
+        .get_config_bool(Config::AutomaticTransportManagement)
+        .await?
+    {
+        info!(context, "dbg automatic transport management disabled");
+        return Ok(());
+    }
+    // Set the config at the beginning to avoid endless loops.
+    // Race conditions are not a concern because we locked the mutex.
+    context
+        .set_config_internal(
+            Config::LastAutomaticTransportManagement,
+            Some(&now.to_string()),
+        )
+        .await?;
+
+    // Using `for` instead of `while` to prevent infinite loop
+    for _ in 0..NUM_TRANSPORTS_TARGET {
+        if context.count_transports().await? >= NUM_TRANSPORTS_TARGET {
+            info!(context, "dbg target reached");
+            return Ok(());
+        }
+
+        // First, query all candidates that were not tried since `BACKOFF_PERIOD_FOR_NOT_WORKING_TRANSPORT` seconds.
+        // Domains that are already used are excluded.
+        let cutoff_timestamp = now.saturating_sub(BACKOFF_PERIOD_FOR_NOT_WORKING_TRANSPORT);
+        let candidates: Vec<String> = context
+            .sql
+            .query_map_vec(
+                "SELECT domain FROM relay_candidates WHERE last_tried<?
+                AND NOT EXISTS (
+                    SELECT 1
+                    FROM transports
+                    WHERE substr(addr, instr(addr, '@') + 1) = domain
+                )",
+                (cutoff_timestamp,),
+                |row| Ok(row.get::<_, String>(0)?),
+            )
+            .await?;
+
+        let Some(domain) = candidates.choose(&mut rand::rng()) else {
+            info!(
+                context,
+                "maybe_add_additional_relays: No suitable candidates"
+            );
+            return Ok(());
+        };
+        info!(context, "dbg from {candidates:?}, chose {domain}");
+
+        let param = login_param_from_domain(domain);
+        let res = crate::configure::configure(context, &param).await;
+        if res.is_err() {
+            info!(context, "dbg error {res:?}");
+            context
+                .sql
+                .execute("UPDATE relay_candidates SET last_tried=?", (now,))
+                .await?;
+        }
+
+        // TODO: Decide whether we want to immediately try again with another relay,
+        // if this one failed. If yes, remove the next line.
+        res?;
+
+        transport_added = true;
+        info!(context, "dbg success");
+    }
+    if transport_added {
+        info!(context, "dbg restarting");
+        context.restart_io_if_running().await;
+    }
+
+    Ok(())
+}
+
+pub(crate) fn login_param_from_domain(domain: &str) -> EnteredLoginParam {
+    let rng = &mut rand::rng();
+    let username = Alphanumeric.sample_string(rng, 9);
+    let addr = username + "@" + domain;
+    let addr = addr_normalize(&addr);
+    // 22 * log2(26 * 2 + 10) = 130 bits of entropy
+    let password = Alphanumeric.sample_string(rng, 22);
+
+    let param = EnteredLoginParam {
+        addr,
+        imap: EnteredImapLoginParam {
+            password,
+            ..Default::default()
+        },
+        smtp: Default::default(),
+        certificate_checks: EnteredCertificateChecks::Strict,
+    };
+    param
+}

src/config.rs

@@ -353,6 +353,13 @@ pub enum Config {
     /// Timestamp of the last `CantDecryptOutgoingMsgs` notification.
     LastCantDecryptOutgoingMsgs,
 
+    /// Timestamp of the last call to
+    /// [`crate::automatic_transport_management::maybe_add_additional_transports`]
+    LastAutomaticTransportManagement,
+
+    /// Whether to automatically add/remove transports
+    AutomaticTransportManagement,
+
     /// Whether to avoid using IMAP IDLE even if the server supports it.
     ///
     /// This is a developer option for testing "fake idle".

src/configure.rs

@@ -49,21 +49,6 @@ use crate::{chat, provider};
 /// See <https://github.com/chatmail/core/issues/7608>.
 pub(crate) const MAX_RELAYS: usize = 5;
 
-/// Hard-coded candidates for default relays.
-/// In the future, we want to use it during onboarding;
-/// note that before onboarding automatically on any of these,
-/// we need to ask the admins whether their relay is able to handle this.
-/// For now, this is just the first 6 relays from chatmail.at/relays.
-#[allow(unused)]
-const DEFAULT_RELAY_CANDIDATES: &[&str] = &[
-    "mehl.cloud",
-    "mailchat.pl",
-    "chatmail.woodpeckersnest.space",
-    "chatmail.culturanerd.it",
-    "tarpit.fun",
-    "d.gaufr.es",
-];
-
 macro_rules! progress {
     ($context:tt, $progress:expr, $comment:expr) => {
         assert!(
@@ -238,39 +223,45 @@ impl Context {
         let removed_transport_id = self
             .sql
             .transaction(|transaction| {
-                let primary_addr = transaction.query_row(
-                    "SELECT value FROM config WHERE keyname='configured_addr'",
-                    (),
-                    |row| {
-                        let addr: String = row.get(0)?;
-                        Ok(addr)
-                    },
-                )?;
+                let primary_addr = transaction
+                    .query_row(
+                        "SELECT value FROM config WHERE keyname='configured_addr'",
+                        (),
+                        |row| {
+                            let addr: String = row.get(0)?;
+                            Ok(addr)
+                        },
+                    )
+                    .context("querying addr")?;
 
                 if primary_addr == addr {
                     bail!("Cannot delete primary transport");
                 }
-                let (transport_id, add_timestamp) = transaction.query_row(
-                    "DELETE FROM transports WHERE addr=? RETURNING id, add_timestamp",
-                    (addr,),
-                    |row| {
-                        let id: u32 = row.get(0)?;
-                        let add_timestamp: i64 = row.get(1)?;
-                        Ok((id, add_timestamp))
-                    },
-                )?;
+                let (transport_id, add_timestamp) = transaction
+                    .query_row(
+                        "DELETE FROM transports WHERE addr=? RETURNING id, add_timestamp",
+                        (addr,),
+                        |row| {
+                            let id: u32 = row.get(0)?;
+                            let add_timestamp: i64 = row.get(1)?;
+                            Ok((id, add_timestamp))
+                        },
+                    )
+                    .with_context(|| format!("Deleting transport, addr: {addr}"))?;
 
                 // Removal timestamp should not be lower than addition timestamp
                 // to be accepted by other devices when synced.
                 let remove_timestamp = std::cmp::max(now, add_timestamp);
 
-                transaction.execute(
-                    "INSERT INTO removed_transports (addr, remove_timestamp)
+                transaction
+                    .execute(
+                        "INSERT INTO removed_transports (addr, remove_timestamp)
                      VALUES (?, ?)
                      ON CONFLICT (addr)
                      DO UPDATE SET remove_timestamp = excluded.remove_timestamp",
-                    (addr, remove_timestamp),
-                )?;
+                        (addr, remove_timestamp),
+                    )
+                    .context("Inserting into removed transport")?;
 
                 Ok(transport_id)
             })
@@ -604,7 +595,11 @@ async fn get_configured_param(
     Ok(configured_login_param)
 }
 
-async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Option<&'static Provider>> {
+// TODO maybe this function name should be changed
+pub(crate) async fn configure(
+    ctx: &Context,
+    param: &EnteredLoginParam,
+) -> Result<Option<&'static Provider>> {
     progress!(ctx, 1);
 
     let configured_param = get_configured_param(ctx, param).await?;

src/imap/idle.rs

@@ -8,7 +8,7 @@ use tokio::time::timeout;
 use super::Imap;
 use super::session::Session;
 use crate::context::Context;
-use crate::log::warn;
+use crate::log::{LogExt as _, warn};
 use crate::net::TIMEOUT;
 use crate::tools::{self, time_elapsed};
 
@@ -51,6 +51,13 @@ impl Session {
             return Ok(self);
         }
 
+        // we try to add additional relays right before going into IDLE mode,
+        // because apparently we are connected, but don't have anything important to do.
+        tokio::task::spawn(
+            crate::automatic_transport_management::maybe_add_additional_transports(context.clone()),
+        );
+        info!(context, "dbg spawned transports task");
+
         let mut handle = self.inner.idle();
         handle
             .init()

src/lib.rs

@@ -55,6 +55,7 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
+mod automatic_transport_management;
 pub mod blob;
 pub mod calls;
 pub mod chat;

src/mimeparser.rs

@@ -525,8 +525,6 @@ impl MimeMessage {
             // let known protected headers from the decrypted
             // part override the unencrypted top-level
 
-            // Signature was checked for original From, so we
-            // do not allow overriding it.
             let mut inner_from = None;
 
             MimeMessage::merge_headers(
@@ -558,19 +556,22 @@ impl MimeMessage {
                     // This _might_ be because the sender's mail server
                     // replaced the sending address, e.g. in a mailing list.
                     // Or it's because someone is doing some replay attack.
-                    // Resending encrypted messages via mailing lists
-                    // without reencrypting is not useful anyway,
-                    // so we return an error below.
                     warn!(
                         context,
                         "From header in encrypted part doesn't match the outer one",
                     );
 
-                    // Return an error from the parser.
-                    // This will result in creating a tombstone
-                    // and no further message processing
-                    // as if the MIME structure is broken.
-                    bail!("From header is forged");
+                    // If there are no valid signatures,
+                    // possibly because we don't have the public key,
+                    // the message will be associated with the address-contact.
+                    // If the address is possibly forged, we trash the message.
+                    if signatures.is_empty() {
+                        // Return an error from the parser.
+                        // This will result in creating a tombstone
+                        // and no further message processing
+                        // as if the MIME structure is broken.
+                        bail!("From header is forged");
+                    }
                 }
                 from = inner_from;
             }

src/qr.rs

@@ -13,6 +13,7 @@ use rand::TryRngCore as _;
 use rand::distr::{Alphanumeric, SampleString};
 use serde::Deserialize;
 
+use crate::automatic_transport_management::login_param_from_domain;
 use crate::config::Config;
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
@@ -828,20 +829,7 @@ pub(crate) async fn login_param_from_account_qr(
         .context("Invalid DCACCOUNT scheme")?;
 
     if !payload.starts_with(HTTPS_SCHEME) {
-        let rng = &mut rand::rngs::OsRng.unwrap_err();
-        let username = Alphanumeric.sample_string(rng, 9);
-        let addr = username + "@" + payload;
-        let password = Alphanumeric.sample_string(rng, 50);
-
-        let param = EnteredLoginParam {
-            addr,
-            imap: EnteredImapLoginParam {
-                password,
-                ..Default::default()
-            },
-            smtp: Default::default(),
-            certificate_checks: EnteredCertificateChecks::Strict,
-        };
+        let param = login_param_from_domain(payload);
         return Ok(param);
     }
 

src/receive_imf/receive_imf_tests.rs

@@ -3456,6 +3456,8 @@ async fn test_prefer_encrypt_mutual_if_encrypted() -> Result<()> {
     Ok(())
 }
 
+/// Tests reception of encrypted and signed message with forged From header
+/// when the signature cannot be checked because the public key is not available.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_forged_from_and_no_valid_signatures() -> Result<()> {
     let mut tcm = TestContextManager::new();
@@ -4718,6 +4720,10 @@ async fn test_no_op_member_added_is_trash() -> Result<()> {
     Ok(())
 }
 
+/// Tests reception of a message with a valid signature and forged From header.
+///
+/// The message is accepted because the sender contact is associated with the key
+/// rather than the address.
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_forged_from() -> Result<()> {
     let mut tcm = TestContextManager::new();
@@ -4732,8 +4738,16 @@ async fn test_forged_from() -> Result<()> {
         .payload
         .replace("bob@example.net", "notbob@example.net");
 
-    let msg = alice.recv_msg_opt(&sent_msg).await;
-    assert!(msg.is_none());
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.text, "hi!");
+    assert!(msg.get_showpadlock());
+    let contact = Contact::get_by_id(&alice, msg.from_id).await?;
+    assert!(contact.is_key_contact());
+
+    // We take the address from the encrypted part
+    // and send replies there.
+    assert_eq!(contact.get_addr(), "bob@example.net");
+
     Ok(())
 }
 

src/sql/migrations.rs

@@ -2522,6 +2522,40 @@ UPDATE msgs SET state=24 WHERE state=18; -- Change OutPreparing to OutFailed.
         .await?;
     }
 
+    // TODO should we call it relay_candidates or transport_candidates?
+    inc_and_check(&mut migration_version, 159)?;
+    if dbversion < migration_version {
+        // TODO put a better list here
+        const DEFAULT_RELAY_CANDIDATES: &[&str] = &[
+            "mehl.cloud",
+            "mailchat.pl",
+            "chatmail.woodpeckersnest.space",
+            "chatmail.culturanerd.it",
+            "tarpit.fun",
+            "d.gaufr.es",
+        ];
+
+        sql.execute_migration_transaction(
+            |transaction| {
+                transaction.execute(
+                    "CREATE TABLE relay_candidates(
+                        domain TEXT PRIMARY KEY NOT NULL,
+                        last_tried TEXT NOT NULL DEFAULT 0
+                    ) STRICT",
+                    (),
+                )?;
+                let mut statement =
+                    transaction.prepare("INSERT INTO relay_candidates(domain) VALUES (?)")?;
+                for domain in DEFAULT_RELAY_CANDIDATES {
+                    statement.execute((domain,))?;
+                }
+                Ok(())
+            },
+            migration_version,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?

src/transport.rs

@@ -831,7 +831,9 @@ pub(crate) async fn sync_transports(
 
 /// Same as `context.restart_io_if_running()`, but `Box::pin`ed and with a `+ Send` bound,
 /// so that it can be called recursively.
-fn restart_io_if_running_boxed(context: Context) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+pub(crate) fn restart_io_if_running_boxed(
+    context: Context,
+) -> Pin<Box<dyn Future<Output = ()> + Send>> {
     Box::pin(async move { context.restart_io_if_running().await })
 }