From 373f1840a6157477ca9bde85280ff01daf612cd3 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 9 Sep 2026 09:32:23 +0000 Subject: [PATCH] feat: queue messages for SMTP before encryption Headers like From and Autocrypt are now added late, right before sending the message over SMTP. This way we advertise the latest list of transports and use the correct From address in the encrypted part even for messages queued while being offline. BCC-self recipients are also added late. For unencrypted messages we only want to send a copy to the sending address, but we don't know the sending address when queueing the message. Adding bcc-self recipients when dequeuing the message also makes it possible to send copies to updated list of relays. --- .../tests/test_multitransport.py | 19 + docs/schema.sql | 63 +++- src/chat.rs | 337 +++++++++--------- src/chat/chat_tests.rs | 5 +- src/config.rs | 7 - src/download.rs | 3 - src/ephemeral/ephemeral_tests.rs | 2 +- src/keyupdate.rs | 10 +- src/location.rs | 16 - src/message.rs | 2 +- src/message/message_tests.rs | 2 +- src/mimefactory.rs | 162 +++++---- src/mimefactory/mimefactory_tests.rs | 4 +- src/receive_imf.rs | 12 +- src/receive_imf/receive_imf_tests.rs | 35 +- src/securejoin.rs | 8 +- src/securejoin/bob.rs | 7 +- src/smtp.rs | 234 ++++++++++-- src/sql/migrations.rs | 31 ++ src/test_utils.rs | 96 +++-- src/tests/pre_messages/sending.rs | 4 +- 21 files changed, 699 insertions(+), 360 deletions(-) diff --git a/deltachat-rpc-client/tests/test_multitransport.py b/deltachat-rpc-client/tests/test_multitransport.py index 6a014ba18..e0e04eb2b 100644 --- a/deltachat-rpc-client/tests/test_multitransport.py +++ b/deltachat-rpc-client/tests/test_multitransport.py @@ -98,6 +98,25 @@ def test_change_address(acf) -> None: assert sender_addr2 == new_alice_addr +def test_remove_transport_keep_messages(acf) -> None: + """Test that deleting current sending transport keeps queued messages.""" + alice, bob = acf.get_online_accounts(2) + + qr = acf.get_account_qr() + alice.add_transport_from_qr(qr) + + alice.stop_io() + alice_chat_bob = alice.create_chat(bob) + alice_chat_bob.send_text("Hello!") + + new_alice_addr = alice.list_transports()[1]["addr"] + alice.delete_transport(alice.list_transports()[0]["addr"]) + alice.start_io() + + bob_msg = bob.wait_for_incoming_msg().get_snapshot() + assert bob_msg.sender.get_snapshot().address == new_alice_addr + + def test_download_on_demand(acf, rpcdata) -> None: alice, bob = acf.get_online_accounts(2) alice.set_config("download_limit", "1") diff --git a/docs/schema.sql b/docs/schema.sql index a9a743d1f..ac042ec85 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -400,14 +400,51 @@ CREATE TABLE bobstate ( chat_id INTEGER NOT NULL ); -CREATE TABLE smtp ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - rfc724_mid TEXT NOT NULL, -- Message-ID - mime TEXT NOT NULL, -- SMTP payload - msg_id INTEGER NOT NULL, -- ID of the message in `msgs` table - recipients TEXT NOT NULL, -- List of recipients separated by space - retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message -); +CREATE TABLE smtp2 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + display_name TEXT NOT NULL, -- Display name to put into the From field. + rfc724_mid TEXT NOT NULL, -- Message-ID + + -- Unencrypted payload with some headers. + mime BLOB NOT NULL, + + -- True if Autocrypt header should be added before sending. + should_attach_pubkey INTEGER NOT NULL, + + -- True if OpenPGP-encrypted message may use compression. + should_compress INTEGER NOT NULL, + + -- True if encrypted message should be signed. + should_sign INTEGER NOT NULL, + + -- ID of the message in `msgs` table + msg_id INTEGER NOT NULL, + + -- Space-separated recipient addresses. + recipients TEXT NOT NULL, + + -- Space-separated addresses the message was sent to. + sent_to TEXT NOT NULL DEFAULT '', + + -- If true, copy should be sent to self in addition to the recipient list. + -- + -- For encrypted messages copy is sent to all addresses. + -- For unencrypted messages, copy is sent to the From address only. + bcc_self INTEGER NOT NULL, + + -- True if the message is encrypted. + -- If true, at most one of the shared_secret or encryption_fingerprints should be non-empty. + -- If false, both must be empty. + is_encrypted INTEGER NOT NULL, + + -- Shared secret if the message is to be encrypted symmetrically. + shared_secret TEXT NOT NULL DEFAULT '', + + -- Space-separated fingerprints of the keys the message should be encrypted to. + encryption_fingerprints TEXT NOT NULL DEFAULT '', + + retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message +) STRICT; CREATE TABLE smtp_mdns ( msg_id INTEGER NOT NULL, -- id of the message in msgs table which requested MDN (DEPRECATED 2024-06-21) @@ -781,3 +818,13 @@ CREATE TABLE sending_domains( domain TEXT PRIMARY KEY, dkim_works INTEGER DEFAULT 0 ); + +-- Replaced with smtp2. +CREATE TABLE smtp ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + rfc724_mid TEXT NOT NULL, -- Message-ID + mime TEXT NOT NULL, -- SMTP payload + msg_id INTEGER NOT NULL, -- ID of the message in `msgs` table + recipients TEXT NOT NULL, -- List of recipients separated by space + retries INTEGER NOT NULL DEFAULT 0 -- Number of failed attempts to send the message +); diff --git a/src/chat.rs b/src/chat.rs index d57065209..6b6859369 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -11,7 +11,6 @@ use std::time::Duration; use anyhow::{Context as _, Result, anyhow, bail, ensure}; use chrono::TimeZone; use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line}; -use humansize::{BINARY, format_size}; use mail_builder::mime::MimePart; use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; @@ -27,26 +26,22 @@ use crate::constants::{ use crate::contact::{self, Contact, ContactId, Origin}; use crate::context::Context; use crate::debug_logging::maybe_set_logging_xdc; -use crate::download::{ - DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PRE_MSG_SIZE_WARNING_THRESHOLD, -}; +use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD}; use crate::ensure_and_debug_assert_eq; use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers}; use crate::events::EventType; -use crate::key; -use crate::key::{Fingerprint, self_fingerprint}; -use crate::location; +use crate::key::{DcKey as _, Fingerprint, self_fingerprint}; use crate::log::{LogExt, warn}; use crate::logged_debug_assert; use crate::message::{self, Message, MessageState, MsgId, Viewtype}; use crate::mimefactory; -use crate::mimefactory::{MimeFactory, RenderedEmail}; +use crate::mimefactory::{MimeFactory, QueueSideEffects, QueuedMail, ToBeQueuedMail}; use crate::mimeparser::SystemMessage; use crate::param::{Param, Params}; use crate::pgp::addresses_from_public_key; use crate::reaction::broadcast_reactions; use crate::receive_imf::ReceivedMsg; -use crate::smtp::{self, send_msg_to_smtp}; +use crate::smtp::send_msg_to_smtp; use crate::stock_str; use crate::sync::{self, Sync::*, SyncData}; use crate::tools::{ @@ -336,16 +331,18 @@ impl ChatId { Ok(chat_id) } - async fn set_selfavatar_timestamp(self, context: &Context, timestamp: i64) -> Result<()> { - context - .sql + fn set_selfavatar_timestamp( + self, + transaction: &mut rusqlite::Transaction<'_>, + timestamp: i64, + ) -> Result<()> { + transaction .execute( "UPDATE contacts SET selfavatar_sent=? WHERE id IN(SELECT contact_id FROM chats_contacts WHERE chat_id=? AND add_timestamp >= remove_timestamp)", (timestamp, self), - ) - .await?; + ) ?; Ok(()) } @@ -2776,11 +2773,8 @@ async fn render_mime_message_and_pre_message( context: &Context, msg: &mut Message, mimefactory: MimeFactory, -) -> Result<(Option, RenderedEmail)> { - let from_addr = context.get_primary_self_addr().await?; - let public_key = key::load_self_public_key(context).await?; - let secret_key = key::load_self_secret_key(context).await?; - + bcc_self: bool, +) -> Result<(Option, ToBeQueuedMail)> { let needs_pre_message = msg.viewtype.has_file() && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages && msg @@ -2797,54 +2791,121 @@ async fn render_mime_message_and_pre_message( let mut mimefactory_post_msg = mimefactory.clone(); mimefactory_post_msg.set_as_post_message(); - let (queued_msg, side_effects) = Box::pin(mimefactory_post_msg.into_queued_mail(context)) - .await - .context("Failed to render post-message")?; - - let rendered_msg = mimefactory::render_queued_mail( - queued_msg, - &public_key, - &secret_key, - from_addr.clone(), - side_effects, - )?; + let (queued_msg, side_effects) = + Box::pin(mimefactory_post_msg.into_queued_mail(context, bcc_self)) + .await + .context("Failed to render post-message")?; let mut mimefactory_pre_msg = mimefactory; - mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg); + mimefactory_pre_msg.set_as_pre_message_for(&queued_msg.rfc724_mid); let (queued_pre_msg, pre_side_effects) = - Box::pin(mimefactory_pre_msg.into_queued_mail(context)) + Box::pin(mimefactory_pre_msg.into_queued_mail(context, bcc_self)) .await .context("pre-message failed to render")?; - let rendered_pre_msg = mimefactory::render_queued_mail( - queued_pre_msg, - &public_key, - &secret_key, - from_addr, - pre_side_effects, - )?; - if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD { - warn!( - context, - "Pre-message for message {} is larger than expected: {}.", - msg.id, - rendered_pre_msg.message.len() - ); + Ok(( + Some((queued_pre_msg, pre_side_effects)), + (queued_msg, side_effects), + )) + } else { + let (queued_msg, side_effects) = + Box::pin(mimefactory.into_queued_mail(context, bcc_self)).await?; + + Ok((None, (queued_msg, side_effects))) + } +} + +/// Process side effects and store queued mail. +pub(crate) fn enqueue_mail( + transaction: &mut rusqlite::Transaction<'_>, + now: i64, + msg_id: MsgId, + queued_mail: &QueuedMail, + side_effects: Option<&QueueSideEffects>, +) -> Result { + if let Some(side_effects) = side_effects { + if let Some(last_added_location_timestamp) = side_effects.last_added_location_timestamp { + transaction.execute( + "UPDATE chats SET locations_last_sent=? WHERE id=?;", + (last_added_location_timestamp, side_effects.chat_id), + )?; } - Ok((Some(rendered_pre_msg), rendered_msg)) - } else { - let (queued_msg, side_effects) = Box::pin(mimefactory.into_queued_mail(context)).await?; - let rendered_msg = mimefactory::render_queued_mail( - queued_msg, - &public_key, - &secret_key, - from_addr, - side_effects, - )?; + if side_effects.avatar_is_attached { + side_effects + .chat_id + .set_selfavatar_timestamp(transaction, now) + .context("Failed to set selfavatar timestamp")?; + } - Ok((None, rendered_msg)) + if let Some(ref sync_ids) = side_effects.sync_ids_to_delete { + transaction.execute( + &format!("DELETE FROM multi_device_sync WHERE id IN ({sync_ids})"), + (), + )?; + } } + + // Store mail into queue. + let all_recipients = queued_mail.recipients.join(" "); + let is_encrypted = queued_mail.encryption.is_encrypted(); + + transaction + .execute( + " + INSERT INTO smtp2 ( + display_name, + rfc724_mid, + mime, + should_attach_pubkey, + should_compress, + should_sign, + msg_id, + recipients, + bcc_self, + is_encrypted, + shared_secret, + encryption_fingerprints + ) + VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + ", + ( + &queued_mail.display_name, + &queued_mail.rfc724_mid, + &queued_mail.raw_message, + queued_mail.should_attach_pubkey, + queued_mail.should_compress, + queued_mail.should_sign, + msg_id, + &all_recipients, + queued_mail.bcc_self, + is_encrypted, + if let mimefactory::QueuedEncryption::Symmetric { ref shared_secret } = + queued_mail.encryption + { + shared_secret + } else { + "" + }, + if let mimefactory::QueuedEncryption::Asymmetric { + ref encryption_pubkeys, + } = queued_mail.encryption + { + let res: Vec = encryption_pubkeys + .iter() + .map(|pubkey| pubkey.dc_fingerprint().hex()) + .collect(); + res.join(" ") + } else { + "".to_string() + }, + ), + ) + .context("Failed to insert a row into smtp2 table")?; + let row_id = transaction.last_insert_rowid(); + Ok(row_id) } /// Constructs jobs for sending a message and inserts them into the `smtp` table. @@ -2857,6 +2918,8 @@ async fn render_mime_message_and_pre_message( /// /// The caller has to interrupt SMTP loop or otherwise process new rows. async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result> { + let now = time(); + let cmd = msg.param.get_cmd(); if cmd == SystemMessage::GroupNameChanged || cmd == SystemMessage::GroupDescriptionChanged { msg.chat_id @@ -2888,12 +2951,14 @@ async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -> Result