refactor(mimefactory): separate rendering of message payload and sendable message

This change separates rendering into two separate steps:
1. Rendering of the message payload without the From, Date and Autocrypt headers.
2. Adding the From, Date and Autocrypt headers and possibly encrypting the message.

The goal is to have serializable result of the first step
that can be persisted in the database and sent later with any email address.
This way it will be possible to send queued messages over any relay.

This will make it possible not to remove all messages from the queue
when the sending relay is changed.
Currently changing `configured_addr` deletes everything from `smtp` table.

This change is however only a refactoring and does not implement any features.

src/chat.rs

@@ -35,11 +35,13 @@ use crate::download::{
 use crate::ensure_and_debug_assert_eq;
 use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers};
 use crate::events::EventType;
+use crate::key;
 use crate::key::{Fingerprint, self_fingerprint};
 use crate::location;
 use crate::log::{LogExt, warn};
 use crate::logged_debug_assert;
 use crate::message::{self, Message, MessageState, MsgId, Viewtype};
+use crate::mimefactory;
 use crate::mimefactory::{MimeFactory, RenderedEmail};
 use crate::mimeparser::SystemMessage;
 use crate::param::{Param, Params};
@@ -2766,6 +2768,10 @@ async fn render_mime_message_and_pre_message(
     msg: &mut Message,
     mimefactory: MimeFactory,
 ) -> Result<(Option<RenderedEmail>, RenderedEmail)> {
+    let from_addr = context.get_primary_self_addr().await?;
+    let public_key = key::load_self_public_key(context).await?;
+    let secret_key = key::load_self_secret_key(context).await?;
+
     let needs_pre_message = msg.viewtype.has_file()
         && mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages
         && msg
@@ -2782,15 +2788,31 @@ async fn render_mime_message_and_pre_message(
 
         let mut mimefactory_post_msg = mimefactory.clone();
         mimefactory_post_msg.set_as_post_message();
-        let rendered_msg = Box::pin(mimefactory_post_msg.render(context))
+        let (queued_msg, side_effects) = Box::pin(mimefactory_post_msg.into_queued_mail(context))
             .await
             .context("Failed to render post-message")?;
 
+        let rendered_msg = mimefactory::render_queued_mail(
+            queued_msg,
+            &public_key,
+            &secret_key,
+            from_addr.clone(),
+            side_effects,
+        )?;
+
         let mut mimefactory_pre_msg = mimefactory;
         mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
-        let rendered_pre_msg = Box::pin(mimefactory_pre_msg.render(context))
-            .await
-            .context("pre-message failed to render")?;
+        let (queued_pre_msg, pre_side_effects) =
+            Box::pin(mimefactory_pre_msg.into_queued_mail(context))
+                .await
+                .context("pre-message failed to render")?;
+        let rendered_pre_msg = mimefactory::render_queued_mail(
+            queued_pre_msg,
+            &public_key,
+            &secret_key,
+            from_addr,
+            pre_side_effects,
+        )?;
 
         if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD {
             warn!(
@@ -2803,7 +2825,16 @@ async fn render_mime_message_and_pre_message(
 
         Ok((Some(rendered_pre_msg), rendered_msg))
     } else {
-        Ok((None, Box::pin(mimefactory.render(context)).await?))
+        let (queued_msg, side_effects) = Box::pin(mimefactory.into_queued_mail(context)).await?;
+        let rendered_msg = mimefactory::render_queued_mail(
+            queued_msg,
+            &public_key,
+            &secret_key,
+            from_addr,
+            side_effects,
+        )?;
+
+        Ok((None, rendered_msg))
     }
 }
 
@@ -2848,7 +2879,6 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             return Err(err);
         }
     };
-    let attach_selfavatar = mimefactory.attach_selfavatar;
     let mut recipients = mimefactory.recipients();
 
     // Default Webxdc integrations are hidden messages and must not be sent out:
@@ -2930,14 +2960,22 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
 
     let now = time();
 
-    if let Some(last_added_location_timestamp) = rendered_msg.last_added_location_timestamp {
+    if let Some(last_added_location_timestamp) =
+        rendered_msg.side_effects.last_added_location_timestamp
+    {
         location::set_kml_sent_timestamp(context, msg.chat_id, last_added_location_timestamp)
             .await?;
     }
 
-    if attach_selfavatar && let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, now).await
+    if rendered_msg.side_effects.avatar_is_attached
+        || rendered_pre_msg
+            .as_ref()
+            .is_some_and(|msg| msg.side_effects.avatar_is_attached)
     {
-        error!(context, "Failed to set selfavatar timestamp: {err:#}.");
+        msg.chat_id
+            .set_selfavatar_timestamp(context, now)
+            .await
+            .context("Failed to set selfavatar timestamp")?;
     }
 
     if rendered_msg.is_encrypted {
@@ -2945,7 +2983,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
     } else {
         msg.param.remove(Param::GuaranteeE2ee);
     }
-    msg.subject.clone_from(&rendered_msg.subject);
+    msg.subject.clone_from(&rendered_msg.side_effects.subject);
     // Sort the message to the bottom. Employ `msgs_index7` to compute `timestamp`.
     context
         .sql
@@ -2977,7 +3015,7 @@ WHERE id=?
     let trans_fn = |t: &mut rusqlite::Transaction| {
         let mut row_ids = Vec::<i64>::new();
 
-        if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
+        if let Some(sync_ids) = rendered_msg.side_effects.sync_ids_to_delete {
             t.execute(
                 &format!("DELETE FROM multi_device_sync WHERE id IN ({sync_ids})"),
                 (),

src/download.rs

@@ -163,7 +163,9 @@ pub(crate) async fn download_msg(
         delete_from_available_post_msgs(context, &rfc724_mid).await?;
         return Ok(None);
     };
-    Box::pin(session.fetch_single_msg(context, &server_folder, server_uid, rfc724_mid)).await?;
+    session
+        .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid)
+        .await?;
 
     let bcc_self = context.get_config_bool(Config::BccSelf).await?;
     if ephemeral::should_delete_all_downloaded_messages(bcc_self, session.is_chatmail()) {
@@ -204,7 +206,7 @@ impl Session {
         let (sender, receiver) = async_channel::unbounded();
         {
             let _fetch_msgs_lock_guard = context.fetch_msgs_mutex.lock().await;
-            self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender)
+            Box::pin(self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender))
                 .await?;
         }
         if receiver.recv().await.is_err() {

src/e2ee.rs

@@ -1,105 +1,9 @@
 //! End-to-end encryption support.
 
-use std::io::Cursor;
-
-use anyhow::Result;
-use mail_builder::mime::MimePart;
-
-use crate::aheader::{Aheader, EncryptPreference};
-use crate::context::Context;
-use crate::key::{SignedPublicKey, load_self_public_key, load_self_secret_key};
-use crate::pgp::{self, SeipdVersion};
-
-#[derive(Debug)]
-pub struct EncryptHelper {
-    pub addr: String,
-    pub public_key: SignedPublicKey,
-}
-
-impl EncryptHelper {
-    pub async fn new(context: &Context) -> Result<EncryptHelper> {
-        let addr = context.get_primary_self_addr().await?;
-        let public_key = load_self_public_key(context).await?;
-
-        Ok(EncryptHelper { addr, public_key })
-    }
-
-    pub fn get_aheader(&self) -> Aheader {
-        Aheader {
-            addr: self.addr.clone(),
-            public_key: self.public_key.clone(),
-            prefer_encrypt: EncryptPreference::Mutual,
-            verified: false,
-        }
-    }
-
-    /// Tries to encrypt the passed in `mail`.
-    pub async fn encrypt(
-        self,
-        context: &Context,
-        keyring: Vec<SignedPublicKey>,
-        mail_to_encrypt: MimePart<'static>,
-        compress: bool,
-        seipd_version: SeipdVersion,
-    ) -> Result<String> {
-        let mut raw_message = Vec::new();
-        let cursor = Cursor::new(&mut raw_message);
-        mail_to_encrypt.clone().write_part(cursor).ok();
-
-        let ctext = self
-            .encrypt_raw(context, keyring, raw_message, compress, seipd_version)
-            .await?;
-        Ok(ctext)
-    }
-
-    pub async fn encrypt_raw(
-        self,
-        context: &Context,
-        keyring: Vec<SignedPublicKey>,
-        raw_message: Vec<u8>,
-        compress: bool,
-        seipd_version: SeipdVersion,
-    ) -> Result<String> {
-        let sign_key = load_self_secret_key(context).await?;
-        let ctext =
-            pgp::pk_encrypt(raw_message, keyring, sign_key, compress, seipd_version).await?;
-
-        Ok(ctext)
-    }
-
-    /// Symmetrically encrypt the message. This is used for broadcast channels.
-    /// `shared secret` is the secret that will be used for symmetric encryption.
-    pub async fn encrypt_symmetrically(
-        self,
-        context: &Context,
-        shared_secret: &str,
-        mail_to_encrypt: MimePart<'static>,
-        compress: bool,
-        sign: bool,
-    ) -> Result<String> {
-        let sign_key = if sign {
-            Some(load_self_secret_key(context).await?)
-        } else {
-            None
-        };
-
-        let shared_secret = shared_secret.to_string();
-        let mut raw_message = Vec::new();
-        let cursor = Cursor::new(&mut raw_message);
-        mail_to_encrypt.clone().write_part(cursor).ok();
-
-        let ctext = tokio::task::spawn_blocking(move || {
-            pgp::symm_encrypt_message(raw_message, sign_key, shared_secret, compress)
-        })
-        .await??;
-
-        Ok(ctext)
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use anyhow::Result;
+
     use crate::chat;
     use crate::chat::send_text_msg;
     use crate::config::Config;

src/pgp.rs

@@ -21,7 +21,6 @@ use pgp::types::{
 };
 use rand_old::{Rng as _, thread_rng};
 use sha2::Sha256;
-use tokio::runtime::Handle;
 
 use crate::configure::MAX_RELAYS;
 use crate::key::{DcKey, Fingerprint};
@@ -110,97 +109,95 @@ pub enum SeipdVersion {
 /// Encrypts `plain` text using `public_keys_for_encryption`
 /// and signs it using `private_key_for_signing`.
 #[expect(clippy::arithmetic_side_effects)]
-pub async fn pk_encrypt(
+pub fn pk_encrypt(
     plain: Vec<u8>,
     public_keys_for_encryption: Vec<SignedPublicKey>,
     private_key_for_signing: SignedSecretKey,
     compress: bool,
     seipd_version: SeipdVersion,
 ) -> Result<String> {
-    Handle::current()
-        .spawn_blocking(move || {
-            let mut rng = thread_rng();
+    tokio::task::block_in_place(|| {
+        let mut rng = thread_rng();
 
-            let pkeys = public_keys_for_encryption
-                .iter()
-                .filter_map(select_pk_for_encryption);
-            let subpkts = {
-                let mut hashed = Vec::with_capacity(1 + public_keys_for_encryption.len() + 1);
-                hashed.push(Subpacket::critical(SubpacketData::SignatureCreationTime(
-                    pgp::types::Timestamp::now(),
+        let pkeys = public_keys_for_encryption
+            .iter()
+            .filter_map(select_pk_for_encryption);
+        let subpkts = {
+            let mut hashed = Vec::with_capacity(1 + public_keys_for_encryption.len() + 1);
+            hashed.push(Subpacket::critical(SubpacketData::SignatureCreationTime(
+                pgp::types::Timestamp::now(),
+            ))?);
+            for key in &public_keys_for_encryption {
+                let data = SubpacketData::IntendedRecipientFingerprint(key.fingerprint());
+                let subpkt = match private_key_for_signing.version() < KeyVersion::V6 {
+                    true => Subpacket::regular(data)?,
+                    false => Subpacket::critical(data)?,
+                };
+                hashed.push(subpkt);
+            }
+            hashed.push(Subpacket::regular(SubpacketData::IssuerFingerprint(
+                private_key_for_signing.fingerprint(),
+            ))?);
+            let mut unhashed = vec![];
+            if private_key_for_signing.version() <= KeyVersion::V4 {
+                unhashed.push(Subpacket::regular(SubpacketData::IssuerKeyId(
+                    private_key_for_signing.legacy_key_id(),
                 ))?);
-                for key in &public_keys_for_encryption {
-                    let data = SubpacketData::IntendedRecipientFingerprint(key.fingerprint());
-                    let subpkt = match private_key_for_signing.version() < KeyVersion::V6 {
-                        true => Subpacket::regular(data)?,
-                        false => Subpacket::critical(data)?,
-                    };
-                    hashed.push(subpkt);
+            }
+            SubpacketConfig::UserDefined { hashed, unhashed }
+        };
+
+        let msg = MessageBuilder::from_bytes("", plain);
+        let encoded_msg = match seipd_version {
+            SeipdVersion::V1 => {
+                let mut msg = msg.seipd_v1(&mut rng, SYMMETRIC_KEY_ALGORITHM);
+
+                for pkey in pkeys {
+                    msg.encrypt_to_key_anonymous(&mut rng, &pkey)?;
                 }
-                hashed.push(Subpacket::regular(SubpacketData::IssuerFingerprint(
-                    private_key_for_signing.fingerprint(),
-                ))?);
-                let mut unhashed = vec![];
-                if private_key_for_signing.version() <= KeyVersion::V4 {
-                    unhashed.push(Subpacket::regular(SubpacketData::IssuerKeyId(
-                        private_key_for_signing.legacy_key_id(),
-                    ))?);
+
+                let hash_algorithm = private_key_for_signing.hash_alg();
+                msg.sign_with_subpackets(
+                    &*private_key_for_signing,
+                    Password::empty(),
+                    hash_algorithm,
+                    subpkts,
+                );
+                if compress {
+                    msg.compression(CompressionAlgorithm::ZLIB);
                 }
-                SubpacketConfig::UserDefined { hashed, unhashed }
-            };
 
-            let msg = MessageBuilder::from_bytes("", plain);
-            let encoded_msg = match seipd_version {
-                SeipdVersion::V1 => {
-                    let mut msg = msg.seipd_v1(&mut rng, SYMMETRIC_KEY_ALGORITHM);
+                msg.to_armored_string(&mut rng, Default::default())?
+            }
+            SeipdVersion::V2 => {
+                let mut msg = msg.seipd_v2(
+                    &mut rng,
+                    SYMMETRIC_KEY_ALGORITHM,
+                    AeadAlgorithm::Ocb,
+                    ChunkSize::C8KiB,
+                );
 
-                    for pkey in pkeys {
-                        msg.encrypt_to_key_anonymous(&mut rng, &pkey)?;
-                    }
-
-                    let hash_algorithm = private_key_for_signing.hash_alg();
-                    msg.sign_with_subpackets(
-                        &*private_key_for_signing,
-                        Password::empty(),
-                        hash_algorithm,
-                        subpkts,
-                    );
-                    if compress {
-                        msg.compression(CompressionAlgorithm::ZLIB);
-                    }
-
-                    msg.to_armored_string(&mut rng, Default::default())?
+                for pkey in pkeys {
+                    msg.encrypt_to_key_anonymous(&mut rng, &pkey)?;
                 }
-                SeipdVersion::V2 => {
-                    let mut msg = msg.seipd_v2(
-                        &mut rng,
-                        SYMMETRIC_KEY_ALGORITHM,
-                        AeadAlgorithm::Ocb,
-                        ChunkSize::C8KiB,
-                    );
 
-                    for pkey in pkeys {
-                        msg.encrypt_to_key_anonymous(&mut rng, &pkey)?;
-                    }
-
-                    let hash_algorithm = private_key_for_signing.hash_alg();
-                    msg.sign_with_subpackets(
-                        &*private_key_for_signing,
-                        Password::empty(),
-                        hash_algorithm,
-                        subpkts,
-                    );
-                    if compress {
-                        msg.compression(CompressionAlgorithm::ZLIB);
-                    }
-
-                    msg.to_armored_string(&mut rng, Default::default())?
+                let hash_algorithm = private_key_for_signing.hash_alg();
+                msg.sign_with_subpackets(
+                    &*private_key_for_signing,
+                    Password::empty(),
+                    hash_algorithm,
+                    subpkts,
+                );
+                if compress {
+                    msg.compression(CompressionAlgorithm::ZLIB);
                 }
-            };
 
-            Ok(encoded_msg)
-        })
-        .await?
+                msg.to_armored_string(&mut rng, Default::default())?
+            }
+        };
+
+        Ok(encoded_msg)
+    })
 }
 
 /// Returns fingerprints
@@ -261,35 +258,37 @@ pub fn symm_encrypt_message(
     shared_secret: String,
     compress: bool,
 ) -> Result<String> {
-    let shared_secret = Password::from(shared_secret);
+    tokio::task::block_in_place(|| {
+        let shared_secret = Password::from(shared_secret);
 
-    let msg = MessageBuilder::from_bytes("", plain);
-    let mut rng = thread_rng();
-    let mut salt = [0u8; 8];
-    rng.fill(&mut salt[..]);
-    let s2k = StringToKey::Salted {
-        hash_alg: HashAlgorithm::default(),
-        salt,
-    };
-    let mut msg = msg.seipd_v2(
-        &mut rng,
-        SYMMETRIC_KEY_ALGORITHM,
-        AeadAlgorithm::Ocb,
-        ChunkSize::C8KiB,
-    );
-    msg.encrypt_with_password(&mut rng, s2k, &shared_secret)?;
+        let msg = MessageBuilder::from_bytes("", plain);
+        let mut rng = thread_rng();
+        let mut salt = [0u8; 8];
+        rng.fill(&mut salt[..]);
+        let s2k = StringToKey::Salted {
+            hash_alg: HashAlgorithm::default(),
+            salt,
+        };
+        let mut msg = msg.seipd_v2(
+            &mut rng,
+            SYMMETRIC_KEY_ALGORITHM,
+            AeadAlgorithm::Ocb,
+            ChunkSize::C8KiB,
+        );
+        msg.encrypt_with_password(&mut rng, s2k, &shared_secret)?;
 
-    if let Some(private_key_for_signing) = private_key_for_signing.as_deref() {
-        let hash_algorithm = private_key_for_signing.hash_alg();
-        msg.sign(private_key_for_signing, Password::empty(), hash_algorithm);
-    }
-    if compress {
-        msg.compression(CompressionAlgorithm::ZLIB);
-    }
+        if let Some(private_key_for_signing) = private_key_for_signing.as_deref() {
+            let hash_algorithm = private_key_for_signing.hash_alg();
+            msg.sign(private_key_for_signing, Password::empty(), hash_algorithm);
+        }
+        if compress {
+            msg.compression(CompressionAlgorithm::ZLIB);
+        }
 
-    let encoded_msg = msg.to_armored_string(&mut rng, Default::default())?;
+        let encoded_msg = msg.to_armored_string(&mut rng, Default::default())?;
 
-    Ok(encoded_msg)
+        Ok(encoded_msg)
+    })
 }
 
 /// Merges and minimizes OpenPGP certificates.
@@ -492,7 +491,7 @@ mod tests {
         config::Config,
         decrypt,
         key::{load_self_public_key, self_fingerprint, store_self_keypair},
-        mimefactory::{render_outer_message, wrap_encrypted_part},
+        mimefactory::{part_to_bytes, wrap_encrypted_part},
         test_utils::{TestContext, TestContextManager, alice_keypair, bob_keypair},
         token,
     };
@@ -518,8 +517,8 @@ mod tests {
         store_self_keypair(t, secret_key).await?;
 
         let mime_message = wrap_encrypted_part(bytes.try_into().unwrap());
-        let rendered = render_outer_message(vec![], mime_message);
-        let parsed = mailparse::parse_mail(rendered.as_bytes())?;
+        let rendered = part_to_bytes(mime_message);
+        let parsed = mailparse::parse_mail(&rendered)?;
         let (decrypted, _fp) = decrypt::decrypt(t, &parsed).await?.unwrap();
         Ok(decrypted)
     }
@@ -592,7 +591,6 @@ mod tests {
                     compress,
                     SeipdVersion::V2,
                 )
-                .await
                 .unwrap()
             })
             .await
@@ -783,8 +781,7 @@ mod tests {
             KEYS.alice_secret.clone(),
             compress,
             SeipdVersion::V2,
-        )
-        .await?;
+        )?;
 
         // Trying to decrypt it should fail with an OK error message:
         let bob_private_keyring = crate::key::load_self_secret_keyring(bob).await?;

src/reaction.rs

@@ -1170,8 +1170,7 @@ Content-Transfer-Encoding: base64\r
             alice_secret_key,
             compress,
             SeipdVersion::V2,
-        )
-        .await?;
+        )?;
 
         let boundary = "boundary123";
         let rcvd_mail = format!(

src/test_utils.rs

@@ -22,6 +22,7 @@ use tokio::runtime::Handle;
 use tokio::{fs, task};
 use uuid::Uuid;
 
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::chat::{
     self, Chat, ChatId, ChatIdBlocked, MessageListOptions, add_to_chat_contacts_table, create_group,
 };
@@ -33,7 +34,6 @@ use crate::contact::{
     Contact, ContactId, Modifier, Origin, import_vcard, make_vcard, mark_contact_id_as_verified,
 };
 use crate::context::Context;
-use crate::e2ee::EncryptHelper;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{self, DcKey, self_fingerprint};
 use crate::log::warn;
@@ -1240,8 +1240,15 @@ pub async fn encrypt_raw_message(
     receivers: &[&TestContext],
     payload: &[u8],
 ) -> Result<String> {
-    let encryption_helper = EncryptHelper::new(context).await?;
-    let mut encryption_keyring = vec![encryption_helper.public_key.clone()];
+    let public_key = key::load_self_public_key(context).await?;
+    let aheader = Aheader {
+        addr: context.get_primary_self_addr().await?,
+        public_key: public_key.clone(),
+        prefer_encrypt: EncryptPreference::Mutual,
+        verified: false,
+    };
+
+    let mut encryption_keyring = vec![public_key.clone()];
 
     for receiver in receivers {
         encryption_keyring.push(key::load_self_public_key(receiver).await?);
@@ -1250,18 +1257,17 @@ pub async fn encrypt_raw_message(
     let from = context.get_primary_self_addr().await?;
     let compress = false;
 
-    let mut cleartext = format!("Autocrypt: {}", encryption_helper.get_aheader()).into_bytes();
+    let mut cleartext = format!("Autocrypt: {aheader}").into_bytes();
     cleartext.extend_from_slice(b"\r\n");
     cleartext.extend_from_slice(payload);
-    let encrypted_payload = encryption_helper
-        .encrypt_raw(
-            context,
-            encryption_keyring,
-            cleartext,
-            compress,
-            SeipdVersion::V2,
-        )
-        .await?;
+    let sign_key = key::load_self_secret_key(context).await?;
+    let encrypted_payload = crate::pgp::pk_encrypt(
+        cleartext,
+        encryption_keyring,
+        sign_key,
+        compress,
+        SeipdVersion::V2,
+    )?;
     let boundary = Uuid::new_v4();
 
     let res = format!(