feat: send pre-message on messages with large attachments (#7410)

part of #7367

progress / what's to do:
- [x] send pre-message
- [x] The message's state must be set to MessageState::OutDelivered only
after both messages are sent. If a read receipt is received, the message
can be OutMdnRcvd or OutPending; let's just do whatever is easiest for
now. Take care not to revert from OutMdnReceived to OutDelivered if we
first receive a read receipt and then deliver the full message.
   - this is already the case:
- `OutDelivered` is set when a message is sent out and has no remaining
send jobs in the smtp table for this message id - so already works since
full message and pre message have same msgId in that table.
- `OutMdnRcvd` is a "virtual" state
(https://github.com/chatmail/core/issues/7367#issuecomment-3500891040),
so going back to `OutDelivered` can't happen anymore
- [x] delimit `ChatFullMessageId` with `<` and `>` like the other
message ids
- [x] add tests
	- [x] test that pre message is sent for attachment larger than X
		- test that correct headers are present on both messages
- also test that Autocrypt-gossip and selfavatar should never go into
full-messages
	- [x] test that no pre message is sent for attachment smaller than X
		- no "is full message" header should be present
- [x] test that pre message is not send for large webxdc update or large
text
- [x] fix test
`receive_imf::receive_imf_tests::test_dont_reverify_by_self_on_outgoing_msg`

---------

Co-authored-by: iequidoo <117991069+iequidoo@users.noreply.github.com>
Co-authored-by: Hocuri <hocuri@gmx.de>

src/download.rs

@@ -17,6 +17,22 @@ use crate::{EventType, chatlist_events};
 /// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
 pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
 
+/// From this point onward outgoing messages are considered large
+/// and get a pre-message, which announces the full message.
+// this is only about sending so we can modify it any time.
+// current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB)
+pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000;
+
+/// Max message size to be fetched in the background.
+/// This limit defines what messages are fully fetched in the background.
+/// This is for all messages that don't have the Chat-Is-Full-Message header.
+#[allow(unused)]
+pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000;
+
+/// Max size for pre messages. A warning is emitted when this is exceeded.
+/// Should be well below `MAX_FETCH_MSG_SIZE`
+pub(crate) const PRE_MSG_SIZE_WARNING_THRESHOLD: usize = 150_000;
+
 /// Download state of the message.
 #[derive(
     Debug,
@@ -192,12 +208,17 @@ impl Session {
 
 #[cfg(test)]
 mod tests {
+    use mailparse::MailHeaderMap;
     use num_traits::FromPrimitive;
+    use tokio::fs;
 
     use super::*;
-    use crate::chat::send_msg;
+    use crate::chat::{self, create_group, send_msg};
+    use crate::config::Config;
+    use crate::headerdef::{HeaderDef, HeaderDefMap};
+    use crate::message::Viewtype;
     use crate::receive_imf::receive_imf_from_inbox;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{self, TestContext, TestContextManager};
 
     #[test]
     fn test_downloadstate_values() {
@@ -294,4 +315,327 @@ mod tests {
 
         Ok(())
     }
+    /// Tests that pre message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
+    /// Also test that pre message is sent first, before the full message
+    /// And that Autocrypt-gossip and selfavatar never go into full-messages
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sending_pre_message() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+        let fiona = &tcm.fiona().await;
+        let group_id = alice
+            .create_group_with_members("test group", &[bob, fiona])
+            .await;
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
+        msg.set_text("test".to_owned());
+
+        // assert that test attachment is bigger than limit
+        assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+        let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        //   pre-message and full message should be present
+        //   and test that correct headers are present on both messages
+        assert_eq!(smtp_rows.len(), 2);
+        let pre_message = smtp_rows.first().expect("first element exists");
+        let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?;
+        let full_message = smtp_rows.get(1).expect("second element exists");
+        let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?;
+
+        assert!(
+            pre_message_parsed
+                .headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            full_message_parsed
+                .headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_some()
+        );
+
+        assert_eq!(
+            full_message_parsed
+                .headers
+                .get_header_value(HeaderDef::MessageId),
+            Some(format!("<{}>", msg.rfc724_mid)),
+            "full message should have the rfc message id of the database message"
+        );
+
+        assert_ne!(
+            pre_message_parsed
+                .headers
+                .get_header_value(HeaderDef::MessageId),
+            full_message_parsed
+                .headers
+                .get_header_value(HeaderDef::MessageId),
+            "message ids of pre message and full message should be different"
+        );
+
+        let decrypted_full_message = bob.parse_msg(full_message).await;
+        assert_eq!(decrypted_full_message.decrypting_failed, false);
+        assert_eq!(
+            decrypted_full_message.header_exists(HeaderDef::ChatFullMessageId),
+            false
+        );
+
+        let decrypted_pre_message = bob.parse_msg(pre_message).await;
+        assert_eq!(
+            decrypted_pre_message
+                .get_header(HeaderDef::ChatFullMessageId)
+                .map(String::from),
+            full_message_parsed
+                .headers
+                .get_header_value(HeaderDef::MessageId)
+        );
+        assert!(
+            pre_message_parsed
+                .headers
+                .get_header_value(HeaderDef::ChatFullMessageId)
+                .is_none(),
+            "no Chat-Full-Message-ID header in unprotected headers of Pre-Message"
+        );
+
+        Ok(())
+    }
+
+    /// Tests that pre message has autocrypt gossip headers and self avatar
+    /// and full message doesn't have these headers
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+        let fiona = &tcm.fiona().await;
+        let group_id = alice
+            .create_group_with_members("test group", &[bob, fiona])
+            .await;
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
+        msg.set_text("test".to_owned());
+
+        // assert that test attachment is bigger than limit
+        assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+        // simulate conditions for sending self avatar
+        let avatar_src = alice.get_blobdir().join("avatar.png");
+        fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?;
+        alice
+            .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await?;
+
+        let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        assert_eq!(smtp_rows.len(), 2);
+        let pre_message = smtp_rows.first().expect("first element exists");
+        let full_message = smtp_rows.get(1).expect("second element exists");
+        let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?;
+
+        let decrypted_pre_message = bob.parse_msg(pre_message).await;
+        assert!(
+            decrypted_pre_message
+                .get_header(HeaderDef::ChatFullMessageId)
+                .is_some(),
+            "tested message is not a pre-message, sending order may be broken"
+        );
+        assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0);
+        assert_ne!(decrypted_pre_message.user_avatar, None);
+
+        let decrypted_full_message = bob.parse_msg(full_message).await;
+        assert!(
+            full_message_parsed
+                .headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_some(),
+            "tested message is not a full-message, sending order may be broken"
+        );
+        assert_eq!(decrypted_full_message.gossiped_keys.len(), 0);
+        assert_eq!(decrypted_full_message.user_avatar, None);
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_unecrypted_gets_no_pre_message() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+
+        let chat = alice
+            .create_chat_with_contact("example", "email@example.org")
+            .await;
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
+        msg.set_text("test".to_owned());
+
+        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?;
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        assert_eq!(smtp_rows.len(), 1);
+        let message_bytes = smtp_rows
+            .first()
+            .expect("first element exists")
+            .payload
+            .as_bytes();
+        let message = mailparse::parse_mail(message_bytes)?;
+        assert!(
+            message
+                .headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none(),
+        );
+        Ok(())
+    }
+
+    /// Tests that no pre message is sent for normal message
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_not_sending_pre_message_no_attachment() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+        let chat = alice.create_chat(bob).await;
+
+        // send normal text message
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("test".to_owned());
+        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
+
+        let msg = smtp_rows.first().expect("first element exists");
+        let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
+
+        assert!(
+            mail.headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none(),
+            "no 'Chat-Is-Full-Message'-header should be present"
+        );
+        assert!(
+            mail.headers
+                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
+                .is_none(),
+            "no 'Chat-Full-Message-ID'-header should be present in clear text headers"
+        );
+        let decrypted_message = bob.parse_msg(msg).await;
+        assert!(
+            !decrypted_message.header_exists(HeaderDef::ChatFullMessageId),
+            "no 'Chat-Full-Message-ID'-header should be present"
+        );
+
+        // test that pre message is not send for large large text
+        let mut msg = Message::new(Viewtype::Text);
+        let long_text = String::from_utf8(vec![b'a'; 300_000])?;
+        assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
+        msg.set_text(long_text);
+        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
+
+        let msg = smtp_rows.first().expect("first element exists");
+        let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
+
+        assert!(
+            mail.headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            mail.headers
+                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
+                .is_none(),
+            "no 'Chat-Full-Message-ID'-header should be present in clear text headers"
+        );
+        let decrypted_message = bob.parse_msg(msg).await;
+        assert!(
+            !decrypted_message.header_exists(HeaderDef::ChatFullMessageId),
+            "no 'Chat-Full-Message-ID'-header should be present"
+        );
+        Ok(())
+    }
+
+    /// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+        let chat = alice.create_chat(bob).await;
+
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 100_000], None)?;
+        msg.set_text("test".to_owned());
+
+        // assert that test attachment is smaller than limit
+        assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+        //   only one message and no "is full message" header should be present
+        assert_eq!(smtp_rows.len(), 1);
+
+        let msg = smtp_rows.first().expect("first element exists");
+        let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
+
+        assert!(
+            mail.headers
+                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
+                .is_none()
+        );
+        assert!(
+            mail.headers
+                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
+                .is_none(),
+            "no 'Chat-Full-Message-ID'-header should be present in clear text headers"
+        );
+        let decrypted_message = bob.parse_msg(msg).await;
+        assert!(
+            !decrypted_message.header_exists(HeaderDef::ChatFullMessageId),
+            "no 'Chat-Full-Message-ID'-header should be present"
+        );
+
+        Ok(())
+    }
+
+    /// Tests that pre message is not send for large webxdc updates
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_render_webxdc_status_update_object_range() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id = create_group(&t, "a chat").await?;
+
+        let instance = {
+            let mut instance = Message::new(Viewtype::File);
+            instance.set_file_from_bytes(
+                &t,
+                "minimal.xdc",
+                include_bytes!("../test-data/webxdc/minimal.xdc"),
+                None,
+            )?;
+            let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?;
+            assert_eq!(instance.viewtype, Viewtype::Webxdc);
+            Message::load_from_db(&t, instance_msg_id).await
+        }
+        .unwrap();
+
+        t.pop_sent_msg().await;
+        assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0);
+
+        let long_text = String::from_utf8(vec![b'a'; 300_000])?;
+        assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
+        t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}"))
+            .await?;
+        t.flush_status_updates().await?;
+
+        assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1);
+        Ok(())
+    }
 }