feat: receive pre-messages and adapt download on demand

fix python lint errors

receive pre-mesages, start with changes to imap loop.

refactor: move download code from `scheduler.rs` to `download.rs`, also
move `get_msg_id_by_rfc724_mid` to `MsgId::get_by_rfc724_mid`

`MAX_FETCH_MSG_SIZE` is no longer unused

Parse if it is a pre-message or full-message

start with receiving logic

get rid of `MsgId::get_by_rfc724_mid` because it was a duplicate of
`message::rfc724_mid_exists`

docs: add hint to `MimeMessage::from_bytes` stating that it has
side-effects.

receiving full message

send and receive `attachment_size` and set viewtype to text in
pre_message

metadata as struct in pre-message in header. And fill params that we can
already fill from the metadata. Also add a new api to check what
viewtype the message will have once downloaded.

api: jsonrpc: add `full_message_view_type` to `Message` and
`MessageInfo`

make PreMsgMetadata.to_header_value not consume self/PreMsgMetadata

add api to merge params

on download full message: merge new params into old params and remove
full-message metadata params

move tests to `src/tests/pre_messages.rs`

dynamically allocate test attachment bytes

fix detection of pre-messages. (it looked for the ChatFullMessageId
header in the unencrypted headers before)

fix setting dl state to avaiable on pre-messages

fix: save pre message with rfc724_mid of full message als disable
replacement for full messages

add some receiving tests and update test todo for premessage metadata

test: process full message before pre-message

test receive normal message

some serialization tests for PreMsgMetadata

remove outdated todo comment

test that pre-message contains message text

PreMsgMetadata: test_build_from_file_msg and test_build_from_file_msg

test: test_receive_pre_message_image

Test receiving the full message after receiving an edit after receiving
the pre-message

test_reaction_on_pre_message

test_full_download_after_trashed

test_webxdc_update_for_not_downloaded_instance

simplify fake webxdc generation in
test_webxdc_update_for_not_downloaded_instance

test_markseen_pre_msg

test_pre_msg_can_start_chat and test_full_msg_can_start_chat

test_download_later_keeps_message_order

test_chatlist_event_on_full_msg_download

fix download not working

log splitting into pre-message

add pre-message info to text when loading from db. this can be disabled
with config key `hide_pre_message_metadata_text` if ui wants to display
it in a prettier way.

update `download_limit` documentation

more logging: log size of pre and post messages

rename full message to Post-Message

split up the pre-message tests into multiple files

dedup test code by extracting code to create test messages into util
methods

remove post_message_view_type from api, now it is only used internally
for tests

remove `hide_pre_message_metadata_text` config option, as there
currently is no way to get the full message viewtype anymore

Update src/download.rs
resolve comment

use `parse_message_id` instead of removing `<>`parenthesis it manually

fix available_post_msgs gets no entries
handle forwarding and add a test for it.

convert comment to log warning event on unexpected download failure

add doc comment to `simple_imap_loop`

more logging

handle saving pre-message to self messages and test.

deltachat-ffi/deltachat.h

@@ -488,11 +488,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    0=use IMAP IDLE if the server supports it.
  *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
- *                    For larger messages, only the header is downloaded and a placeholder is shown.
+ *                    For messages with large attachements, two messages are sent:
+ *                    a Pre-Message containing metadata and a Post-Message containing the attachment.
+ *                    Pre-Messages are always downloaded and show a placeholder message 
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
- *                    The limit is compared against raw message sizes, including headers.
+ *                    Post-Messages are automatically downloaded if they are smaller than the download_limit.
- *                    The actually used limit may be corrected
- *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
  * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
@@ -4311,6 +4311,7 @@ char*             dc_msg_get_webxdc_info      (const dc_msg_t* msg);
 /**
  * Get the size of the file. Returns the size of the file associated with a
  * message, if applicable. 
+ * If message is a pre-message, then this returns size of the to be downloaded file.
  *
  * Typically, this is used to show the size of document files, e.g. a PDF.
  *

deltachat-jsonrpc/src/api/types/message.rs

@@ -92,6 +92,8 @@ pub struct MessageObject {
 
     file: Option<String>,
     file_mime: Option<String>,
+    /// Returns the size of the file in bytes, if applicable.
+    /// If message is a pre-message, then this returns size of the to be downloaded file.
     file_bytes: u64,
     file_name: Option<String>,
 

deltachat-rpc-client/tests/test_something.py

@@ -10,7 +10,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from deltachat_rpc_client import EventType, events
-from deltachat_rpc_client.const import DownloadState, MessageState
+from deltachat_rpc_client.const import MessageState
 from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS
 from deltachat_rpc_client.rpc import JsonRpcError
 
@@ -354,7 +354,6 @@ def test_receive_imf_failure(acfactory) -> None:
     message = bob.wait_for_incoming_msg()
     snapshot = message.get_snapshot()
     assert snapshot.text == "Hello again!"
-    assert snapshot.download_state == DownloadState.DONE
     assert snapshot.error is None
 
 

src/chat.rs

@@ -12,6 +12,7 @@ use std::time::Duration;
 use anyhow::{Context as _, Result, anyhow, bail, ensure};
 use chrono::TimeZone;
 use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line};
+use humansize::{BINARY, format_size};
 use mail_builder::mime::MimePart;
 use serde::{Deserialize, Serialize};
 use strum_macros::EnumIter;
@@ -2730,10 +2731,10 @@ async fn prepare_send_msg(
     Ok(row_ids)
 }
 
-/// Renders the message or Full-Message and Pre-Message.
+/// Renders the Message or splits it into Post-Message and Pre-Message.
 ///
-/// Pre-Message is a small message with metadata which announces a larger Full-Message.
+/// Pre-Message is a small message with metadata which announces a larger Post-Message.
-/// Full messages are not downloaded in the background.
+/// Post-Messages are not downloaded in the background.
 ///
 /// If pre-message is not nessesary this returns a normal message instead.
 async fn render_mime_message_and_pre_message(
@@ -2750,9 +2751,14 @@ async fn render_mime_message_and_pre_message(
             > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
 
     if needs_pre_message {
-        let mut mimefactory_full_msg = mimefactory.clone();
+        info!(
-        mimefactory_full_msg.set_as_full_message();
+            context,
-        let rendered_msg = mimefactory_full_msg.render(context).await?;
+            "Message is large and will be split into a pre- and a post-message.",
+        );
+
+        let mut mimefactory_post_msg = mimefactory.clone();
+        mimefactory_post_msg.set_as_post_message();
+        let rendered_msg = mimefactory_post_msg.render(context).await?;
 
         let mut mimefactory_pre_msg = mimefactory;
         mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
@@ -2856,6 +2862,21 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             }
         }?;
 
+    if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) {
+        info!(
+            context,
+            "Message Sizes: Pre-Message {}; Post-Message: {}",
+            format_size(pre_msg.message.len(), BINARY),
+            format_size(post_msg.message.len(), BINARY)
+        );
+    } else {
+        info!(
+            context,
+            "Message will be sent as normal message (no pre- and post message). Size: {}",
+            format_size(rendered_msg.message.len(), BINARY)
+        );
+    }
+
     if needs_encryption && !rendered_msg.is_encrypted {
         /* unrecoverable */
         message::set_msg_failed(
@@ -4319,6 +4340,14 @@ pub async fn forward_msgs_2ctx(
             msg.viewtype = Viewtype::Text;
         }
 
+        if msg.download_state != DownloadState::Done {
+            // we don't use Message.get_text() here,
+            // because it may change in future,
+            // when UI shows this info itself,
+            // then the additional_text will not be added in get_text anymore.
+            msg.text += &msg.additional_text;
+        }
+
         let param = &mut param;
         msg.param.steal(param, Param::File);
         msg.param.steal(param, Param::Filename);
@@ -4329,6 +4358,7 @@ pub async fn forward_msgs_2ctx(
         msg.param.steal(param, Param::ProtectQuote);
         msg.param.steal(param, Param::Quote);
         msg.param.steal(param, Param::Summary1);
+
         msg.in_reply_to = None;
 
         // do not leak data as group names; a default subject is generated by mimefactory
@@ -4395,12 +4425,22 @@ pub(crate) async fn save_copy_in_self_talk(
     msg.param.remove(Param::WebxdcDocumentTimestamp);
     msg.param.remove(Param::WebxdcSummary);
     msg.param.remove(Param::WebxdcSummaryTimestamp);
+    msg.param.remove(Param::PostMessageFileBytes);
+    msg.param.remove(Param::PostMessageViewtype);
+
+    if msg.download_state != DownloadState::Done {
+        // we don't use Message.get_text() here,
+        // because it may change in future,
+        // when UI shows this info itself,
+        // then the additional_text will not be added in get_text anymore.
+        msg.text += &msg.additional_text;
+    }
 
     if !msg.original_msg_id.is_unset() {
         bail!("message already saved.");
     }
 
-    let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt,
+    let copy_fields = "from_id, to_id, timestamp_rcvd, type,
                        mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg";
     let row_id = context
         .sql
@@ -4408,7 +4448,7 @@ pub(crate) async fn save_copy_in_self_talk(
             &format!(
                 "INSERT INTO msgs ({copy_fields},
                                    timestamp_sent,
-                                   chat_id, rfc724_mid, state, timestamp, param, starred)
+                                   txt, chat_id, rfc724_mid, state, timestamp, param, starred)
                  SELECT            {copy_fields},
                                    -- Outgoing messages on originating device
                                    -- have timestamp_sent == 0.
@@ -4416,10 +4456,11 @@ pub(crate) async fn save_copy_in_self_talk(
                                    -- so UIs display the same timestamp
                                    -- for saved and original message.
                                    IIF(timestamp_sent == 0, timestamp, timestamp_sent),
-                                   ?, ?, ?, ?, ?, ?
+                                   ?, ?, ?, ?, ?, ?, ?
                  FROM msgs WHERE id=?;"
             ),
             (
+                msg.text,
                 dest_chat_id,
                 dest_rfc724_mid,
                 if msg.from_id == ContactId::SELF {

src/context.rs

@@ -609,7 +609,7 @@ impl Context {
                     convert_folder_meaning(self, folder_meaning).await?
                 {
                     connection
-                        .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning)
+                        .fetch_move_delete(self, &mut session, true, &watch_folder, folder_meaning)
                         .await?;
                 }
             }
@@ -622,6 +622,12 @@ impl Context {
             {
                 warn!(self, "Failed to update quota: {err:#}.");
             }
+
+            // OPTIONAL TODO: if time left start downloading messages
+            // while (msg = download_when_normal_starts) {
+            //  if not time_left {break;}
+            //  connection.download_message(msg) }
+            // }
         }
 
         info!(
@@ -1080,13 +1086,6 @@ impl Context {
                 .await?
                 .unwrap_or_default(),
         );
-        res.insert(
-            "fail_on_receiving_full_msg",
-            self.sql
-                .get_raw_config("fail_on_receiving_full_msg")
-                .await?
-                .unwrap_or_default(),
-        );
         res.insert(
             "std_header_protection_composing",
             self.sql

src/context/context_tests.rs

@@ -297,7 +297,7 @@ async fn test_get_info_completeness() {
         "encrypted_device_token",
         "stats_last_update",
         "stats_last_old_contact_id",
-        "simulate_receive_imf_error",
+        "simulate_receive_imf_error", // only used in tests
     ];
     let t = TestContext::new().await;
     let info = t.get_info().await.unwrap();

src/download.rs

@@ -8,9 +8,12 @@ use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
 use crate::imap::session::Session;
-use crate::message::{Message, MsgId};
+use crate::log::warn;
+use crate::message::{self, Message, MsgId, rfc724_mid_exists};
 use crate::{EventType, chatlist_events};
 
+pub(crate) mod pre_msg_metadata;
+
 /// If a message is downloaded only partially
 /// and `delete_server_after` is set to small timeouts (eg. "at once"),
 /// the user might have no chance to actually download that message.
@@ -18,16 +21,15 @@ use crate::{EventType, chatlist_events};
 pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
 
 /// From this point onward outgoing messages are considered large
-/// and get a pre-message, which announces the full message.
+/// and get a Pre-Message, which announces the Post-Message.
 // this is only about sending so we can modify it any time.
 // current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB)
 pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000;
 
 /// Max message size to be fetched in the background.
 /// This limit defines what messages are fully fetched in the background.
-/// This is for all messages that don't have the Chat-Is-Full-Message header.
+/// This is for all messages that don't have the Post-Message header.
-#[allow(unused)]
+pub(crate) const MAX_FETCH_MSG_SIZE: u32 = 1_000_000;
-pub(crate) const MAX_FETCH_MSG_SIZE: usize = 1_000_000;
 
 /// Max size for pre messages. A warning is emitted when this is exceeded.
 /// Should be well below `MAX_FETCH_MSG_SIZE`
@@ -69,7 +71,7 @@ pub enum DownloadState {
 }
 
 impl MsgId {
-    /// Schedules full message download for partially downloaded message.
+    /// Schedules Post-Message download for partially downloaded message.
     pub async fn download_full(self, context: &Context) -> Result<()> {
         let msg = Message::load_from_db(context, self).await?;
         match msg.download_state() {
@@ -78,11 +80,17 @@ impl MsgId {
             }
             DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
             DownloadState::Available | DownloadState::Failure => {
+                if msg.rfc724_mid().is_empty() {
+                    return Err(anyhow!("Download not possible, message has no rfc724_mid"));
+                }
                 self.update_download_state(context, DownloadState::InProgress)
                     .await?;
                 context
                     .sql
-                    .execute("INSERT INTO download (msg_id) VALUES (?)", (self,))
+                    .execute(
+                        "INSERT INTO download (rfc724_mid) VALUES (?)",
+                        (msg.rfc724_mid(),),
+                    )
                     .await?;
                 context.scheduler.interrupt_inbox().await;
             }
@@ -131,25 +139,14 @@ impl Message {
 /// Most messages are downloaded automatically on fetch instead.
 pub(crate) async fn download_msg(
     context: &Context,
-    msg_id: MsgId,
+    rfc724_mid: String,
     session: &mut Session,
 ) -> Result<()> {
-    let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else {
-        // If partially downloaded message was already deleted
-        // we do not know its Message-ID anymore
-        // so cannot download it.
-        //
-        // Probably the message expired due to `delete_device_after`
-        // setting or was otherwise removed from the device,
-        // so we don't want it to reappear anyway.
-        return Ok(());
-    };
-
     let row = context
         .sql
         .query_row_optional(
             "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
-            (&msg.rfc724_mid,),
+            (&rfc724_mid,),
             |row| {
                 let server_uid: u32 = row.get(0)?;
                 let server_folder: String = row.get(1)?;
@@ -164,7 +161,7 @@ pub(crate) async fn download_msg(
     };
 
     session
-        .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
+        .fetch_single_msg(context, &server_folder, server_uid, rfc724_mid)
         .await?;
     Ok(())
 }
@@ -206,19 +203,152 @@ impl Session {
     }
 }
 
+async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> {
+    if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? {
+        // Update download state to failure
+        // so it can be retried.
+        //
+        // On success update_download_state() is not needed
+        // as receive_imf() already
+        // set the state and emitted the event.
+        msg_id
+            .update_download_state(context, DownloadState::Failure)
+            .await?;
+    }
+    Ok(())
+}
+
+async fn available_post_msgs_contains_rfc724_mid(
+    context: &Context,
+    rfc724_mid: &str,
+) -> Result<bool> {
+    Ok(context
+        .sql
+        .query_get_value::<String>(
+            "SELECT rfc724_mid FROM available_post_msgs WHERE rfc724_mid=?",
+            (&rfc724_mid,),
+        )
+        .await?
+        .is_some())
+}
+
+async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "DELETE FROM available_post_msgs WHERE rfc724_mid=?",
+            (&rfc724_mid,),
+        )
+        .await?;
+    Ok(())
+}
+
+async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> {
+    context
+        .sql
+        .execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,))
+        .await?;
+    Ok(())
+}
+
+// this is a dedicated method because it is used in multiple places.
+pub(crate) async fn premessage_is_downloaded_for(
+    context: &Context,
+    rfc724_mid: &str,
+) -> Result<bool> {
+    Ok(message::rfc724_mid_exists(context, rfc724_mid)
+        .await?
+        .is_some())
+}
+
+pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> {
+    let rfc724_mids = context
+        .sql
+        .query_map_vec("SELECT rfc724_mid FROM download", (), |row| {
+            let rfc724_mid: String = row.get(0)?;
+            Ok(rfc724_mid)
+        })
+        .await?;
+
+    for rfc724_mid in &rfc724_mids {
+        let res = download_msg(context, rfc724_mid.clone(), session).await;
+        if res.is_ok() {
+            remove_from_download_table(context, rfc724_mid).await?;
+            remove_from_available_post_msgs_table(context, rfc724_mid).await?;
+        }
+        if let Err(err) = res {
+            warn!(
+                context,
+                "Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err
+            );
+            if !premessage_is_downloaded_for(context, rfc724_mid).await? {
+                // This is probably a classical email that vanished before we could download it
+                warn!(
+                    context,
+                    "{rfc724_mid} is probably a classical email that vanished before we could download it"
+                );
+                remove_from_download_table(context, rfc724_mid).await?;
+            } else if available_post_msgs_contains_rfc724_mid(context, rfc724_mid).await? {
+                warn!(
+                    context,
+                    "{rfc724_mid} is in available_post_msgs table but we failed to fetch it,
+                    so set the message to DownloadState::Failure - probably it was deleted on the server in the meantime"
+                );
+                set_msg_state_to_failed(context, rfc724_mid).await?;
+                remove_from_download_table(context, rfc724_mid).await?;
+                remove_from_available_post_msgs_table(context, rfc724_mid).await?;
+            } else {
+                // leave the message in DownloadState::InProgress;
+                // it will be downloaded once it arrives.
+            }
+        }
+    }
+
+    Ok(())
+}
+
+/// Download known post messages without pre_message
+/// in order to guard against lost pre-messages:
+pub(crate) async fn download_known_post_messages_without_pre_message(
+    context: &Context,
+    session: &mut Session,
+) -> Result<()> {
+    let rfc724_mids = context
+        .sql
+        .query_map_vec("SELECT rfc724_mid FROM available_post_msgs", (), |row| {
+            let rfc724_mid: String = row.get(0)?;
+            Ok(rfc724_mid)
+        })
+        .await?;
+    for rfc724_mid in &rfc724_mids {
+        if !premessage_is_downloaded_for(context, rfc724_mid).await? {
+            // Download the Post-Message unconditionally,
+            // because the Pre-Message got lost.
+            // The message may be in the wrong order,
+            // but at least we have it at all.
+            let res = download_msg(context, rfc724_mid.clone(), session).await;
+            if res.is_ok() {
+                remove_from_available_post_msgs_table(context, rfc724_mid).await?;
+            }
+            if let Err(err) = res {
+                warn!(
+                    context,
+                    "download_known_post_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.",
+                    err
+                );
+            }
+        }
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
-    use mailparse::MailHeaderMap;
     use num_traits::FromPrimitive;
-    use tokio::fs;
 
     use super::*;
-    use crate::chat::{self, create_group, send_msg};
+    use crate::chat::send_msg;
-    use crate::config::Config;
+    use crate::test_utils::TestContext;
-    use crate::headerdef::{HeaderDef, HeaderDefMap};
-    use crate::message::Viewtype;
-    use crate::receive_imf::receive_imf_from_inbox;
-    use crate::test_utils::{self, TestContext, TestContextManager};
 
     #[test]
     fn test_downloadstate_values() {
@@ -268,374 +398,5 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    // NOTE: The download tests for pre-messages are in src/tests/pre_messages.rs
-    async fn test_download_stub_message() -> Result<()> {
-        let t = TestContext::new_alice().await;
-
-        let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-             From: bob@example.com\n\
-             To: alice@example.org\n\
-             Subject: foo\n\
-             Message-ID: <Mr.12345678901@example.com>\n\
-             Chat-Version: 1.0\n\
-             Date: Sun, 22 Mar 2020 22:37:57 +0000\
-             Content-Type: text/plain";
-
-        t.sql
-            .execute(
-                r#"INSERT INTO chats VALUES(
-                    11001,100,'bob@example.com',0,'',2,'',
-                    replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0,'');
-                "#,
-                (),
-            )
-            .await?;
-        t.sql.execute(r#"INSERT INTO msgs VALUES(
-                11001,'Mr.12345678901@example.com','',0,
-                11001,11001,1,1763151754,10,10,1,0,
-                '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'',
-                '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0);
-        "#, ()).await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.download_state(), DownloadState::Available);
-        assert_eq!(msg.get_subject(), "foo");
-        assert!(msg.get_text().contains("[97.66 KiB message]"));
-
-        receive_imf_from_inbox(
-            &t,
-            "Mr.12345678901@example.com",
-            format!("{header}\n\n100k text...").as_bytes(),
-            false,
-        )
-        .await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.download_state(), DownloadState::Done);
-        assert_eq!(msg.get_subject(), "foo");
-        assert_eq!(msg.get_text(), "100k text...");
-
-        Ok(())
-    }
-    /// Tests that pre message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
-    /// Also test that pre message is sent first, before the full message
-    /// And that Autocrypt-gossip and selfavatar never go into full-messages
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_sending_pre_message() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-        let fiona = &tcm.fiona().await;
-        let group_id = alice
-            .create_group_with_members("test group", &[bob, fiona])
-            .await;
-
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
-        msg.set_text("test".to_owned());
-
-        // assert that test attachment is bigger than limit
-        assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
-
-        let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
-        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-
-        //   pre-message and full message should be present
-        //   and test that correct headers are present on both messages
-        assert_eq!(smtp_rows.len(), 2);
-        let pre_message = smtp_rows.first().expect("first element exists");
-        let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?;
-        let full_message = smtp_rows.get(1).expect("second element exists");
-        let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?;
-
-        assert!(
-            pre_message_parsed
-                .headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_none()
-        );
-        assert!(
-            full_message_parsed
-                .headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_some()
-        );
-
-        assert_eq!(
-            full_message_parsed
-                .headers
-                .get_header_value(HeaderDef::MessageId),
-            Some(format!("<{}>", msg.rfc724_mid)),
-            "full message should have the rfc message id of the database message"
-        );
-
-        assert_ne!(
-            pre_message_parsed
-                .headers
-                .get_header_value(HeaderDef::MessageId),
-            full_message_parsed
-                .headers
-                .get_header_value(HeaderDef::MessageId),
-            "message ids of pre message and full message should be different"
-        );
-
-        let decrypted_full_message = bob.parse_msg(full_message).await;
-        assert_eq!(decrypted_full_message.decrypting_failed, false);
-        assert_eq!(
-            decrypted_full_message.header_exists(HeaderDef::ChatFullMessageId),
-            false
-        );
-
-        let decrypted_pre_message = bob.parse_msg(pre_message).await;
-        assert_eq!(
-            decrypted_pre_message
-                .get_header(HeaderDef::ChatFullMessageId)
-                .map(String::from),
-            full_message_parsed
-                .headers
-                .get_header_value(HeaderDef::MessageId)
-        );
-        assert!(
-            pre_message_parsed
-                .headers
-                .get_header_value(HeaderDef::ChatFullMessageId)
-                .is_none(),
-            "no Chat-Full-Message-ID header in unprotected headers of Pre-Message"
-        );
-
-        Ok(())
-    }
-
-    /// Tests that pre message has autocrypt gossip headers and self avatar
-    /// and full message doesn't have these headers
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-        let fiona = &tcm.fiona().await;
-        let group_id = alice
-            .create_group_with_members("test group", &[bob, fiona])
-            .await;
-
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
-        msg.set_text("test".to_owned());
-
-        // assert that test attachment is bigger than limit
-        assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
-
-        // simulate conditions for sending self avatar
-        let avatar_src = alice.get_blobdir().join("avatar.png");
-        fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?;
-        alice
-            .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await?;
-
-        let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
-        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-
-        assert_eq!(smtp_rows.len(), 2);
-        let pre_message = smtp_rows.first().expect("first element exists");
-        let full_message = smtp_rows.get(1).expect("second element exists");
-        let full_message_parsed = mailparse::parse_mail(full_message.payload.as_bytes())?;
-
-        let decrypted_pre_message = bob.parse_msg(pre_message).await;
-        assert!(
-            decrypted_pre_message
-                .get_header(HeaderDef::ChatFullMessageId)
-                .is_some(),
-            "tested message is not a pre-message, sending order may be broken"
-        );
-        assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0);
-        assert_ne!(decrypted_pre_message.user_avatar, None);
-
-        let decrypted_full_message = bob.parse_msg(full_message).await;
-        assert!(
-            full_message_parsed
-                .headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_some(),
-            "tested message is not a full-message, sending order may be broken"
-        );
-        assert_eq!(decrypted_full_message.gossiped_keys.len(), 0);
-        assert_eq!(decrypted_full_message.user_avatar, None);
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_unecrypted_gets_no_pre_message() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-
-        let chat = alice
-            .create_chat_with_contact("example", "email@example.org")
-            .await;
-
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
-        msg.set_text("test".to_owned());
-
-        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?;
-        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-
-        assert_eq!(smtp_rows.len(), 1);
-        let message_bytes = smtp_rows
-            .first()
-            .expect("first element exists")
-            .payload
-            .as_bytes();
-        let message = mailparse::parse_mail(message_bytes)?;
-        assert!(
-            message
-                .headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_none(),
-        );
-        Ok(())
-    }
-
-    /// Tests that no pre message is sent for normal message
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_not_sending_pre_message_no_attachment() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-        let chat = alice.create_chat(bob).await;
-
-        // send normal text message
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text("test".to_owned());
-        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
-        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-
-        assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
-
-        let msg = smtp_rows.first().expect("first element exists");
-        let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
-
-        assert!(
-            mail.headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_none(),
-            "no 'Chat-Is-Full-Message'-header should be present"
-        );
-        assert!(
-            mail.headers
-                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
-                .is_none(),
-            "no 'Chat-Full-Message-ID'-header should be present in clear text headers"
-        );
-        let decrypted_message = bob.parse_msg(msg).await;
-        assert!(
-            !decrypted_message.header_exists(HeaderDef::ChatFullMessageId),
-            "no 'Chat-Full-Message-ID'-header should be present"
-        );
-
-        // test that pre message is not send for large large text
-        let mut msg = Message::new(Viewtype::Text);
-        let long_text = String::from_utf8(vec![b'a'; 300_000])?;
-        assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
-        msg.set_text(long_text);
-        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
-        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-
-        assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
-
-        let msg = smtp_rows.first().expect("first element exists");
-        let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
-
-        assert!(
-            mail.headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_none()
-        );
-        assert!(
-            mail.headers
-                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
-                .is_none(),
-            "no 'Chat-Full-Message-ID'-header should be present in clear text headers"
-        );
-        let decrypted_message = bob.parse_msg(msg).await;
-        assert!(
-            !decrypted_message.header_exists(HeaderDef::ChatFullMessageId),
-            "no 'Chat-Full-Message-ID'-header should be present"
-        );
-        Ok(())
-    }
-
-    /// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-        let chat = alice.create_chat(bob).await;
-
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file_from_bytes(alice, "test.bin", &[0u8; 100_000], None)?;
-        msg.set_text("test".to_owned());
-
-        // assert that test attachment is smaller than limit
-        assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
-
-        let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
-        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
-
-        //   only one message and no "is full message" header should be present
-        assert_eq!(smtp_rows.len(), 1);
-
-        let msg = smtp_rows.first().expect("first element exists");
-        let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
-
-        assert!(
-            mail.headers
-                .get_first_header(HeaderDef::ChatIsFullMessage.get_headername())
-                .is_none()
-        );
-        assert!(
-            mail.headers
-                .get_first_header(HeaderDef::ChatFullMessageId.get_headername())
-                .is_none(),
-            "no 'Chat-Full-Message-ID'-header should be present in clear text headers"
-        );
-        let decrypted_message = bob.parse_msg(msg).await;
-        assert!(
-            !decrypted_message.header_exists(HeaderDef::ChatFullMessageId),
-            "no 'Chat-Full-Message-ID'-header should be present"
-        );
-
-        Ok(())
-    }
-
-    /// Tests that pre message is not send for large webxdc updates
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_render_webxdc_status_update_object_range() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group(&t, "a chat").await?;
-
-        let instance = {
-            let mut instance = Message::new(Viewtype::File);
-            instance.set_file_from_bytes(
-                &t,
-                "minimal.xdc",
-                include_bytes!("../test-data/webxdc/minimal.xdc"),
-                None,
-            )?;
-            let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?;
-            assert_eq!(instance.viewtype, Viewtype::Webxdc);
-            Message::load_from_db(&t, instance_msg_id).await
-        }
-        .unwrap();
-
-        t.pop_sent_msg().await;
-        assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0);
-
-        let long_text = String::from_utf8(vec![b'a'; 300_000])?;
-        assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
-        t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}"))
-            .await?;
-        t.flush_status_updates().await?;
-
-        assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1);
-        Ok(())
-    }
 }

src/download/pre_msg_metadata.rs

@@ -0,0 +1,248 @@
+use anyhow::{Context as _, Result};
+use num_traits::ToPrimitive;
+use serde::{Deserialize, Serialize};
+
+use crate::context::Context;
+use crate::log::warn;
+use crate::message::Message;
+use crate::message::Viewtype;
+use crate::param::{Param, Params};
+
+/// Metadata contained in Pre-Message that describes the Post-Message.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct PreMsgMetadata {
+    /// size of the attachment in bytes
+    pub(crate) size: u64,
+    /// Real viewtype of message
+    pub(crate) viewtype: Viewtype,
+    /// the original file name
+    pub(crate) filename: String,
+    /// Dimensions: width and height of image or video
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) dimensions: Option<(i32, i32)>,
+    /// Duration of audio file or video in milliseconds
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub(crate) duration: Option<i32>,
+}
+
+impl PreMsgMetadata {
+    // Returns PreMsgMetadata for messages with files and None for messages without file attachment
+    pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result<Option<Self>> {
+        if !message.viewtype.has_file() {
+            return Ok(None);
+        }
+
+        let size = message
+            .get_filebytes(context)
+            .await?
+            .context("unexpected: file has no size")?;
+        let filename = message
+            .param
+            .get(Param::Filename)
+            .unwrap_or_default()
+            .to_owned();
+        let dimensions = {
+            match (
+                message.param.get_int(Param::Width),
+                message.param.get_int(Param::Height),
+            ) {
+                (None, None) => None,
+                (Some(width), Some(height)) => Some((width, height)),
+                _ => {
+                    warn!(context, "Message has misses either width or height");
+                    None
+                }
+            }
+        };
+        let duration = message.param.get_int(Param::Duration);
+
+        Ok(Some(Self {
+            size,
+            filename,
+            viewtype: message.viewtype,
+            dimensions,
+            duration,
+        }))
+    }
+
+    pub(crate) fn to_header_value(&self) -> Result<String> {
+        Ok(serde_json::to_string(&self)?)
+    }
+
+    pub(crate) fn try_from_header_value(value: &str) -> Result<Self> {
+        Ok(serde_json::from_str(value)?)
+    }
+}
+
+impl Params {
+    /// Applies data from pre_msg_metadata to Params
+    pub(crate) fn apply_from_pre_msg_metadata(
+        &mut self,
+        pre_msg_metadata: &PreMsgMetadata,
+    ) -> &mut Self {
+        self.set(Param::PostMessageFileBytes, pre_msg_metadata.size);
+        if !pre_msg_metadata.filename.is_empty() {
+            self.set(Param::Filename, &pre_msg_metadata.filename);
+        }
+        self.set_i64(
+            Param::PostMessageViewtype,
+            pre_msg_metadata.viewtype.to_i64().unwrap_or_default(),
+        );
+        if let Some((width, height)) = pre_msg_metadata.dimensions {
+            self.set(Param::Width, width);
+            self.set(Param::Height, height);
+        }
+        if let Some(duration) = pre_msg_metadata.duration {
+            self.set(Param::Duration, duration);
+        }
+
+        self
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use anyhow::Result;
+    use pretty_assertions::assert_eq;
+
+    use crate::{
+        message::{Message, Viewtype},
+        test_utils::{TestContextManager, create_test_image},
+    };
+
+    use super::PreMsgMetadata;
+
+    /// Build from message with file attachment
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_build_from_file_msg() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+
+        let mut file_msg = Message::new(Viewtype::File);
+        file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?;
+        let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?;
+        assert_eq!(
+            pre_mesage_metadata,
+            Some(PreMsgMetadata {
+                size: 1_000_000,
+                viewtype: Viewtype::File,
+                filename: "test.bin".to_string(),
+                dimensions: None,
+                duration: None,
+            })
+        );
+        Ok(())
+    }
+
+    /// Build from message with image attachment
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_build_from_image_msg() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let mut image_msg = Message::new(Viewtype::Image);
+
+        let (width, height) = (1080, 1920);
+        let test_img = create_test_image(width, height)?;
+        image_msg.set_file_from_bytes(alice, "vacation.png", &test_img, None)?;
+        // this is usually done while sending,
+        // but we don't send it here, so we need to call it ourself
+        image_msg.try_calc_and_set_dimensions(alice).await?;
+        let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?;
+        assert_eq!(
+            pre_mesage_metadata,
+            Some(PreMsgMetadata {
+                size: 1816098,
+                viewtype: Viewtype::Image,
+                filename: "vacation.png".to_string(),
+                dimensions: Some((width as i32, height as i32)),
+                duration: None,
+            })
+        );
+
+        Ok(())
+    }
+
+    /// Test that serialisation results in expected format
+    #[test]
+    fn test_serialize_to_header() -> Result<()> {
+        assert_eq!(
+            PreMsgMetadata {
+                size: 1_000_000,
+                viewtype: Viewtype::File,
+                filename: "test.bin".to_string(),
+                dimensions: None,
+                duration: None,
+            }
+            .to_header_value()?,
+            "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}"
+        );
+        assert_eq!(
+            PreMsgMetadata {
+                size: 5_342_765,
+                viewtype: Viewtype::Image,
+                filename: "vacation.png".to_string(),
+                dimensions: Some((1080, 1920)),
+                duration: None,
+            }
+            .to_header_value()?,
+            "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}"
+        );
+        assert_eq!(
+            PreMsgMetadata {
+                size: 5_000,
+                viewtype: Viewtype::Audio,
+                filename: "audio-DD-MM-YY.ogg".to_string(),
+                dimensions: None,
+                duration: Some(152_310),
+            }
+            .to_header_value()?,
+            "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}"
+        );
+
+        Ok(())
+    }
+
+    /// Test that deserialisation from expected format works
+    /// This test will become important for compatibility between versions in the future
+    #[test]
+    fn test_deserialize_from_header() -> Result<()> {
+        assert_eq!(
+            serde_json::from_str::<PreMsgMetadata>(
+                "{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}"
+            )?,
+            PreMsgMetadata {
+                size: 1_000_000,
+                viewtype: Viewtype::File,
+                filename: "test.bin".to_string(),
+                dimensions: None,
+                duration: None,
+            }
+        );
+        assert_eq!(
+            serde_json::from_str::<PreMsgMetadata>(
+                "{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}"
+            )?,
+            PreMsgMetadata {
+                size: 5_342_765,
+                viewtype: Viewtype::Image,
+                filename: "vacation.png".to_string(),
+                dimensions: Some((1080, 1920)),
+                duration: None,
+            }
+        );
+        assert_eq!(
+            serde_json::from_str::<PreMsgMetadata>(
+                "{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}"
+            )?,
+            PreMsgMetadata {
+                size: 5_000,
+                viewtype: Viewtype::Audio,
+                filename: "audio-DD-MM-YY.ogg".to_string(),
+                dimensions: None,
+                duration: Some(152_310),
+            }
+        );
+
+        Ok(())
+    }
+}

src/headerdef.rs

@@ -104,15 +104,19 @@ pub enum HeaderDef {
     ChatBroadcastSecret,
     /// A message with a large attachment is split into two MIME messages:
     /// A pre-message, which contains everything but the attachment,
-    /// and a full-message.
+    /// and a Post-Message.
-    /// The pre-message gets a `Chat-Full-Message-Id` header
+    /// The Pre-Message gets a `Chat-Post-Message-Id` header
-    /// referencing the full-message's rfc724_mid.
+    /// referencing the Post-Message's rfc724_mid.
-    ChatFullMessageId,
+    ChatPostMessageId,
 
-    /// This message is preceded by a pre-message
+    /// Announce Post-Message metadata in a Pre-Message.
+    /// contains serialized PreMsgMetadata struct
+    ChatPostMessageMetadata,
+
+    /// This message is preceded by a Pre-Message
     /// and thus this message can be skipped while fetching messages.
     /// This is a cleartext / unproteced header.
-    ChatIsFullMessage,
+    ChatIsPostMessage,
 
     /// [Autocrypt](https://autocrypt.org/) header.
     Autocrypt,

src/imap.rs

@@ -23,7 +23,6 @@ use num_traits::FromPrimitive;
 use ratelimit::Ratelimit;
 use url::Url;
 
-use crate::calls::{create_fallback_ice_servers, create_ice_servers_from_metadata};
 use crate::chat::{self, ChatId, ChatIdBlocked, add_device_msg};
 use crate::chatlist_events;
 use crate::config::Config;
@@ -48,6 +47,10 @@ use crate::tools::{self, create_id, duration_to_str, time};
 use crate::transport::{
     ConfiguredLoginParam, ConfiguredServerLoginParam, prioritize_server_login_params,
 };
+use crate::{
+    calls::{create_fallback_ice_servers, create_ice_servers_from_metadata},
+    download::MAX_FETCH_MSG_SIZE,
+};
 
 pub(crate) mod capabilities;
 mod client;
@@ -520,6 +523,7 @@ impl Imap {
         &mut self,
         context: &Context,
         session: &mut Session,
+        is_background_fetch: bool,
         watch_folder: &str,
         folder_meaning: FolderMeaning,
     ) -> Result<()> {
@@ -529,7 +533,13 @@ impl Imap {
         }
 
         let msgs_fetched = self
-            .fetch_new_messages(context, session, watch_folder, folder_meaning)
+            .fetch_new_messages(
+                context,
+                session,
+                is_background_fetch,
+                watch_folder,
+                folder_meaning,
+            )
             .await
             .context("fetch_new_messages")?;
         if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
@@ -555,6 +565,7 @@ impl Imap {
         &mut self,
         context: &Context,
         session: &mut Session,
+        is_background_fetch: bool,
         folder: &str,
         folder_meaning: FolderMeaning,
     ) -> Result<bool> {
@@ -582,7 +593,13 @@ impl Imap {
         let mut read_cnt = 0;
         loop {
             let (n, fetch_more) = self
-                .fetch_new_msg_batch(context, session, folder, folder_meaning)
+                .fetch_new_msg_batch(
+                    context,
+                    session,
+                    is_background_fetch,
+                    folder,
+                    folder_meaning,
+                )
                 .await?;
             read_cnt += n;
             if !fetch_more {
@@ -596,6 +613,7 @@ impl Imap {
         &mut self,
         context: &Context,
         session: &mut Session,
+        is_background_fetch: bool,
         folder: &str,
         folder_meaning: FolderMeaning,
     ) -> Result<(usize, bool)> {
@@ -615,10 +633,22 @@ impl Imap {
         let read_cnt = msgs.len();
 
         let mut uids_fetch = Vec::<u32>::with_capacity(msgs.len() + 1);
+        let mut available_post_msgs = Vec::<String>::with_capacity(msgs.len());
+        let mut download_when_normal_starts = Vec::<String>::with_capacity(msgs.len());
         let mut uid_message_ids = BTreeMap::new();
         let mut largest_uid_skipped = None;
         let delete_target = context.get_delete_msgs_target().await?;
 
+        let download_limit = {
+            let download_limit: Option<u32> =
+                context.get_config_parsed(Config::DownloadLimit).await?;
+            if download_limit == Some(0) {
+                None
+            } else {
+                download_limit
+            }
+        };
+
         // Store the info about IMAP messages in the database.
         for (uid, ref fetch_response) in msgs {
             let headers = match get_fetch_headers(fetch_response) {
@@ -630,6 +660,9 @@ impl Imap {
             };
 
             let message_id = prefetch_get_message_id(&headers);
+            let size = fetch_response
+                .size
+                .context("imap fetch response does not contain size")?;
 
             // Determine the target folder where the message should be moved to.
             //
@@ -704,8 +737,47 @@ impl Imap {
                 )
                 .await.context("prefetch_should_download")?
             {
-                uids_fetch.push(uid);
+                let fetch_now: bool = if headers
-                uid_message_ids.insert(uid, message_id);
+                    .get_header_value(HeaderDef::ChatIsPostMessage)
+                    .is_some()
+                {
+                    info!(context, "{} is a post message", message_id.clone());
+                    // This is a Post-Message
+                    available_post_msgs.push(message_id.clone());
+
+                    // whether it fits download size limit
+                    if download_limit.is_none_or(|download_limit| size < download_limit) {
+                        if is_background_fetch {
+                            download_when_normal_starts.push(message_id.clone());
+                            false
+                        } else {
+                            true
+                        }
+                    } else {
+                        false
+                    }
+                } else {
+                    info!(context, "{} is not a post message", message_id.clone());
+                    // This is not a Post-Message
+                    if is_background_fetch {
+                        if size < MAX_FETCH_MSG_SIZE {
+                            // may be a Pre-Message or a small normal message, fetch now
+                            true
+                        } else {
+                            // This is e.g. a classical email or large webxdc status update
+                            // Queue for full download, in order to prevent missing messages
+                            download_when_normal_starts.push(message_id.clone());
+                            false
+                        }
+                    } else {
+                        true
+                    }
+                };
+
+                if fetch_now {
+                    uids_fetch.push(uid);
+                    uid_message_ids.insert(uid, message_id);
+                }
             } else {
                 largest_uid_skipped = Some(uid);
             }
@@ -777,6 +849,31 @@ impl Imap {
 
         chat::mark_old_messages_as_noticed(context, received_msgs).await?;
 
+        // TODO: is there correct place for this?
+        if fetch_res.is_ok() {
+            info!(
+                context,
+                "available_post_msgs: {}, download_when_normal_starts: {}",
+                available_post_msgs.len(),
+                download_when_normal_starts.len()
+            );
+            for rfc724_mid in available_post_msgs {
+                context
+                    .sql
+                    .insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,))
+                    .await?;
+            }
+            for rfc724_mid in download_when_normal_starts {
+                context
+                    .sql
+                    .insert(
+                        "INSERT INTO download (rfc724_mid) VALUES (?)",
+                        (rfc724_mid,),
+                    )
+                    .await?;
+            }
+        }
+
         // Now fail if fetching failed, so we will
         // establish a new session if this one is broken.
         fetch_res?;

src/imap/scan_folders.rs

@@ -76,7 +76,7 @@ impl Imap {
                 && folder_meaning != FolderMeaning::Trash
                 && folder_meaning != FolderMeaning::Unknown
             {
-                self.fetch_move_delete(context, session, folder.name(), folder_meaning)
+                self.fetch_move_delete(context, session, false, folder.name(), folder_meaning)
                     .await
                     .context("Can't fetch new msgs in scanned folder")
                     .log_err(context)

src/imap/session.rs

@@ -17,6 +17,7 @@ use crate::tools;
 /// - Chat-Version to check if a message is a chat message
 /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
 ///   not necessarily sent by Delta Chat.
+/// - Chat-Is-Post-Message to skip it in background fetch or when it is too large
 const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
                               MESSAGE-ID \
                               DATE \
@@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE
                               FROM \
                               IN-REPLY-TO REFERENCES \
                               CHAT-VERSION \
+                              CHAT-IS-POST-MESSAGE \
                               AUTO-SUBMITTED \
                               AUTOCRYPT-SETUP-MESSAGE\
                               )])";

src/message.rs

@@ -8,6 +8,9 @@ use std::str;
 use anyhow::{Context as _, Result, ensure, format_err};
 use deltachat_contact_tools::{VcardContact, parse_vcard};
 use deltachat_derive::{FromSql, ToSql};
+use humansize::BINARY;
+use humansize::format_size;
+use num_traits::FromPrimitive;
 use serde::{Deserialize, Serialize};
 use tokio::{fs, io};
 
@@ -425,6 +428,10 @@ pub struct Message {
     pub(crate) ephemeral_timer: EphemeralTimer,
     pub(crate) ephemeral_timestamp: i64,
     pub(crate) text: String,
+    /// Text that is added to the end of Message.text
+    ///
+    /// Currently used for adding the download information on pre-messages
+    pub(crate) additional_text: String,
 
     /// Message subject.
     ///
@@ -483,7 +490,7 @@ impl Message {
             !id.is_special(),
             "Can not load special message ID {id} from DB"
         );
-        let msg = context
+        let mut msg = context
             .sql
             .query_row_optional(
                 concat!(
@@ -565,6 +572,7 @@ impl Message {
                         original_msg_id: row.get("original_msg_id")?,
                         mime_modified: row.get("mime_modified")?,
                         text,
+                        additional_text: String::new(),
                         subject: row.get("subject")?,
                         param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
                         hidden: row.get("hidden")?,
@@ -579,9 +587,48 @@ impl Message {
             .await
             .with_context(|| format!("failed to load message {id} from the database"))?;
 
+        if let Some(msg) = &mut msg {
+            msg.additional_text =
+                Self::get_additional_text(context, msg.download_state, &msg.param).await?;
+        }
+
         Ok(msg)
     }
 
+    /// Returns additional text which is appended to the message's text field
+    /// when it is loaded from the database.
+    /// Currently this is used to add infomation to pre-messages of what the download will be and how large it is
+    async fn get_additional_text(
+        context: &Context,
+        download_state: DownloadState,
+        param: &Params,
+    ) -> Result<String> {
+        if download_state != DownloadState::Done {
+            let file_size = param
+                .get(Param::PostMessageFileBytes)
+                .and_then(|s| s.parse().ok())
+                .map(|file_size: usize| format_size(file_size, BINARY))
+                .unwrap_or("?".to_owned());
+            let viewtype = param
+                .get_i64(Param::PostMessageViewtype)
+                .and_then(Viewtype::from_i64)
+                .unwrap_or(Viewtype::Unknown);
+            let file_name = param
+                .get(Param::Filename)
+                .map(sanitize_filename)
+                .unwrap_or("?".to_owned());
+
+            return match viewtype {
+                Viewtype::File => Ok(format!(" [{file_name} - {file_size}]")),
+                _ => {
+                    let translated_viewtype = viewtype.to_locale_string(context).await;
+                    Ok(format!(" [{translated_viewtype} - {file_size}]"))
+                }
+            };
+        }
+        Ok(String::new())
+    }
+
     /// Returns the MIME type of an attached file if it exists.
     ///
     /// If the MIME type is not known, the function guesses the MIME type
@@ -764,7 +811,7 @@ impl Message {
 
     /// Returns the text of the message.
     pub fn get_text(&self) -> String {
-        self.text.clone()
+        self.text.clone() + &self.additional_text
     }
 
     /// Returns message subject.
@@ -786,7 +833,18 @@ impl Message {
     }
 
     /// Returns the size of the file in bytes, if applicable.
+    /// If message is a pre-message, then this returns size of the to be downloaded file.
     pub async fn get_filebytes(&self, context: &Context) -> Result<Option<u64>> {
+        // if download state is not downloaded then return value from from params metadata
+        if self.download_state != DownloadState::Done {
+            if let Some(file_size) = self
+                .param
+                .get(Param::PostMessageFileBytes)
+                .and_then(|s| s.parse().ok())
+            {
+                return Ok(Some(file_size));
+            }
+        }
         if let Some(path) = self.param.get_file_path(context)? {
             Ok(Some(get_filebytes(context, &path).await.with_context(
                 || format!("failed to get {} size in bytes", path.display()),
@@ -796,6 +854,22 @@ impl Message {
         }
     }
 
+    /// If message is a Pre-Message,
+    /// then this returns the viewtype it will have when it is downloaded.
+    #[cfg(test)]
+    pub(crate) fn get_post_message_viewtype(&self) -> Option<Viewtype> {
+        if self.download_state != DownloadState::Done {
+            if let Some(viewtype) = self
+                .param
+                .get_i64(Param::PostMessageViewtype)
+                .and_then(Viewtype::from_i64)
+            {
+                return Some(viewtype);
+            }
+        }
+        None
+    }
+
     /// Returns width of associated image or video file.
     pub fn get_width(&self) -> i32 {
         self.param.get_int(Param::Width).unwrap_or_default()
@@ -1676,9 +1750,17 @@ pub async fn delete_msgs_ex(
         let update_db = |trans: &mut rusqlite::Transaction| {
             trans.execute(
                 "UPDATE imap SET target=? WHERE rfc724_mid=?",
-                (target, msg.rfc724_mid),
+                (target, &msg.rfc724_mid),
             )?;
             trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?;
+            trans.execute(
+                "DELETE FROM download WHERE rfc724_mid=?",
+                (&msg.rfc724_mid,),
+            )?;
+            trans.execute(
+                "DELETE FROM available_post_msgs WHERE rfc724_mid=?",
+                (&msg.rfc724_mid,),
+            )?;
             Ok(())
         };
         if let Err(e) = context.sql.transaction(update_db).await {

src/mimefactory.rs

@@ -21,6 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG};
 use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
 use crate::contact::{Contact, ContactId, Origin};
 use crate::context::Context;
+use crate::download::pre_msg_metadata::PreMsgMetadata;
 use crate::e2ee::EncryptHelper;
 use crate::ensure_and_debug_assert;
 use crate::ephemeral::Timer as EphemeralTimer;
@@ -61,11 +62,11 @@ pub enum Loaded {
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum PreMessageMode {
-    /// adds the Chat-Is-Full-Message header in unprotected part
+    /// adds the Chat-Is-Post-Message header in unprotected part
-    FullMessage,
+    PostMessage,
-    /// adds the Chat-Full-Message-ID header to protected part
+    /// adds the Chat-Post-Message-ID header to protected part
     /// also adds metadata and explicitly excludes attachment
-    PreMessage { full_msg_rfc724_mid: String },
+    PreMessage { post_msg_rfc724_mid: String },
 }
 
 /// Helper to construct mime messages.
@@ -156,7 +157,7 @@ pub struct MimeFactory {
     /// This field is used to sustain the topic id of webxdcs needed for peer channels.
     webxdc_topic: Option<TopicId>,
 
-    /// This field is used when this is either a pre-message or a full-message.
+    /// This field is used when this is either a pre-message or a Post-Message.
     pre_message_mode: Option<PreMessageMode>,
 }
 
@@ -996,18 +997,18 @@ impl MimeFactory {
             mail_builder::headers::raw::Raw::new("1.0").into(),
         ));
 
-        if self.pre_message_mode == Some(PreMessageMode::FullMessage) {
+        if self.pre_message_mode == Some(PreMessageMode::PostMessage) {
             unprotected_headers.push((
-                "Chat-Is-Full-Message",
+                "Chat-Is-Post-Message",
                 mail_builder::headers::raw::Raw::new("1").into(),
             ));
         } else if let Some(PreMessageMode::PreMessage {
-            full_msg_rfc724_mid,
+            post_msg_rfc724_mid,
         }) = self.pre_message_mode.clone()
         {
             protected_headers.push((
-                "Chat-Full-Message-ID",
+                "Chat-Post-Message-ID",
-                mail_builder::headers::message_id::MessageId::new(full_msg_rfc724_mid).into(),
+                mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid).into(),
             ));
         }
 
@@ -1152,7 +1153,7 @@ impl MimeFactory {
                         for (addr, key) in &encryption_pubkeys {
                             let fingerprint = key.dc_fingerprint().hex();
                             let cmd = msg.param.get_cmd();
-                            if self.pre_message_mode == Some(PreMessageMode::FullMessage) {
+                            if self.pre_message_mode == Some(PreMessageMode::PostMessage) {
                                 continue;
                             }
 
@@ -1913,6 +1914,14 @@ impl MimeFactory {
         // add attachment part
         if msg.viewtype.has_file() {
             if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode {
+                let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else {
+                    bail!("failed to generate metadata for pre-message")
+                };
+
+                headers.push((
+                    HeaderDef::ChatPostMessageMetadata.into(),
+                    mail_builder::headers::raw::Raw::new(metadata.to_header_value()?).into(),
+                ));
                 // TODO: generate thumbnail and attach it instead (if it makes sense)
             } else {
                 let file_part = build_body_file(context, &msg).await?;
@@ -1963,7 +1972,7 @@ impl MimeFactory {
         }
 
         self.attach_selfavatar =
-            self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::FullMessage);
+            self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::PostMessage);
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_avatar_file(context, &path).await {
@@ -2038,13 +2047,13 @@ impl MimeFactory {
         self.encryption_pubkeys.is_some()
     }
 
-    pub fn set_as_full_message(&mut self) {
+    pub fn set_as_post_message(&mut self) {
-        self.pre_message_mode = Some(PreMessageMode::FullMessage);
+        self.pre_message_mode = Some(PreMessageMode::PostMessage);
     }
 
-    pub fn set_as_pre_message_for(&mut self, full_message: &RenderedEmail) {
+    pub fn set_as_pre_message_for(&mut self, post_message: &RenderedEmail) {
         self.pre_message_mode = Some(PreMessageMode::PreMessage {
-            full_msg_rfc724_mid: full_message.rfc724_mid.clone(),
+            post_msg_rfc724_mid: post_message.rfc724_mid.clone(),
         });
     }
 }

src/mimeparser.rs

@@ -23,6 +23,7 @@ use crate::contact::ContactId;
 use crate::context::Context;
 use crate::decrypt::{try_decrypt, validate_detached_signature};
 use crate::dehtml::dehtml;
+use crate::download::pre_msg_metadata::PreMsgMetadata;
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring};
@@ -147,6 +148,23 @@ pub(crate) struct MimeMessage {
     /// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized
     /// clocks, but not too much.
     pub(crate) timestamp_sent: i64,
+
+    pub(crate) pre_message: Option<PreMessageMode>,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) enum PreMessageMode {
+    /// This is Post-Message
+    /// it replaces it's Pre-Message attachment if it exists already,
+    /// and if the Pre-Message does not exist it is treated as normal message
+    PostMessage,
+    /// This is a Pre-Message,
+    /// it adds a message preview for a Post-Message
+    /// and it is ignored if the Post-Message was downloaded already
+    PreMessage {
+        post_msg_rfc724_mid: String,
+        metadata: Option<PreMsgMetadata>,
+    },
 }
 
 #[derive(Debug, PartialEq)]
@@ -239,6 +257,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
 impl MimeMessage {
     /// Parse a mime message.
+    ///
+    /// This method has some side-effects,
+    /// such as saving blobs and saving found public keys to the database.
     pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
@@ -346,6 +367,16 @@ impl MimeMessage {
 
         let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into());
 
+        let mut pre_message = if mail
+            .headers
+            .get_header_value(HeaderDef::ChatIsPostMessage)
+            .is_some()
+        {
+            Some(PreMessageMode::PostMessage)
+        } else {
+            None
+        };
+
         let mail_raw; // Memory location for a possible decrypted message.
         let decrypted_msg; // Decrypted signed OpenPGP message.
         let secrets: Vec<String> = context
@@ -574,6 +605,37 @@ impl MimeMessage {
             signatures.clear();
         }
 
+        if let (Ok(mail), true) = (mail, is_encrypted) {
+            if let Some(post_msg_rfc724_mid) =
+                mail.headers.get_header_value(HeaderDef::ChatPostMessageId)
+            {
+                let post_msg_rfc724_mid = parse_message_id(&post_msg_rfc724_mid)?;
+                let metadata = if let Some(value) = mail
+                    .headers
+                    .get_header_value(HeaderDef::ChatPostMessageMetadata)
+                {
+                    match PreMsgMetadata::try_from_header_value(&value) {
+                        Ok(metadata) => Some(metadata),
+                        Err(error) => {
+                            error!(
+                                context,
+                                "failed to parse metadata header in pre-message: {error:#?}"
+                            );
+                            None
+                        }
+                    }
+                } else {
+                    warn!(context, "expected pre-message to have metadata header");
+                    None
+                };
+
+                pre_message = Some(PreMessageMode::PreMessage {
+                    post_msg_rfc724_mid,
+                    metadata,
+                });
+            }
+        }
+
         let mut parser = MimeMessage {
             parts: Vec::new(),
             headers,
@@ -609,6 +671,7 @@ impl MimeMessage {
             is_bot: None,
             timestamp_rcvd,
             timestamp_sent,
+            pre_message,
         };
 
         match mail {

src/param.rs

@@ -251,6 +251,13 @@ pub enum Param {
 
     /// For info messages: Contact ID in added or removed to a group.
     ContactAddedRemoved = b'5',
+
+    /// For (pre-)Message: ViewType of the Post-Message,
+    /// because pre message is always `Viewtype::Text`.
+    PostMessageViewtype = b'8',
+
+    /// For (pre-)Message: File byte size of Post-Message attachment
+    PostMessageFileBytes = b'9',
 }
 
 /// An object for handling key=value parameter lists.
@@ -441,6 +448,15 @@ impl Params {
         }
         self
     }
+
+    /// Merge in parameters from other Params struct,
+    /// overwriting the keys that are in both
+    /// with the values from the new Params struct.
+    pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self {
+        let mut new_params = new_params;
+        self.inner.append(&mut new_params.inner);
+        self
+    }
 }
 
 #[cfg(test)]
@@ -503,4 +519,18 @@ mod tests {
         assert_eq!(p.get(Param::Height), Some("14"));
         Ok(())
     }
+
+    #[test]
+    fn test_merge() -> Result<()> {
+        let mut p = Params::from_str("w=12\na=5\nh=14")?;
+        let p2 = Params::from_str("L=1\nh=17")?;
+        assert_eq!(p.len(), 3);
+        p.merge_in_from_params(p2);
+        assert_eq!(p.len(), 4);
+        assert_eq!(p.get(Param::Width), Some("12"));
+        assert_eq!(p.get(Param::Height), Some("17"));
+        assert_eq!(p.get(Param::Forwarded), Some("5"));
+        assert_eq!(p.get(Param::IsEdited), Some("1"));
+        Ok(())
+    }
 }

src/receive_imf.rs

@@ -20,16 +20,14 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX,
 use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified};
 use crate::context::Context;
 use crate::debug_logging::maybe_set_logging_xdc_inner;
-use crate::download::DownloadState;
+use crate::download::{DownloadState, premessage_is_downloaded_for};
 use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table};
 use crate::key::{DcKey, Fingerprint};
 use crate::key::{self_fingerprint, self_fingerprint_opt};
-use crate::log::LogExt;
+use crate::log::{LogExt as _, warn};
-use crate::log::warn;
-use crate::logged_debug_assert;
 use crate::message::{
     self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists,
 };
@@ -47,6 +45,7 @@ use crate::tools::{
     self, buf_compress, normalize_text, remove_subject_prefix, validate_broadcast_secret,
 };
 use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location};
+use crate::{logged_debug_assert, mimeparser};
 
 /// This is the struct that is returned after receiving one email (aka MIME message).
 ///
@@ -517,7 +516,15 @@ pub(crate) async fn receive_imf_inner(
     // check, if the mail is already in our database.
     // make sure, this check is done eg. before securejoin-processing.
     let (replace_msg_id, replace_chat_id);
-    if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
+    if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) {
+        // Post-Message just replace the attachment and mofified Params, not the whole message
+        // This is done in the `handle_post_message` method.
+        replace_msg_id = None;
+        replace_chat_id = None;
+    } else if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
+        // This code handles the download of old partial download stub messages
+        // It will be removed after a transitioning period,
+        // after we have released a few versions with pre-messages
         replace_msg_id = Some(old_msg_id);
         replace_chat_id = if let Some(msg) = Message::load_from_db_optional(context, old_msg_id)
             .await?
@@ -526,8 +533,6 @@ pub(crate) async fn receive_imf_inner(
             // the message was partially downloaded before and is fully downloaded now.
             info!(context, "Message already partly in DB, replacing.");
             Some(msg.chat_id)
-
-            // TODO: look at this place
         } else {
             // The message was already fully downloaded
             // or cannot be loaded because it is deleted.
@@ -1097,6 +1102,38 @@ async fn decide_chat_assignment(
     {
         info!(context, "Chat edit/delete/iroh/sync message (TRASH).");
         true
+    } else if let Some(pre_message) = &mime_parser.pre_message {
+        use crate::mimeparser::PreMessageMode::*;
+        match pre_message {
+            PostMessage => {
+                // if pre message exist, then trash after replacing, otherwise treat as normal message
+                let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?;
+                info!(
+                    context,
+                    "Message is a Post-Message ({}).",
+                    if pre_message_exists {
+                        "pre-message exists already, so trash after replacing attachment"
+                    } else {
+                        "no pre-message -> Keep"
+                    }
+                );
+                pre_message_exists
+            }
+            PreMessage {
+                post_msg_rfc724_mid,
+                ..
+            } => {
+                // if post message already exists, then trash/ignore
+                let post_msg_exists =
+                    premessage_is_downloaded_for(context, post_msg_rfc724_mid).await?;
+                info!(
+                    context,
+                    "Message is a Pre-Message (post_msg_exists:{post_msg_exists})."
+                );
+                post_msg_exists
+                // TODO find out if trashing affects multi device usage?
+            }
+        }
     } else if mime_parser.is_system_message == SystemMessage::CallAccepted
         || mime_parser.is_system_message == SystemMessage::CallEnded
     {
@@ -1909,6 +1946,7 @@ async fn add_parts(
     }
 
     handle_edit_delete(context, mime_parser, from_id).await?;
+    handle_post_message(context, mime_parser, from_id).await?;
 
     if mime_parser.is_system_message == SystemMessage::CallAccepted
         || mime_parser.is_system_message == SystemMessage::CallEnded
@@ -1992,6 +2030,14 @@ async fn add_parts(
             }
         };
 
+        if let Some(mimeparser::PreMessageMode::PreMessage {
+            metadata: Some(metadata),
+            ..
+        }) = &mime_parser.pre_message
+        {
+            param.apply_from_pre_msg_metadata(metadata);
+        };
+
         // If you change which information is skipped if the message is trashed,
         // also change `MsgId::trash()` and `delete_expired_messages()`
         let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified);
@@ -2035,14 +2081,20 @@ RETURNING id
 "#)?;
                 let row_id: MsgId = stmt.query_row(params![
                     replace_msg_id,
-                    rfc724_mid_orig,
+                    if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message {
+                        post_msg_rfc724_mid
+                    } else { rfc724_mid_orig },
                     if trash { DC_CHAT_ID_TRASH } else { chat_id },
                     if trash { ContactId::UNDEFINED } else { from_id },
                     if trash { ContactId::UNDEFINED } else { to_id },
                     sort_timestamp,
                     if trash { 0 } else { mime_parser.timestamp_sent },
                     if trash { 0 } else { mime_parser.timestamp_rcvd },
-                    if trash { Viewtype::Unknown } else { typ },
+                    if trash {
+                        Viewtype::Unknown
+                    } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message {
+                        Viewtype::Text
+                    } else { typ },
                     if trash { MessageState::Undefined } else { state },
                     if trash { MessengerMessage::No } else { is_dc_message },
                     if trash || hidden { "" } else { msg },
@@ -2054,7 +2106,11 @@ RETURNING id
                         param.to_string()
                     },
                     !trash && hidden,
-                    if trash { 0 } else { part.bytes as isize },
+                    if trash {
+                        0
+                    } else {
+                        part.bytes as isize
+                    },
                     if save_mime_modified && !(trash || hidden) {
                         mime_headers.clone()
                     } else {
@@ -2070,6 +2126,8 @@ RETURNING id
                         DownloadState::Done
                     } else if mime_parser.decrypting_failed {
                         DownloadState::Undecipherable
+                    } else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message {
+                        DownloadState::Available
                     } else {
                         DownloadState::Done
                     },
@@ -2262,6 +2320,82 @@ async fn handle_edit_delete(
     Ok(())
 }
 
+async fn handle_post_message(
+    context: &Context,
+    mime_parser: &MimeMessage,
+    from_id: ContactId,
+) -> Result<()> {
+    if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message {
+        // if Pre-Message exist, replace attachment
+        // only replacing attachment ensures that doesn't overwrite the text if it was edited before.
+        let rfc724_mid = mime_parser
+            .get_rfc724_mid()
+            .context("expected Post-Message to have a message id")?;
+
+        let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else {
+            warn!(
+                context,
+                "Download Post-Message: Database entry does not exist."
+            );
+            return Ok(());
+        };
+        let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else {
+            // else: message is processed like a normal message
+            warn!(
+                context,
+                "Download Post-Message: pre message was not downloaded, yet so treat as normal message"
+            );
+            return Ok(());
+        };
+
+        if original_msg.from_id != from_id {
+            warn!(context, "Download Post-Message: Bad sender.");
+            return Ok(());
+        }
+        if let Some(part) = mime_parser.parts.first() {
+            if !part.typ.has_file() {
+                warn!(
+                    context,
+                    "Download Post-Message: First mime part's message-viewtype has no file"
+                );
+                return Ok(());
+            }
+
+            let edit_msg_showpadlock = part
+                .param
+                .get_bool(Param::GuaranteeE2ee)
+                .unwrap_or_default();
+
+            if edit_msg_showpadlock || !original_msg.get_showpadlock() {
+                let mut new_params = original_msg.param.clone();
+                new_params
+                    .merge_in_from_params(part.param.clone())
+                    .remove(Param::PostMessageFileBytes)
+                    .remove(Param::PostMessageViewtype);
+                context
+                        .sql
+                        .execute(
+                            "UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?",
+                            (
+                                new_params.to_string(),
+                                part.typ,
+                                part.bytes as isize,
+                                part.error.as_deref().unwrap_or_default(),
+                                DownloadState::Done as u32,
+                                original_msg.id,
+                            ),
+                        )
+                        .await?;
+                context.emit_msgs_changed(original_msg.chat_id, original_msg.id);
+            } else {
+                warn!(context, "Download Post-Message: Not encrypted.");
+            }
+        }
+    }
+
+    Ok(())
+}
+
 async fn tweak_sort_timestamp(
     context: &Context,
     mime_parser: &mut MimeMessage,

src/scheduler.rs

@@ -14,13 +14,12 @@ pub(crate) use self::connectivity::ConnectivityStore;
 use crate::config::{self, Config};
 use crate::contact::{ContactId, RecentlySeenLoop};
 use crate::context::Context;
-use crate::download::{DownloadState, download_msg};
+use crate::download::{download_known_post_messages_without_pre_message, download_msgs};
 use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::events::EventType;
 use crate::imap::{FolderMeaning, Imap, session::Session};
 use crate::location;
 use crate::log::{LogExt, warn};
-use crate::message::MsgId;
 use crate::smtp::{Smtp, send_smtp_messages};
 use crate::sql;
 use crate::stats::maybe_send_stats;
@@ -349,38 +348,6 @@ pub(crate) struct Scheduler {
     recently_seen_loop: RecentlySeenLoop,
 }
 
-async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> {
-    let msg_ids = context
-        .sql
-        .query_map_vec("SELECT msg_id FROM download", (), |row| {
-            let msg_id: MsgId = row.get(0)?;
-            Ok(msg_id)
-        })
-        .await?;
-
-    for msg_id in msg_ids {
-        if let Err(err) = download_msg(context, msg_id, session).await {
-            warn!(context, "Failed to download message {msg_id}: {:#}.", err);
-
-            // Update download state to failure
-            // so it can be retried.
-            //
-            // On success update_download_state() is not needed
-            // as receive_imf() already
-            // set the state and emitted the event.
-            msg_id
-                .update_download_state(context, DownloadState::Failure)
-                .await?;
-        }
-        context
-            .sql
-            .execute("DELETE FROM download WHERE msg_id=?", (msg_id,))
-            .await?;
-    }
-
-    Ok(())
-}
-
 async fn inbox_loop(
     ctx: Context,
     started: oneshot::Sender<()>,
@@ -606,7 +573,7 @@ async fn fetch_idle(
     {
         // Fetch the watched folder.
         connection
-            .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
+            .fetch_move_delete(ctx, &mut session, true, &watch_folder, folder_meaning)
             .await
             .context("fetch_move_delete")?;
 
@@ -617,6 +584,11 @@ async fn fetch_idle(
         delete_expired_imap_messages(ctx)
             .await
             .context("delete_expired_imap_messages")?;
+
+        //-------
+        // TODO: verify that this is the correct position for this call
+        // in order to guard against lost pre-messages:
+        download_known_post_messages_without_pre_message(ctx, &mut session).await?;
     } else if folder_config == Config::ConfiguredInboxFolder {
         session.last_full_folder_scan.lock().await.take();
     }
@@ -645,7 +617,7 @@ async fn fetch_idle(
                 // no new messages. We want to select the watched folder anyway before going IDLE
                 // there, so this does not take additional protocol round-trip.
                 connection
-                    .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
+                    .fetch_move_delete(ctx, &mut session, true, &watch_folder, folder_meaning)
                     .await
                     .context("fetch_move_delete after scan_folders")?;
             }
@@ -702,6 +674,7 @@ async fn fetch_idle(
     Ok(session)
 }
 
+/// The simplified IMAP IDLE loop to watch non primary folders (non-inbox folders)
 async fn simple_imap_loop(
     ctx: Context,
     started: oneshot::Sender<()>,

src/sql/migrations.rs

@@ -1504,6 +1504,26 @@ ORDER BY last_seen DESC LIMIT 10000
             .await?;
     }
 
+    inc_and_check(&mut migration_version, 144)?;
+    if dbversion < migration_version {
+        sql.execute_migration(
+            "CREATE TABLE download_new (
+                rfc724_mid TEXT NOT NULL
+            );
+            INSERT INTO download_new (rfc724_mid)
+             SELECT m.rfc724_mid FROM download d
+             JOIN msgs m ON d.msg_id = m.id
+             WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != '';
+            DROP TABLE download;
+            ALTER TABLE download_new RENAME TO download;
+            CREATE TABLE available_post_msgs (
+                rfc724_mid TEXT NOT NULL
+            );",
+            migration_version,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?

src/stock_str.rs

@@ -1242,6 +1242,26 @@ pub(crate) async fn chat_unencrypted_explanation(context: &Context) -> String {
     translated(context, StockMessage::ChatUnencryptedExplanation).await
 }
 
+impl Viewtype {
+    /// returns Localized name for message viewtype
+    pub async fn to_locale_string(&self, context: &Context) -> String {
+        match self {
+            Viewtype::Image => image(context).await,
+            Viewtype::Gif => gif(context).await,
+            Viewtype::Sticker => sticker(context).await,
+            Viewtype::Audio => audio(context).await,
+            Viewtype::Voice => voice_message(context).await,
+            Viewtype::Video => video(context).await,
+            Viewtype::File => file(context).await,
+            Viewtype::Webxdc => "Mini App".to_owned(), // TODO stock-string
+            Viewtype::Vcard => "Contact".to_owned(),   // TODO stock-string
+            // OPTIONAL TODO - would not be used right now,
+            // because this method is only used for metadata on large attachments
+            Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(),
+        }
+    }
+}
+
 impl Context {
     /// Set the stock string for the [StockMessage].
     ///

src/test_utils.rs

@@ -1719,6 +1719,21 @@ Until the false-positive is fixed:
     }
 }
 
+/// Method to create a test image file
+pub(crate) fn create_test_image(width: u32, height: u32) -> anyhow::Result<Vec<u8>> {
+    use image::{ImageBuffer, Rgb, RgbImage};
+    use std::io::Cursor;
+
+    let mut img: RgbImage = ImageBuffer::new(width, height);
+    // fill with some pattern so it stays large after compression
+    for (x, y, pixel) in img.enumerate_pixels_mut() {
+        *pixel = Rgb([(x % 255) as u8, (x + y % 255) as u8, (y % 255) as u8]);
+    }
+    let mut bytes: Vec<u8> = Vec::new();
+    img.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?;
+    Ok(bytes)
+}
+
 mod tests {
     use super::*;
 

src/tests.rs

@@ -1,3 +1,4 @@
 mod account_events;
 mod aeap;
+mod pre_messages;
 mod verified_chats;

src/tests/pre_messages/additional_text.rs

@@ -0,0 +1,40 @@
+use anyhow::Result;
+use pretty_assertions::assert_eq;
+
+use crate::message::Viewtype;
+use crate::test_utils::TestContextManager;
+use crate::tests::pre_messages::util::{
+    send_large_file_message, send_large_image_message, send_large_webxdc_message,
+};
+
+/// Test the addition of the download info to message text
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_additional_text_on_different_viewtypes() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let a_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    tcm.section("Test metadata preview text for File");
+    let (pre_message, _, _) =
+        send_large_file_message(alice, a_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
+    let msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(msg.text, "test".to_owned());
+    assert_eq!(msg.get_text(), "test [test.bin - 976.56 KiB]".to_owned());
+
+    tcm.section("Test metadata preview text for webxdc app");
+    let (pre_message, _, _) = send_large_webxdc_message(alice, a_group_id).await?;
+    let msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(msg.text, "test".to_owned());
+    assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Webxdc));
+    assert_eq!(msg.get_text(), "test [Mini App - 976.68 KiB]".to_owned());
+
+    tcm.section("Test metadata preview text for Image");
+
+    let (pre_message, _, _) = send_large_image_message(alice, a_group_id).await?;
+    let msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(msg.text, "test".to_owned());
+    assert_eq!(msg.get_text(), "test [Image - 146.12 KiB]".to_owned());
+
+    Ok(())
+}

src/tests/pre_messages/forward_and_save.rs

@@ -0,0 +1,122 @@
+//! Tests about forwarding and saving Pre-Messages
+use anyhow::Result;
+use pretty_assertions::assert_eq;
+
+use crate::chat::{self};
+use crate::chat::{forward_msgs, save_msgs};
+use crate::chatlist::get_last_message_for_chat;
+use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD};
+use crate::message::{Message, Viewtype};
+use crate::test_utils::TestContextManager;
+
+/// Test that forwarding Pre-Message should forward additional text to not be empty
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_forwarding_pre_message_empty_text() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let pre_message = {
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?;
+        assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+        let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?;
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+        assert_eq!(smtp_rows.len(), 2);
+        smtp_rows.first().expect("Pre-Message exists").to_owned()
+    };
+
+    let bob_msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(bob_msg.download_state, DownloadState::Available);
+    bob_msg.chat_id.accept(bob).await?;
+    tcm.section("forward pre message and check it on bobs side");
+    forward_msgs(bob, &[bob_msg.id], bob_msg.chat_id).await?;
+    let forwarded_msg_id = get_last_message_for_chat(bob, bob_msg.chat_id)
+        .await?
+        .unwrap();
+    let forwarded_msg = Message::load_from_db(bob, forwarded_msg_id).await?;
+    assert_eq!(forwarded_msg.is_forwarded(), true);
+    assert_eq!(forwarded_msg.download_state(), DownloadState::Done);
+    assert_eq!(
+        forwarded_msg
+            .param
+            .exists(crate::param::Param::PostMessageFileBytes),
+        false,
+        "PostMessageFileBytes not set"
+    );
+    assert_eq!(
+        forwarded_msg
+            .param
+            .exists(crate::param::Param::PostMessageViewtype),
+        false,
+        "PostMessageViewtype not set"
+    );
+    assert_eq!(
+        forwarded_msg.get_text(),
+        " [test.bin - 976.56 KiB]".to_owned()
+    );
+    assert_eq!(forwarded_msg.get_viewtype(), Viewtype::Text);
+    assert!(forwarded_msg.additional_text.is_empty());
+    tcm.section("check it on alices side");
+    let sent_forward_msg = bob.pop_sent_msg().await;
+    let alice_forwarded_msg = alice.recv_msg(&sent_forward_msg).await;
+    assert!(alice_forwarded_msg.additional_text.is_empty());
+    assert_eq!(alice_forwarded_msg.is_forwarded(), true);
+    assert_eq!(alice_forwarded_msg.download_state(), DownloadState::Done);
+    assert_eq!(
+        alice_forwarded_msg
+            .param
+            .exists(crate::param::Param::PostMessageFileBytes),
+        false,
+        "PostMessageFileBytes not set"
+    );
+    assert_eq!(
+        alice_forwarded_msg
+            .param
+            .exists(crate::param::Param::PostMessageViewtype),
+        false,
+        "PostMessageViewtype not set"
+    );
+    assert_eq!(
+        alice_forwarded_msg.get_text(),
+        " [test.bin - 976.56 KiB]".to_owned()
+    );
+
+    Ok(())
+}
+
+/// Test that forwarding Pre-Message should forward additional text to not be empty
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_saving_pre_message_empty_text() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let pre_message = {
+        let mut msg = Message::new(Viewtype::File);
+        msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?;
+        assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+        let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?;
+        let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+        assert_eq!(smtp_rows.len(), 2);
+        smtp_rows.first().expect("Pre-Message exists").to_owned()
+    };
+
+    let bob_msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(bob_msg.download_state, DownloadState::Available);
+    bob_msg.chat_id.accept(bob).await?;
+    tcm.section("save pre message and check it");
+    save_msgs(bob, &[bob_msg.id]).await?;
+    let saved_msg_id = get_last_message_for_chat(bob, bob.get_self_chat().await.id)
+        .await?
+        .unwrap();
+    let saved_msg = Message::load_from_db(bob, saved_msg_id).await?;
+    assert!(saved_msg.additional_text.is_empty());
+    assert!(saved_msg.get_original_msg_id(bob).await?.is_some());
+    assert_eq!(saved_msg.download_state(), DownloadState::Done);
+    assert_eq!(saved_msg.get_text(), " [test.bin - 976.56 KiB]".to_owned());
+
+    Ok(())
+}

src/tests/pre_messages/legacy.rs

@@ -0,0 +1,61 @@
+//! Test that downloading old stub messages still works
+use anyhow::Result;
+
+use crate::download::DownloadState;
+use crate::receive_imf::receive_imf_from_inbox;
+use crate::test_utils::TestContext;
+
+// The code for downloading stub messages stays
+// during the transition perios to pre-messages
+// so people can still download their files shortly after they updated.
+// After there are a few release with pre-message rolled out,
+// we will remove the ability to download stub messages and replace the following test
+// so it checks that it doesn't crash or that the messages are replaced by sth.
+// like "download failed/expired, please ask sender to send it again"
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_download_stub_message() -> Result<()> {
+    let t = TestContext::new_alice().await;
+
+    let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+             From: bob@example.com\n\
+             To: alice@example.org\n\
+             Subject: foo\n\
+             Message-ID: <Mr.12345678901@example.com>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\
+             Content-Type: text/plain";
+
+    t.sql
+        .execute(
+            r#"INSERT INTO chats VALUES(
+                    11001,100,'bob@example.com',0,'',2,'',
+                    replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0,'');
+                "#,
+            (),
+        )
+        .await?;
+    t.sql.execute(r#"INSERT INTO msgs VALUES(
+                11001,'Mr.12345678901@example.com','',0,
+                11001,11001,1,1763151754,10,10,1,0,
+                '[97.66 KiB message]','','',0,1763151754,1763151754,0,X'',
+                '','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0);
+        "#, ()).await?;
+    let msg = t.get_last_msg().await;
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.get_subject(), "foo");
+    assert!(msg.get_text().contains("[97.66 KiB message]"));
+
+    receive_imf_from_inbox(
+        &t,
+        "Mr.12345678901@example.com",
+        format!("{header}\n\n100k text...").as_bytes(),
+        false,
+    )
+    .await?;
+    let msg = t.get_last_msg().await;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.get_subject(), "foo");
+    assert_eq!(msg.get_text(), "100k text...");
+
+    Ok(())
+}

src/tests/pre_messages/mod.rs

@@ -0,0 +1,6 @@
+mod additional_text;
+mod forward_and_save;
+mod legacy;
+mod receiving;
+mod sending;
+mod util;

src/tests/pre_messages/receiving.rs

@@ -0,0 +1,522 @@
+//! Tests about receiving Pre-Messages and Post-Message
+use anyhow::Result;
+use pretty_assertions::assert_eq;
+
+use crate::EventType;
+use crate::chat;
+use crate::contact;
+use crate::download::{
+    DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, pre_msg_metadata::PreMsgMetadata,
+};
+use crate::message::{Message, MessageState, Viewtype, delete_msgs, markseen_msgs};
+use crate::mimeparser::MimeMessage;
+use crate::param::Param;
+use crate::reaction::{get_msg_reactions, send_reaction};
+use crate::test_utils::TestContextManager;
+use crate::tests::pre_messages::util::{
+    send_large_file_message, send_large_image_message, send_large_webxdc_message,
+};
+use crate::webxdc::StatusUpdateSerial;
+
+/// Test that mimeparser can correctly detect and parse pre-messages and Post-Messages
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_mimeparser_pre_message_and_post_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, post_message, _alice_msg_id) =
+        send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
+            .await?;
+
+    let parsed_pre_message = MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?;
+    let parsed_post_message = MimeMessage::from_bytes(bob, post_message.payload.as_bytes()).await?;
+
+    assert_eq!(
+        parsed_post_message.pre_message,
+        Some(crate::mimeparser::PreMessageMode::PostMessage)
+    );
+
+    assert_eq!(
+        parsed_pre_message.pre_message,
+        Some(crate::mimeparser::PreMessageMode::PreMessage {
+            post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(),
+            metadata: Some(PreMsgMetadata {
+                size: 1_000_000,
+                viewtype: Viewtype::File,
+                filename: "test.bin".to_string(),
+                dimensions: None,
+                duration: None
+            })
+        })
+    );
+
+    Ok(())
+}
+
+/// Test receiving pre-messages and creation of the placeholder message with the metadata
+/// for file attachment
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_pre_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, _post_message, _alice_msg_id) =
+        send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
+            .await?;
+
+    let msg = bob.recv_msg(&pre_message).await;
+
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.viewtype, Viewtype::Text);
+    assert_eq!(msg.text, "test".to_owned());
+
+    // test that metadata is correctly returned by methods
+    assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000));
+    assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::File));
+    assert_eq!(msg.get_filename(), Some("test.bin".to_owned()));
+
+    Ok(())
+}
+
+/// Test receiving the Post-Message after receiving the pre-message
+/// for file attachment
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_pre_message_and_dl_post_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, post_message, _alice_msg_id) =
+        send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
+            .await?;
+
+    let msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.viewtype, Viewtype::Text);
+    assert!(msg.param.exists(Param::PostMessageViewtype));
+    assert!(msg.param.exists(Param::PostMessageFileBytes));
+    assert_eq!(msg.text, "test".to_owned());
+    let _ = bob.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(bob, msg.id).await?;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.viewtype, Viewtype::File);
+    assert_eq!(msg.param.exists(Param::PostMessageViewtype), false);
+    assert_eq!(msg.param.exists(Param::PostMessageFileBytes), false);
+    assert_eq!(msg.text, "test".to_owned());
+    Ok(())
+}
+
+/// Test out of order receiving. Post-Message is received & downloaded before pre-message.
+/// In that case pre-message shall be trashed.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_out_of_order_receiving() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, post_message, _alice_msg_id) =
+        send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
+            .await?;
+
+    let msg = bob.recv_msg(&post_message).await;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.viewtype, Viewtype::File);
+    let _ = bob.recv_msg_trash(&pre_message).await;
+    Ok(())
+}
+
+/// Test receiving the Post-Message after receiving an edit after receiving the pre-message
+/// for file attachment
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_pre_message_then_edit_and_then_dl_post_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, post_message, alice_msg_id) =
+        send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
+            .await?;
+
+    chat::send_edit_request(alice, alice_msg_id, "new_text".to_owned()).await?;
+    let edit_request = alice.pop_sent_msg().await;
+
+    let msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.text, "test".to_owned());
+    let _ = bob.recv_msg_trash(&edit_request).await;
+    let msg = Message::load_from_db(bob, msg.id).await?;
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.text, "new_text".to_owned());
+    let _ = bob.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(bob, msg.id).await?;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.viewtype, Viewtype::File);
+    assert_eq!(msg.text, "new_text".to_owned());
+    Ok(())
+}
+
+/// Process normal message with file attachment (neither post nor pre message)
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_normal_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let mut msg = Message::new(Viewtype::File);
+    msg.set_file_from_bytes(
+        alice,
+        "test.bin",
+        &vec![0u8; (PRE_MSG_ATTACHMENT_SIZE_THRESHOLD - 10_000) as usize],
+        None,
+    )?;
+    msg.set_text("test".to_owned());
+    let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?;
+
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+    assert_eq!(smtp_rows.len(), 1);
+    let message = smtp_rows.first().expect("message exists");
+
+    let msg = bob.recv_msg(message).await;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+    assert_eq!(msg.viewtype, Viewtype::File);
+    assert_eq!(msg.text, "test".to_owned());
+    Ok(())
+}
+
+/// Test receiving pre-messages and creation of the placeholder message with the metadata
+/// for image attachment
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_receive_pre_message_image() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, _post_message, _alice_msg_id) =
+        send_large_image_message(alice, alice_group_id).await?;
+
+    let msg = bob.recv_msg(&pre_message).await;
+
+    assert_eq!(msg.download_state(), DownloadState::Available);
+    assert_eq!(msg.viewtype, Viewtype::Text);
+    assert_eq!(msg.text, "test".to_owned());
+
+    // test that metadata is correctly returned by methods
+    assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Image));
+    // recoded image dimensions
+    assert_eq!(msg.get_filebytes(bob).await?, Some(149632));
+    assert_eq!(msg.get_height(), 1280);
+    assert_eq!(msg.get_width(), 720);
+
+    Ok(())
+}
+
+/// Test receiving reaction on pre-message
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_reaction_on_pre_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    let (pre_message, post_message, alice_msg_id) =
+        send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
+            .await?;
+
+    // Bob receives pre-message
+    let bob_msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(bob_msg.download_state(), DownloadState::Available);
+
+    // Alice sends reaction to her own message
+    send_reaction(alice, alice_msg_id, "👍").await?;
+
+    // Bob receives the reaction
+    bob.recv_msg_hidden(&alice.pop_sent_msg().await).await;
+
+    // Test if Bob sees reaction
+    let reactions = get_msg_reactions(bob, bob_msg.id).await?;
+    assert_eq!(reactions.to_string(), "👍1");
+
+    // Bob downloads Post-Message
+    bob.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(bob, bob_msg.id).await?;
+    assert_eq!(msg.download_state(), DownloadState::Done);
+
+    // Test if Bob still sees reaction
+    let reactions = get_msg_reactions(bob, bob_msg.id).await?;
+    assert_eq!(reactions.to_string(), "👍1");
+
+    Ok(())
+}
+
+/// Tests that fully downloading the message
+/// works but does not reappear when it was already deleted
+/// (as in the Message-ID already exists in the database
+/// and is assigned to the trash chat).
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_full_download_after_trashed() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let bob_group_id = bob.create_group_with_members("test group", &[alice]).await;
+
+    let (pre_message, post_message, _bob_msg_id) =
+        send_large_file_message(bob, bob_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
+
+    // Download message from Bob partially.
+    let alice_msg = alice.recv_msg(&pre_message).await;
+
+    // Delete the received message.
+    // Note that it remains in the database in the trash chat.
+    delete_msgs(alice, &[alice_msg.id]).await?;
+
+    // Fully download message after deletion.
+    alice.recv_msg_trash(&post_message).await;
+
+    // The message does not reappear.
+    let msg = Message::load_from_db_optional(bob, alice_msg.id).await?;
+    assert!(msg.is_none());
+
+    Ok(())
+}
+
+/// Test that webxdc updates are received for pre-messages
+/// and available when the Post-Message is downloaded
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
+
+    // Alice sends a larger instance and an update
+    let (pre_message, post_message, alice_sent_instance_msg_id) =
+        send_large_webxdc_message(alice, alice_group_id).await?;
+    alice
+        .send_webxdc_status_update(
+            alice_sent_instance_msg_id,
+            r#"{"payload": 7, "summary":"sum", "document":"doc"}"#,
+        )
+        .await?;
+    alice.flush_status_updates().await?;
+    let webxdc_update = alice.pop_sent_msg().await;
+
+    // Bob does not download instance but already receives update
+    let bob_instance = bob.recv_msg(&pre_message).await;
+    assert_eq!(bob_instance.download_state, DownloadState::Available);
+    bob.recv_msg_trash(&webxdc_update).await;
+
+    // Bob downloads instance, updates should be assigned correctly
+    bob.recv_msg_trash(&post_message).await;
+
+    let bob_instance = bob.get_last_msg().await;
+    assert_eq!(bob_instance.viewtype, Viewtype::Webxdc);
+    assert_eq!(bob_instance.download_state, DownloadState::Done);
+    assert_eq!(
+        bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial::new(0))
+            .await?,
+        r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"#
+    );
+    let info = bob_instance.get_webxdc_info(bob).await?;
+    assert_eq!(info.document, "doc");
+    assert_eq!(info.summary, "sum");
+
+    Ok(())
+}
+
+/// Test mark seen pre-message
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_markseen_pre_msg() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let bob_chat_id = bob.create_chat(alice).await.id;
+    alice.create_chat(bob).await; // Make sure the chat is accepted.
+
+    tcm.section("Bob sends a large message to Alice");
+    let (pre_message, post_message, _bob_msg_id) =
+        send_large_file_message(bob, bob_chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
+
+    tcm.section("Alice receives a pre-message message from Bob");
+    let msg = alice.recv_msg(&pre_message).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
+    assert_eq!(msg.state, MessageState::InFresh);
+
+    tcm.section("Alice marks the pre-message as read and sends a MDN");
+    markseen_msgs(alice, vec![msg.id]).await?;
+    assert_eq!(msg.id.get_state(alice).await?, MessageState::InSeen);
+    assert_eq!(
+        alice
+            .sql
+            .count("SELECT COUNT(*) FROM smtp_mdns", ())
+            .await?,
+        1
+    );
+
+    tcm.section("Alice downloads message");
+    alice.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(alice, msg.id).await?;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
+    assert_eq!(
+        msg.state,
+        MessageState::InSeen,
+        "The message state mustn't be downgraded to `InFresh`"
+    );
+
+    Ok(())
+}
+
+/// Test that pre-message can start a chat
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_pre_msg_can_start_chat() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    tcm.section("establishing a DM chat between alice and bob");
+    let bob_alice_dm_chat_id = bob.create_chat(alice).await.id;
+    alice.create_chat(bob).await; // Make sure the chat is accepted.
+
+    tcm.section("Alice prepares chat");
+    let chat_id = chat::create_group(alice, "my group").await?;
+    let contacts = contact::Contact::get_all(alice, 0, None).await?;
+    let alice_bob_id = contacts.first().expect("contact exists");
+    chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?;
+
+    tcm.section("Alice sends large message to promote/start chat");
+    let (pre_message, _post_message, _alice_msg_id) =
+        send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
+
+    tcm.section("Bob receives the pre-message message from Alice");
+    let msg = bob.recv_msg(&pre_message).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert_ne!(msg.chat_id, bob_alice_dm_chat_id);
+    let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?;
+    assert_eq!(chat.name, "my group");
+
+    Ok(())
+}
+
+/// Test that Post-Message can start a chat
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_post_msg_can_start_chat() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    tcm.section("establishing a DM chat between alice and bob");
+    let bob_alice_dm_chat_id = bob.create_chat(alice).await.id;
+    alice.create_chat(bob).await; // Make sure the chat is accepted.
+
+    tcm.section("Alice prepares chat");
+    let chat_id = chat::create_group(alice, "my group").await?;
+    let contacts = contact::Contact::get_all(alice, 0, None).await?;
+    let alice_bob_id = contacts.first().expect("contact exists");
+    chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?;
+
+    tcm.section("Alice sends large message to promote/start chat");
+    let (_pre_message, post_message, _bob_msg_id) =
+        send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
+
+    tcm.section("Bob receives the pre-message message from Alice");
+    let msg = bob.recv_msg(&post_message).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_ne!(msg.chat_id, bob_alice_dm_chat_id);
+    let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?;
+    assert_eq!(chat.name, "my group");
+
+    Ok(())
+}
+
+/// Test that message ordering is still correct after downloading
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_download_later_keeps_message_order() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    tcm.section(
+        "establishing a DM chat between alice and bob and bob sends large message to alice",
+    );
+    let bob_alice_dm_chat = bob.create_chat(alice).await.id;
+    alice.create_chat(bob).await; // Make sure the chat is accepted.
+    let (pre_message, post_message, _bob_msg_id) = send_large_file_message(
+        bob,
+        bob_alice_dm_chat,
+        Viewtype::File,
+        &vec![0u8; 1_000_000],
+    )
+    .await?;
+
+    tcm.section("Alice downloads pre-message");
+    let msg = alice.recv_msg(&pre_message).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id);
+
+    tcm.section("Bob sends hi to Alice");
+    let hi_msg = tcm.send_recv(bob, alice, "hi").await;
+    assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
+
+    tcm.section("Alice downloads Post-Message");
+    alice.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(alice, msg.id).await?;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
+    assert!(msg.timestamp_sort <= hi_msg.timestamp_sort);
+
+    Ok(())
+}
+
+/// Test that ChatlistItemChanged event is emitted when downloading Post-Message
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_chatlist_event_on_post_msg_download() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    tcm.section(
+        "establishing a DM chat between alice and bob and bob sends large message to alice",
+    );
+    let bob_alice_dm_chat = bob.create_chat(alice).await.id;
+    alice.create_chat(bob).await; // Make sure the chat is accepted.
+    let (pre_message, post_message, _bob_msg_id) = send_large_file_message(
+        bob,
+        bob_alice_dm_chat,
+        Viewtype::File,
+        &vec![0u8; 1_000_000],
+    )
+    .await?;
+
+    tcm.section("Alice downloads pre-message");
+    let msg = alice.recv_msg(&pre_message).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id);
+
+    tcm.section("Alice downloads Post-Message and waits for ChatlistItemChanged event ");
+    alice.evtracker.clear_events();
+    alice.recv_msg_trash(&post_message).await;
+    let msg = Message::load_from_db(alice, msg.id).await?;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    alice
+        .evtracker
+        .get_matching(|e| {
+            e == &EventType::ChatlistItemChanged {
+                chat_id: Some(msg.chat_id),
+            }
+        })
+        .await;
+
+    Ok(())
+}

src/tests/pre_messages/sending.rs

@@ -0,0 +1,337 @@
+//! Tests about sending pre-messages
+//! - When to send a pre-message and post-message instead of a normal message
+//! - Test that sent pre- and post-message contain the right Headers
+//!   and that they are send in the correct order (pre-message is sent first.)
+use anyhow::Result;
+use mailparse::MailHeaderMap;
+use tokio::fs;
+
+use crate::chat::{self, create_group, send_msg};
+use crate::config::Config;
+use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::message::{Message, Viewtype};
+use crate::test_utils::{self, TestContext, TestContextManager};
+/// Tests that Pre-Message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
+/// Also test that Pre-Message is sent first, before the Post-Message
+/// And that Autocrypt-gossip and selfavatar never go into Post-Messages
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sending_pre_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let fiona = &tcm.fiona().await;
+    let group_id = alice
+        .create_group_with_members("test group", &[bob, fiona])
+        .await;
+
+    let mut msg = Message::new(Viewtype::File);
+    msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
+    msg.set_text("test".to_owned());
+
+    // assert that test attachment is bigger than limit
+    assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+    let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+    //   Pre-Message and Post-Message should be present
+    //   and test that correct headers are present on both messages
+    assert_eq!(smtp_rows.len(), 2);
+    let pre_message = smtp_rows.first().expect("first element exists");
+    let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?;
+    let post_message = smtp_rows.get(1).expect("second element exists");
+    let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?;
+
+    assert!(
+        pre_message_parsed
+            .headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_none()
+    );
+    assert!(
+        post_message_parsed
+            .headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_some()
+    );
+
+    assert_eq!(
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        Some(format!("<{}>", msg.rfc724_mid)),
+        "Post-Message should have the rfc message id of the database message"
+    );
+
+    assert_ne!(
+        pre_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId),
+        "message ids of Pre-Message and Post-Message should be different"
+    );
+
+    let decrypted_post_message = bob.parse_msg(post_message).await;
+    assert_eq!(decrypted_post_message.decrypting_failed, false);
+    assert_eq!(
+        decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId),
+        false
+    );
+
+    let decrypted_pre_message = bob.parse_msg(pre_message).await;
+    assert_eq!(
+        decrypted_pre_message
+            .get_header(HeaderDef::ChatPostMessageId)
+            .map(String::from),
+        post_message_parsed
+            .headers
+            .get_header_value(HeaderDef::MessageId)
+    );
+    assert!(
+        pre_message_parsed
+            .headers
+            .get_header_value(HeaderDef::ChatPostMessageId)
+            .is_none(),
+        "no Chat-Post-Message-ID header in unprotected headers of Pre-Message"
+    );
+
+    Ok(())
+}
+
+/// Tests that Pre-Message has autocrypt gossip headers and self avatar
+/// and Post-Message doesn't have these headers
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let fiona = &tcm.fiona().await;
+    let group_id = alice
+        .create_group_with_members("test group", &[bob, fiona])
+        .await;
+
+    let mut msg = Message::new(Viewtype::File);
+    msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
+    msg.set_text("test".to_owned());
+
+    // assert that test attachment is bigger than limit
+    assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+    // simulate conditions for sending self avatar
+    let avatar_src = alice.get_blobdir().join("avatar.png");
+    fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?;
+    alice
+        .set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+        .await?;
+
+    let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+    assert_eq!(smtp_rows.len(), 2);
+    let pre_message = smtp_rows.first().expect("first element exists");
+    let post_message = smtp_rows.get(1).expect("second element exists");
+    let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?;
+
+    let decrypted_pre_message = bob.parse_msg(pre_message).await;
+    assert!(
+        decrypted_pre_message
+            .get_header(HeaderDef::ChatPostMessageId)
+            .is_some(),
+        "tested message is not a pre-message, sending order may be broken"
+    );
+    assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0);
+    assert_ne!(decrypted_pre_message.user_avatar, None);
+
+    let decrypted_post_message = bob.parse_msg(post_message).await;
+    assert!(
+        post_message_parsed
+            .headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_some(),
+        "tested message is not a Post-Message, sending order may be broken"
+    );
+    assert_eq!(decrypted_post_message.gossiped_keys.len(), 0);
+    assert_eq!(decrypted_post_message.user_avatar, None);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_unecrypted_gets_no_pre_message() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+
+    let chat = alice
+        .create_chat_with_contact("example", "email@example.org")
+        .await;
+
+    let mut msg = Message::new(Viewtype::File);
+    msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 300_000], None)?;
+    msg.set_text("test".to_owned());
+
+    let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?;
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+    assert_eq!(smtp_rows.len(), 1);
+    let message_bytes = smtp_rows
+        .first()
+        .expect("first element exists")
+        .payload
+        .as_bytes();
+    let message = mailparse::parse_mail(message_bytes)?;
+    assert!(
+        message
+            .headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_none(),
+    );
+    Ok(())
+}
+
+/// Tests that no pre message is sent for normal message
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_not_sending_pre_message_no_attachment() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let chat = alice.create_chat(bob).await;
+
+    // send normal text message
+    let mut msg = Message::new(Viewtype::Text);
+    msg.set_text("test".to_owned());
+    let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+    assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
+
+    let msg = smtp_rows.first().expect("first element exists");
+    let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
+
+    assert!(
+        mail.headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_none(),
+        "no 'Chat-Is-Post-Message'-header should be present"
+    );
+    assert!(
+        mail.headers
+            .get_first_header(HeaderDef::ChatPostMessageId.get_headername())
+            .is_none(),
+        "no 'Chat-Post-Message-ID'-header should be present in clear text headers"
+    );
+    let decrypted_message = bob.parse_msg(msg).await;
+    assert!(
+        !decrypted_message.header_exists(HeaderDef::ChatPostMessageId),
+        "no 'Chat-Post-Message-ID'-header should be present"
+    );
+
+    // test that pre message is not send for large large text
+    let mut msg = Message::new(Viewtype::Text);
+    let long_text = String::from_utf8(vec![b'a'; 300_000])?;
+    assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
+    msg.set_text(long_text);
+    let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+    assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
+
+    let msg = smtp_rows.first().expect("first element exists");
+    let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
+
+    assert!(
+        mail.headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_none()
+    );
+    assert!(
+        mail.headers
+            .get_first_header(HeaderDef::ChatPostMessageId.get_headername())
+            .is_none(),
+        "no 'Chat-Post-Message-ID'-header should be present in clear text headers"
+    );
+    let decrypted_message = bob.parse_msg(msg).await;
+    assert!(
+        !decrypted_message.header_exists(HeaderDef::ChatPostMessageId),
+        "no 'Chat-Post-Message-ID'-header should be present"
+    );
+    Ok(())
+}
+
+/// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+    let chat = alice.create_chat(bob).await;
+
+    let mut msg = Message::new(Viewtype::File);
+    msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 100_000], None)?;
+    msg.set_text("test".to_owned());
+
+    // assert that test attachment is smaller than limit
+    assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+    let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
+    let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
+
+    //   only one message and no "is Post-Message" header should be present
+    assert_eq!(smtp_rows.len(), 1);
+
+    let msg = smtp_rows.first().expect("first element exists");
+    let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
+
+    assert!(
+        mail.headers
+            .get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
+            .is_none()
+    );
+    assert!(
+        mail.headers
+            .get_first_header(HeaderDef::ChatPostMessageId.get_headername())
+            .is_none(),
+        "no 'Chat-Post-Message-ID'-header should be present in clear text headers"
+    );
+    let decrypted_message = bob.parse_msg(msg).await;
+    assert!(
+        !decrypted_message.header_exists(HeaderDef::ChatPostMessageId),
+        "no 'Chat-Post-Message-ID'-header should be present"
+    );
+
+    Ok(())
+}
+
+/// Tests that pre message is not send for large webxdc updates
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_render_webxdc_status_update_object_range() -> Result<()> {
+    let t = TestContext::new_alice().await;
+    let chat_id = create_group(&t, "a chat").await?;
+
+    let instance = {
+        let mut instance = Message::new(Viewtype::File);
+        instance.set_file_from_bytes(
+            &t,
+            "minimal.xdc",
+            include_bytes!("../../../test-data/webxdc/minimal.xdc"),
+            None,
+        )?;
+        let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?;
+        assert_eq!(instance.viewtype, Viewtype::Webxdc);
+        Message::load_from_db(&t, instance_msg_id).await
+    }
+    .unwrap();
+
+    t.pop_sent_msg().await;
+    assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0);
+
+    let long_text = String::from_utf8(vec![b'a'; 300_000])?;
+    assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
+    t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}"))
+        .await?;
+    t.flush_status_updates().await?;
+
+    assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1);
+    Ok(())
+}

src/tests/pre_messages/util.rs

@@ -0,0 +1,65 @@
+use anyhow::Result;
+use async_zip::tokio::write::ZipFileWriter;
+use async_zip::{Compression, ZipEntryBuilder};
+use futures::io::Cursor as FuturesCursor;
+use pretty_assertions::assert_eq;
+use tokio_util::compat::FuturesAsyncWriteCompatExt;
+
+use crate::chat::{self, ChatId};
+use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
+use crate::message::{Message, MsgId, Viewtype};
+use crate::test_utils::{SentMessage, TestContext, create_test_image};
+
+pub async fn send_large_file_message<'a>(
+    sender: &'a TestContext,
+    target_chat: ChatId,
+    view_type: Viewtype,
+    content: &[u8],
+) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> {
+    let mut msg = Message::new(view_type);
+    let file_name = if view_type == Viewtype::Webxdc {
+        "test.xdc"
+    } else {
+        "test.bin"
+    };
+    msg.set_file_from_bytes(sender, file_name, content, None)?;
+    msg.set_text("test".to_owned());
+
+    // assert that test attachment is bigger than limit
+    assert!(msg.get_filebytes(sender).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
+
+    let msg_id = chat::send_msg(sender, target_chat, &mut msg).await?;
+    let smtp_rows = sender.get_smtp_rows_for_msg(msg_id).await;
+
+    assert_eq!(smtp_rows.len(), 2);
+    let pre_message = smtp_rows.first().expect("Pre-Message exists");
+    let post_message = smtp_rows.get(1).expect("Post-Message exists");
+    Ok((pre_message.to_owned(), post_message.to_owned(), msg_id))
+}
+
+pub async fn send_large_webxdc_message<'a>(
+    sender: &'a TestContext,
+    target_chat: ChatId,
+) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> {
+    let futures_cursor = FuturesCursor::new(Vec::new());
+    let mut buffer = futures_cursor.compat_write();
+    let mut writer = ZipFileWriter::with_tokio(&mut buffer);
+    writer
+        .write_entry_whole(
+            ZipEntryBuilder::new("index.html".into(), Compression::Stored),
+            &[0u8; 1_000_000],
+        )
+        .await?;
+    writer.close().await?;
+    let big_webxdc_app = buffer.into_inner().into_inner();
+    send_large_file_message(sender, target_chat, Viewtype::Webxdc, &big_webxdc_app).await
+}
+
+pub async fn send_large_image_message<'a>(
+    sender: &'a TestContext,
+    target_chat: ChatId,
+) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> {
+    let (width, height) = (1080, 1920);
+    let test_img = create_test_image(width, height)?;
+    send_large_file_message(sender, target_chat, Viewtype::Image, &test_img).await
+}