feat: truncate long messages when loading instead of saving

src/chat.rs

@@ -50,7 +50,7 @@ use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id,
     create_outgoing_rfc724_mid, create_smeared_timestamp, create_smeared_timestamps, get_abs_path,
-    gm2local_offset, normalize_text, smeared_time, time, truncate_msg_text,
+    gm2local_offset, normalize_text, smeared_time, time,
 };
 use crate::webxdc::StatusUpdateSerial;
 
@@ -1742,7 +1742,6 @@ impl Chat {
     ///
     /// If `update_msg_id` is set, that record is reused;
     /// if `update_msg_id` is None, a new record is created.
-    #[expect(clippy::arithmetic_side_effects)]
     async fn prepare_msg_raw(
         &mut self,
         context: &Context,
@@ -1887,7 +1886,8 @@ impl Chat {
             EphemeralTimer::Enabled { duration } => time().saturating_add(duration.into()),
         };
 
-        let (msg_text, was_truncated) = truncate_msg_text(context, msg.text.clone()).await?;
+        let msg_text = msg.text.clone();
+
         let new_mime_headers = if msg.has_html() {
             msg.param.get(Param::SendHtml).map(|s| s.to_string())
         } else {
@@ -1900,13 +1900,6 @@ impl Chat {
             html_part.write_part(cursor).ok();
             String::from_utf8_lossy(&buffer).to_string()
         });
-        let new_mime_headers = new_mime_headers.or_else(|| match was_truncated {
-            // We need to add some headers so that they are stripped before formatting HTML by
-            // `MsgId::get_html()`, not a part of the actual text. Let's add "Content-Type", it's
-            // anyway a useful metadata about the stored text.
-            true => Some("Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text),
-            false => None,
-        });
         let new_mime_headers = match new_mime_headers {
             Some(h) => Some(tokio::task::block_in_place(move || {
                 buf_compress(h.as_bytes())

src/html.rs

@@ -30,7 +30,7 @@ impl Message {
     /// The corresponding ffi-function is `dc_msg_has_html()`.
     /// To get the HTML-code of the message, use `MsgId.get_html()`.
     pub fn has_html(&self) -> bool {
-        self.mime_modified
+        self.mime_modified || self.full_text.is_some()
     }
 
     /// Set HTML-part part of a message that is about to be sent.
@@ -278,12 +278,23 @@ impl MsgId {
                 }
                 Ok((parser, _)) => Ok(Some(parser.html)),
             }
+        } else {
+            let msg = Message::load_from_db(context, self).await?;
+            if let Some(full_text) = &msg.full_text {
+                let html = PlainText {
+                    text: full_text.clone(),
+                    flowed: false,
+                    delsp: false,
+                }
+                .to_html();
+                Ok(Some(html))
             } else {
                 warn!(context, "get_html: no mime for {}", self);
                 Ok(None)
             }
         }
     }
+}
 
 #[cfg(test)]
 mod tests {

src/message.rs

@@ -34,10 +34,9 @@ use crate::reaction::get_msg_reactions;
 use crate::sql;
 use crate::summary::Summary;
 use crate::sync::SyncData;
-use crate::tools::create_outgoing_rfc724_mid;
 use crate::tools::{
-    buf_compress, buf_decompress, get_filebytes, get_filemeta, gm2local_offset, read_file,
+    buf_compress, buf_decompress, create_outgoing_rfc724_mid, get_filebytes, get_filemeta,
-    sanitize_filename, time, timestamp_to_str,
+    gm2local_offset, read_file, sanitize_filename, time, timestamp_to_str, truncate_msg_text,
 };
 
 /// Message ID, including reserved IDs.
@@ -431,7 +430,13 @@ pub struct Message {
     pub(crate) timestamp_rcvd: i64,
     pub(crate) ephemeral_timer: EphemeralTimer,
     pub(crate) ephemeral_timestamp: i64,
+
+    /// Message text, possibly truncated if the message is large.
     pub(crate) text: String,
+
+    /// Full text if the message text is truncated.
+    pub(crate) full_text: Option<String>,
+
     /// Text that is added to the end of Message.text
     ///
     /// Currently used for adding the download information on pre-messages
@@ -556,6 +561,7 @@ impl Message {
                         }
                         _ => String::new(),
                     };
+
                     let msg = Message {
                         id: row.get("id")?,
                         rfc724_mid: row.get::<_, String>("rfc724mid")?,
@@ -580,6 +586,7 @@ impl Message {
                         original_msg_id: row.get("original_msg_id")?,
                         mime_modified: row.get("mime_modified")?,
                         text,
+                        full_text: None,
                         additional_text: String::new(),
                         subject: row.get("subject")?,
                         param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
@@ -597,6 +604,15 @@ impl Message {
             .with_context(|| format!("failed to load message {id} from the database"))?;
 
         if let Some(msg) = &mut msg {
+            if !msg.mime_modified {
+                let (truncated_text, was_truncated) =
+                    truncate_msg_text(context, msg.text.clone()).await?;
+                if was_truncated {
+                    msg.full_text = Some(msg.text.clone());
+                    msg.text = truncated_text;
+                }
+            }
+
             msg.additional_text =
                 Self::get_additional_text(context, msg.download_state, &msg.param).await?;
         }

src/mimeparser.rs

@@ -32,9 +32,7 @@ use crate::message::{self, Message, MsgId, Viewtype, get_vcard_summary, set_msg_
 use crate::param::{Param, Params};
 use crate::simplify::{SimplifiedText, simplify};
 use crate::sync::SyncItems;
-use crate::tools::{
+use crate::tools::{get_filemeta, parse_receive_headers, smeared_time, time, validate_id};
-    get_filemeta, parse_receive_headers, smeared_time, time, truncate_msg_text, validate_id,
-};
 use crate::{chatlist_events, location, tools};
 
 /// Public key extracted from `Autocrypt-Gossip`
@@ -1472,12 +1470,6 @@ impl MimeMessage {
                             (simplified_txt, top_quote)
                         };
 
-                        let (simplified_txt, was_truncated) =
-                            truncate_msg_text(context, simplified_txt).await?;
-                        if was_truncated {
-                            self.is_mime_modified = was_truncated;
-                        }
-
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part {
                                 dehtml_failed,

src/mimeparser/mimeparser_tests.rs

@@ -1286,12 +1286,12 @@ async fn test_mime_modified_large_plain() -> Result<()> {
 
     {
         let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?;
-        assert!(mimemsg.is_mime_modified);
+        assert!(!mimemsg.is_mime_modified);
-        assert!(
+        assert!(mimemsg.parts[0].msg.matches("just repeated").count() == REPEAT_CNT);
-            mimemsg.parts[0].msg.matches("just repeated").count()
+        assert_eq!(
-                <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len()
+            mimemsg.parts[0].msg.len() + 1,
+            REPEAT_TXT.len() * REPEAT_CNT
         );
-        assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len());
     }
 
     for draft in [false, true] {

src/receive_imf/receive_imf_tests.rs

@@ -3585,39 +3585,17 @@ async fn test_big_forwarded_with_big_attachment() -> Result<()> {
             .starts_with("this text with 42 chars is just repeated.")
     );
     assert!(msg.get_text().ends_with("[...]"));
-    assert!(!msg.has_html());
-
-    let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?;
-    assert_eq!(msg.get_viewtype(), Viewtype::File);
     assert!(msg.has_html());
     let html = msg.id.get_html(t).await?.unwrap();
-    let tail = html
-        .split_once("Hello!")
-        .unwrap()
-        .1
-        .split_once("From: AAA")
-        .unwrap()
-        .1
-        .split_once("aaa@example.org")
-        .unwrap()
-        .1
-        .split_once("To: Alice")
-        .unwrap()
-        .1
-        .split_once("alice@example.org")
-        .unwrap()
-        .1
-        .split_once("Subject: Some subject")
-        .unwrap()
-        .1
-        .split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000")
-        .unwrap()
-        .1;
     assert_eq!(
-        tail.matches("this text with 42 chars is just repeated.")
+        html.matches("this text with 42 chars is just repeated.")
             .count(),
         128
     );
+
+    let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?;
+    assert_eq!(msg.get_viewtype(), Viewtype::File);
+    assert!(!msg.has_html());
     Ok(())
 }