feat: truncate long messages when loading instead of saving

deltachat-jsonrpc/src/api.rs

@@ -23,8 +23,9 @@ use deltachat::ephemeral::Timer;
 use deltachat::imex;
 use deltachat::location;
 use deltachat::message::{
-    self, delete_msgs_ex, get_existing_msg_ids, get_msg_read_receipt_count, get_msg_read_receipts,
-    markseen_msgs, Message, MessageState, MsgId, Viewtype,
+    self, delete_msgs_ex, dont_truncate_long_messages, get_existing_msg_ids,
+    get_msg_read_receipt_count, get_msg_read_receipts, markseen_msgs, Message, MessageState, MsgId,
+    Viewtype,
 };
 use deltachat::peer_channels::{
     leave_webxdc_realtime, send_webxdc_realtime_advertisement, send_webxdc_realtime_data,
@@ -1490,6 +1491,15 @@ impl CommandApi {
         MsgId::new(message_id).get_html(&ctx).await
     }
 
+    /// Opt out of truncating long messages when loading.
+    ///
+    /// Should be used by the UIs that can handle long text messages.
+    async fn dont_truncate_long_messages(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        dont_truncate_long_messages(&ctx);
+        Ok(())
+    }
+
     /// get multiple messages in one call,
     /// if loading one message fails the error is stored in the result object in it's place.
     ///

src/chat.rs

@@ -51,7 +51,6 @@ use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id,
     create_outgoing_rfc724_mid, get_abs_path, gm2local_offset, normalize_text, time,
-    truncate_msg_text,
 };
 use crate::webxdc::StatusUpdateSerial;
 
@@ -1895,7 +1894,8 @@ impl Chat {
             EphemeralTimer::Enabled { duration } => time().saturating_add(duration.into()),
         };
 
-        let (msg_text, was_truncated) = truncate_msg_text(context, msg.text.clone()).await?;
+        let msg_text = msg.text.clone();
+
         let new_mime_headers = if msg.has_html() {
             msg.param.get(Param::SendHtml).map(|s| s.to_string())
         } else {
@@ -1908,13 +1908,6 @@ impl Chat {
             html_part.write_part(cursor).ok();
             String::from_utf8_lossy(&buffer).to_string()
         });
-        let new_mime_headers = new_mime_headers.or_else(|| match was_truncated {
-            // We need to add some headers so that they are stripped before formatting HTML by
-            // `MsgId::get_html()`, not a part of the actual text. Let's add "Content-Type", it's
-            // anyway a useful metadata about the stored text.
-            true => Some("Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text),
-            false => None,
-        });
         let new_mime_headers = match new_mime_headers {
             Some(h) => Some(tokio::task::block_in_place(move || {
                 buf_compress(h.as_bytes())

src/chat/chat_tests.rs

@@ -9,6 +9,7 @@ use crate::constants::{
 use crate::ephemeral::Timer;
 use crate::headerdef::HeaderDef;
 use crate::imex::{ImexMode, has_backup, imex};
+use crate::message;
 use crate::message::{Message, MessengerMessage, delete_msgs};
 use crate::mimeparser::{self, MimeMessage};
 use crate::qr::{Qr, check_qr};
@@ -5884,6 +5885,8 @@ async fn test_send_edit_request() -> Result<()> {
     let forwarded = alice2.get_last_msg().await;
     assert!(!forwarded.is_edited());
 
+    message::dont_truncate_long_messages(alice);
+
     // If a message is too long after editing, it becomes an HTML message on the receiver side. On
     // the sender side it's still text so that it can be edited again.
     static REPEAT_TXT: &str = "this text with 42 chars is just repeated.\n";

src/context.rs

@@ -225,7 +225,23 @@ impl WeakContext {
 pub struct InnerContext {
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
+
     pub(crate) sql: Sql,
+
+    /// True if long text messages should be truncated
+    /// and full message HTML added.
+    ///
+    /// This should be set for the UIs that cannot handle
+    /// long messages but can display HTML messages.
+    /// It is enabled by default.
+    ///
+    /// UIs that can handle long messages should call
+    /// [crate::message::dont_truncate_long_messages]
+    /// to unset it.
+    ///
+    /// Ignored for bots, bots never get truncated messages.
+    pub(crate) truncate_long_messages: AtomicBool,
+
     /// The global "ongoing" process state.
     ///
     /// This is a global mutex-like state for operations which should be modal in the
@@ -484,6 +500,7 @@ impl Context {
             blobdir,
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(dbfile),
+            truncate_long_messages: AtomicBool::new(true),
             oauth2_mutex: Mutex::new(()),
             wrong_pw_warning_mutex: Mutex::new(()),
             housekeeping_mutex: Mutex::new(()),

src/download.rs

@@ -209,7 +209,7 @@ impl Session {
         let (sender, receiver) = async_channel::unbounded();
         {
             let _fetch_msgs_lock_guard = context.fetch_msgs_mutex.lock().await;
-            self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender)
+            Box::pin(self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender))
                 .await?;
         }
         if receiver.recv().await.is_err() {

src/html.rs

@@ -32,7 +32,7 @@ impl Message {
     /// The corresponding ffi-function is `dc_msg_has_html()`.
     /// To get the HTML-code of the message, use `MsgId.get_html()`.
     pub fn has_html(&self) -> bool {
-        self.mime_modified
+        self.mime_modified || self.full_text.is_some()
     }
 
     /// Set HTML-part part of a message that is about to be sent.
@@ -277,7 +277,7 @@ impl MsgId {
         if let Some(html) = param.get(SendHtml) {
             return Ok(Some(html.to_string()));
         }
-        let from_rawmime = |rawmime: Vec<u8>| {
+        let from_rawmime = async |rawmime: Vec<u8>| {
             if !rawmime.is_empty() {
                 match HtmlMsgParser::from_bytes(context, &rawmime) {
                     Err(err) => {
@@ -287,19 +287,30 @@ impl MsgId {
                     Ok((parser, _)) => Ok(Some(parser.html)),
                 }
             } else {
-                warn!(context, "get_html: no mime for {}", self);
-                Ok(None)
+                let msg = Message::load_from_db(context, self).await?;
+                if let Some(full_text) = &msg.full_text {
+                    let html = PlainText {
+                        text: full_text.clone(),
+                        flowed: false,
+                        delsp: false,
+                    }
+                    .to_html();
+                    Ok(Some(html))
+                } else {
+                    warn!(context, "get_html: no mime for {}", self);
+                    Ok(None)
+                }
             }
         };
 
         if compressed {
-            return from_rawmime(buf_decompress(&headers)?);
+            return from_rawmime(buf_decompress(&headers)?).await;
         }
         let headers2 = headers.clone();
         let compressed = match tokio::task::block_in_place(move || buf_compress(&headers2)) {
             Err(e) => {
                 warn!(context, "get_mime_headers: buf_compress() failed: {}", e);
-                return from_rawmime(headers);
+                return from_rawmime(headers).await;
             }
             Ok(o) => o,
         };
@@ -324,7 +335,7 @@ WHERE id=? AND mime_headers!='' AND mime_compressed=0",
                 "get_mime_headers: failed to update mime_headers: {}", e
             );
         }
-        from_rawmime(headers)
+        from_rawmime(headers).await
     }
 }
 

src/imap.rs

@@ -765,8 +765,7 @@ impl Imap {
         };
 
         let actually_download_messages_future = async {
-            session
-                .fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender)
+            Box::pin(session.fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender))
                 .await
                 .context("fetch_many_msgs")
         };

src/message.rs

@@ -3,6 +3,7 @@
 use std::collections::BTreeSet;
 use std::path::{Path, PathBuf};
 use std::str;
+use std::sync::atomic::Ordering;
 
 use anyhow::{Context as _, Result, ensure, format_err};
 use deltachat_contact_tools::{VcardContact, parse_vcard};
@@ -32,10 +33,9 @@ use crate::param::{Param, Params};
 use crate::reaction::get_msg_reactions;
 use crate::summary::Summary;
 use crate::sync::SyncData;
-use crate::tools::create_outgoing_rfc724_mid;
 use crate::tools::{
-    get_filebytes, get_filemeta, gm2local_offset, read_file, sanitize_filename, time,
-    timestamp_to_str,
+    create_outgoing_rfc724_mid, get_filebytes, get_filemeta, gm2local_offset, read_file,
+    sanitize_filename, time, timestamp_to_str, truncate_msg_text,
 };
 
 /// Message ID, including reserved IDs.
@@ -441,7 +441,13 @@ pub struct Message {
     pub(crate) timestamp_rcvd: i64,
     pub(crate) ephemeral_timer: EphemeralTimer,
     pub(crate) ephemeral_timestamp: i64,
+
+    /// Message text, possibly truncated if the message is large.
     pub(crate) text: String,
+
+    /// Full text if the message text is truncated.
+    pub(crate) full_text: Option<String>,
+
     /// Text that is added to the end of Message.text
     ///
     /// Currently used for adding the download information on pre-messages
@@ -566,6 +572,7 @@ impl Message {
                         }
                         _ => String::new(),
                     };
+
                     let msg = Message {
                         id: row.get("id")?,
                         rfc724_mid: row.get::<_, String>("rfc724mid")?,
@@ -590,6 +597,7 @@ impl Message {
                         original_msg_id: row.get("original_msg_id")?,
                         mime_modified: row.get("mime_modified")?,
                         text,
+                        full_text: None,
                         additional_text: String::new(),
                         subject: row.get("subject")?,
                         param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
@@ -607,6 +615,15 @@ impl Message {
             .with_context(|| format!("failed to load message {id} from the database"))?;
 
         if let Some(msg) = &mut msg {
+            if !msg.mime_modified {
+                let (truncated_text, was_truncated) =
+                    truncate_msg_text(context, msg.text.clone()).await?;
+                if was_truncated {
+                    msg.full_text = Some(msg.text.clone());
+                    msg.text = truncated_text;
+                }
+            }
+
             msg.additional_text =
                 Self::get_additional_text(context, msg.download_state, &msg.param)?;
         }
@@ -2342,5 +2359,22 @@ impl Viewtype {
     }
 }
 
+/// Opt out of truncating long messages.
+///
+/// After calling this function, long messages
+/// will not be truncated during loading.
+///
+/// UIs should call this function if they
+/// can handle long messages by cutting them
+/// and displaying "Show full message" option.
+///
+/// Has no effect for bots which never
+/// truncate messages when loading.
+pub fn dont_truncate_long_messages(context: &Context) {
+    context
+        .truncate_long_messages
+        .store(false, Ordering::Relaxed);
+}
+
 #[cfg(test)]
 mod message_tests;

src/mimeparser.rs

@@ -31,7 +31,7 @@ use crate::message::{self, Message, MsgId, Viewtype, get_vcard_summary, set_msg_
 use crate::param::{Param, Params};
 use crate::simplify::{SimplifiedText, simplify};
 use crate::sync::SyncItems;
-use crate::tools::{get_filemeta, parse_receive_headers, time, truncate_msg_text, validate_id};
+use crate::tools::{get_filemeta, parse_receive_headers, time, validate_id};
 use crate::{chatlist_events, location, tools};
 
 /// Public key extracted from `Autocrypt-Gossip`
@@ -1438,12 +1438,6 @@ impl MimeMessage {
                             (simplified_txt, top_quote)
                         };
 
-                        let (simplified_txt, was_truncated) =
-                            truncate_msg_text(context, simplified_txt).await?;
-                        if was_truncated {
-                            self.is_mime_modified = was_truncated;
-                        }
-
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part {
                                 dehtml_failed,

src/mimeparser/mimeparser_tests.rs

@@ -1287,12 +1287,12 @@ async fn test_mime_modified_large_plain() -> Result<()> {
 
     {
         let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?;
-        assert!(mimemsg.is_mime_modified);
-        assert!(
-            mimemsg.parts[0].msg.matches("just repeated").count()
-                <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len()
+        assert!(!mimemsg.is_mime_modified);
+        assert!(mimemsg.parts[0].msg.matches("just repeated").count() == REPEAT_CNT);
+        assert_eq!(
+            mimemsg.parts[0].msg.len() + 1,
+            REPEAT_TXT.len() * REPEAT_CNT
         );
-        assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len());
     }
 
     for draft in [false, true] {

src/receive_imf/receive_imf_tests.rs

@@ -3610,39 +3610,17 @@ async fn test_big_forwarded_with_big_attachment() -> Result<()> {
             .starts_with("this text with 42 chars is just repeated.")
     );
     assert!(msg.get_text().ends_with("[...]"));
-    assert!(!msg.has_html());
-
-    let msg = Message::load_from_db(bob, rcvd.msg_ids[2]).await?;
-    assert_eq!(msg.get_viewtype(), Viewtype::File);
     assert!(msg.has_html());
     let html = msg.id.get_html(bob).await?.unwrap();
-    let tail = html
-        .split_once("Hello!")
-        .unwrap()
-        .1
-        .split_once("From: AAA")
-        .unwrap()
-        .1
-        .split_once("aaa@example.org")
-        .unwrap()
-        .1
-        .split_once("To: Alice")
-        .unwrap()
-        .1
-        .split_once("alice@example.org")
-        .unwrap()
-        .1
-        .split_once("Subject: Some subject")
-        .unwrap()
-        .1
-        .split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000")
-        .unwrap()
-        .1;
     assert_eq!(
-        tail.matches("this text with 42 chars is just repeated.")
+        html.matches("this text with 42 chars is just repeated.")
             .count(),
         128
     );
+
+    let msg = Message::load_from_db(bob, rcvd.msg_ids[2]).await?;
+    assert_eq!(msg.get_viewtype(), Viewtype::File);
+    assert!(!msg.has_html());
     Ok(())
 }
 

src/tools.rs

@@ -9,6 +9,7 @@ use std::mem;
 use std::ops::{AddAssign, Deref};
 use std::path::{Path, PathBuf};
 use std::str::from_utf8;
+use std::sync::atomic::Ordering;
 // If a time value doesn't need to be sent to another host, saved to the db or otherwise used across
 // program restarts, a monotonically nondecreasing clock (`Instant`) should be used. But as
 // `Instant` may use `libc::clock_gettime(CLOCK_MONOTONIC)`, e.g. on Android, and does not advance
@@ -139,7 +140,9 @@ pub(crate) fn truncate_by_lines(
 ///
 /// Returns the resulting text and a bool telling whether a truncation was done.
 pub(crate) async fn truncate_msg_text(context: &Context, text: String) -> Result<(String, bool)> {
-    if context.get_config_bool(Config::Bot).await? {
+    if !context.truncate_long_messages.load(Ordering::Relaxed)
+        || context.get_config_bool(Config::Bot).await?
+    {
         return Ok((text, false));
     }
     // Truncate text if it has too many lines