From 20182b027e7356c9ba10a2ce2fb2881b1bd50468 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sun, 4 Oct 2020 05:05:44 +0300 Subject: [PATCH] Add quote API Sticky encryption rule, requiring that all replies to encrypted messages are encrypted, applies only to messages with a quote now. Co-Authored-By: B. Petersen --- deltachat-ffi/deltachat.h | 57 +++++++++++++++++++++ deltachat-ffi/src/lib.rs | 52 +++++++++++++++++++ python/src/deltachat/message.py | 21 ++++++++ python/tests/test_account.py | 90 ++++++++++++++++++++++++++------- src/chat.rs | 10 ++-- src/format_flowed.rs | 75 ++++++++++++++++++++------- src/message.rs | 88 ++++++++++++++++++++++++++++++++ src/mimefactory.rs | 9 +++- src/mimeparser.rs | 54 ++++++++++++++++---- src/param.rs | 3 ++ src/simplify.rs | 73 ++++++++++++++++---------- 11 files changed, 452 insertions(+), 80 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 152349a97..af0d3142b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -3675,6 +3675,63 @@ void dc_msg_set_location (dc_msg_t* msg, double latitude, d void dc_msg_latefiling_mediasize (dc_msg_t* msg, int width, int height, int duration); +/** + * Set the message replying to. + * This allows optionally to reply to an explicit message + * instead of replying implicitly to the end of the chat. + * + * dc_msg_set_quote() copies some basic data from the quoted message object + * so that dc_msg_get_quoted_text() will always work. + * dc_msg_get_quoted_msg() gets back the quoted message only if it is _not_ deleted. + * + * @memberof dc_msg_t + * @param msg The message object to set the reply to. + * @param quote The quote to set for msg. + */ +void dc_msg_set_quote (dc_msg_t* msg, const dc_msg_t* quote); + + +/** + * Get quoted text, if any. + * You can use this function also check if there is a quote for a message. + * + * The text is a summary of the original text, + * similar to what is shown in the chatlist. + * + * If available, you can get the whole quoted message object using dc_msg_get_quoted_msg(). + * + * @memberof dc_msg_t + * @param msg The message object. + * @return The quoted text or NULL if there is no quote. + * Returned strings must be released using dc_str_unref(). + */ +char* dc_msg_get_quoted_text (const dc_msg_t* msg); + + +/** + * Get quoted message, if available. + * UIs might use this information to offer "jumping back" to the quoted message + * or to enrich displaying the quote. + * + * If this function returns NULL, + * this does not mean there is no quote for the message - + * it might also mean that a quote exist but the quoted message is deleted meanwhile. + * Therefore, do not use this function to check if there is a quote for a message. + * To check if a message has a quote, use dc_msg_get_quoted_text(). + * + * To display the quote in the chat, use dc_msg_get_quoted_text() as a primary source, + * however, one might add information from the message object (eg. an image). + * + * It is not guaranteed that the message belong to the same chat. + * + * @memberof dc_msg_t + * @param msg The message object. + * @return The quoted message or NULL. + * Must be freed using dc_msg_unref() after usage. + */ +dc_msg_t* dc_msg_get_quoted_msg (const dc_msg_t* msg); + + /** * @class dc_contact_t * diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 247a6b433..f1b860cac 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -2975,6 +2975,58 @@ pub unsafe extern "C" fn dc_msg_get_error(msg: *mut dc_msg_t) -> *mut libc::c_ch } } +#[no_mangle] +pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_msg_t) { + if msg.is_null() { + eprintln!("ignoring careless call to dc_msg_set_quote()"); + return; + } + let ffi_msg = &mut *msg; + let ffi_quote = &*quote; + + ffi_msg + .message + .set_quote(&ffi_quote.message) + .log_err(&*ffi_msg.context, "failed to set quote") + .ok(); +} + +#[no_mangle] +pub unsafe extern "C" fn dc_msg_get_quoted_text(msg: *const dc_msg_t) -> *mut libc::c_char { + if msg.is_null() { + eprintln!("ignoring careless call to dc_msg_get_quoted_text()"); + return ptr::null_mut(); + } + let ffi_msg: &MessageWrapper = &*msg; + ffi_msg + .message + .quoted_text() + .map_or_else(ptr::null_mut, |s| s.strdup()) +} + +#[no_mangle] +pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_msg_t { + if msg.is_null() { + eprintln!("ignoring careless call to dc_get_quoted_msg()"); + return ptr::null_mut(); + } + let ffi_msg: &MessageWrapper = &*msg; + let context = &*ffi_msg.context; + let res = block_on(async move { + ffi_msg + .message + .quoted_message(context) + .await + .log_err(context, "failed to get quoted message") + .unwrap_or(None) + }); + + match res { + Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })), + None => ptr::null_mut(), + } +} + // dc_contact_t /// FFI struct for [dc_contact_t] diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 9912d3855..756a9bc79 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -175,6 +175,27 @@ class Message(object): if ts: return datetime.utcfromtimestamp(ts) + @property + def quoted_text(self): + """Text inside the quote + + :returns: Quoted text""" + return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg)) + + @property + def quote(self): + """Quote getter + + :returns: Quoted message, if found in the database""" + msg = lib.dc_msg_get_quoted_msg(self._dc_msg) + if msg: + return Message(self.account, ffi.gc(msg, lib.dc_msg_unref)) + + @quote.setter + def quote(self, quoted_message): + """Quote setter""" + lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg) + def get_mime_headers(self): """ return mime-header object for an incoming message. diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 0be5e4aeb..b387849cf 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -476,6 +476,21 @@ class TestOfflineChat: assert not res.is_ask_verifygroup() assert res.contact_id == 10 + def test_quote(self, chat1): + """Offline quoting test""" + msg = Message.new_empty(chat1.account, "text") + msg.set_text("message") + assert msg.quoted_text is None + + # Prepare message to assign it a Message-Id. + # Messages without Message-Id cannot be quoted. + msg = chat1.prepare_message(msg) + + reply_msg = Message.new_empty(chat1.account, "text") + reply_msg.set_text("reply") + reply_msg.quote = msg + assert reply_msg.quoted_text == "message" + def test_group_chat_many_members_add_remove(self, ac1, lp): lp.sec("ac1: creating group chat with 10 other members") chat = ac1.create_group_chat(name="title1") @@ -1067,8 +1082,8 @@ class TestOnlineAccount: # Majority prefers encryption now assert msg5.is_encrypted() - @pytest.mark.xfail(reason="Sticky encryption rule was removed") def test_reply_encrypted(self, acfactory, lp): + """Test that replies to encrypted messages are encrypted.""" ac1, ac2 = acfactory.get_two_online_accounts() lp.sec("ac1: create chat with ac2") @@ -1096,26 +1111,26 @@ class TestOnlineAccount: print("ac2: e2ee_enabled={}".format(ac2.get_config("e2ee_enabled"))) ac1.set_config("e2ee_enabled", "0") - # Set unprepared and unencrypted draft to test that it is not - # taken into account when determining whether last message is - # encrypted. - msg_draft = Message.new_empty(ac1, "text") - msg_draft.set_text("message2 -- should be encrypted") - chat.set_draft(msg_draft) + for quoted_msg in msg1, msg3: + # Save the draft with a quote. + # It should be encrypted if quoted message is encrypted. + msg_draft = Message.new_empty(ac1, "text") + msg_draft.set_text("message reply") + msg_draft.quote = quoted_msg + chat.set_draft(msg_draft) - # Get the draft, prepare and send it. - msg_draft = chat.get_draft() - msg_out = chat.prepare_message(msg_draft) - chat.send_prepared(msg_out) + # Get the draft, prepare and send it. + msg_draft = chat.get_draft() + msg_out = chat.prepare_message(msg_draft) + chat.send_prepared(msg_out) - chat.set_draft(None) - assert chat.get_draft() is None + chat.set_draft(None) + assert chat.get_draft() is None - lp.sec("wait for ac2 to receive message") - ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg_in = ac2.get_message_by_id(ev.data2) - assert msg_in.text == "message2 -- should be encrypted" - assert msg_in.is_encrypted() + msg_in = ac2._evtracker.wait_next_incoming_message() + assert msg_in.text == "message reply" + assert msg_in.quoted_text == quoted_msg.text + assert msg_in.is_encrypted() == quoted_msg.is_encrypted() def test_saved_mime_on_received_message(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -1869,6 +1884,45 @@ class TestOnlineAccount: # No renames should happen after explicit rename assert updated_name == "Renamed" + def test_group_quote(self, acfactory, lp): + """Test quoting in a group with a new member who have not seen the quoted message.""" + ac1, ac2, ac3 = accounts = acfactory.get_many_online_accounts(3) + acfactory.introduce_each_other(accounts) + chat = ac1.create_group_chat(name="quote group") + chat.add_contact(ac2) + + lp.sec("ac1: sending message") + out_msg = chat.send_text("hello") + + lp.sec("ac2: receiving message") + msg = ac2._evtracker.wait_next_incoming_message() + assert msg.text == "hello" + + chat.add_contact(ac3) + ac2._evtracker.wait_next_incoming_message() + ac3._evtracker.wait_next_incoming_message() + + lp.sec("ac2: sending reply with a quote") + reply_msg = Message.new_empty(msg.chat.account, "text") + reply_msg.set_text("reply") + reply_msg.quote = msg + reply_msg = msg.chat.prepare_message(reply_msg) + assert reply_msg.quoted_text == "hello" + msg.chat.send_prepared(reply_msg) + + lp.sec("ac3: receiving reply") + received_reply = ac3._evtracker.wait_next_incoming_message() + assert received_reply.text == "reply" + assert received_reply.quoted_text == "hello" + # ac3 was not in the group and has not received quoted message + assert received_reply.quote is None + + lp.sec("ac1: receiving reply") + received_reply = ac1._evtracker.wait_next_incoming_message() + assert received_reply.text == "reply" + assert received_reply.quoted_text == "hello" + assert received_reply.quote.id == out_msg.id + class TestGroupStressTests: def test_group_many_members_add_leave_remove(self, acfactory, lp): diff --git a/src/chat.rs b/src/chat.rs index cbe0cec40..725428eb4 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -753,7 +753,6 @@ impl Chat { timestamp: i64, ) -> Result { let mut new_references = "".into(); - let mut new_in_reply_to = "".into(); let mut msg_id = 0; let mut to_id = 0; let mut location_id = 0; @@ -828,8 +827,11 @@ impl Chat { if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) = self.id.get_parent_mime_headers(context).await { - if !parent_rfc724_mid.is_empty() { - new_in_reply_to = parent_rfc724_mid.clone(); + // "In-Reply-To:" is not changed if it is set manually. + // This does not affect "References:" header, it will contain "default parent" (the + // latest message in the thread) anyway. + if msg.in_reply_to.is_none() && !parent_rfc724_mid.is_empty() { + msg.in_reply_to = Some(parent_rfc724_mid.clone()); } // the whole list of messages referenced may be huge; @@ -928,7 +930,7 @@ impl Chat { msg.text.as_ref().cloned().unwrap_or_default(), msg.param.to_string(), msg.hidden, - new_in_reply_to, + msg.in_reply_to.as_deref().unwrap_or_default(), new_references, location_id as i32, ephemeral_timer, diff --git a/src/format_flowed.rs b/src/format_flowed.rs index bf3fe14e1..45701cdce 100644 --- a/src/format_flowed.rs +++ b/src/format_flowed.rs @@ -21,21 +21,28 @@ /// length. However, this should be rare and should not result in /// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters, /// and Spam Assassin limit is 78 characters. -fn format_line_flowed(line: &str) -> String { +fn format_line_flowed(line: &str, prefix: &str) -> String { let mut result = String::new(); - let mut buffer = String::new(); + let mut buffer = prefix.to_string(); let mut after_space = false; for c in line.chars() { if c == ' ' { buffer.push(c); after_space = true; + } else if c == '>' { + if buffer.is_empty() { + // Space stuffing, see RFC 3676 + buffer.push(' '); + } + buffer.push(c); + after_space = false; } else { - if after_space && buffer.len() >= 72 && !c.is_whitespace() && c != '>' { + if after_space && buffer.len() >= 72 && !c.is_whitespace() { // Flush the buffer and insert soft break (SP CRLF). result += &buffer; result += "\r\n"; - buffer = String::new(); + buffer = prefix.to_string(); } buffer.push(c); after_space = false; @@ -44,6 +51,28 @@ fn format_line_flowed(line: &str) -> String { result + &buffer } +fn format_flowed_prefix(text: &str, prefix: &str) -> String { + let mut result = String::new(); + + for line in text.split('\n') { + if !result.is_empty() { + result += "\r\n"; + } + let line = line.trim_end(); + if prefix.len() + line.len() > 78 { + result += &format_line_flowed(line, prefix); + } else { + result += prefix; + if prefix.is_empty() && line.starts_with('>') { + // Space stuffing, see RFC 3676 + result.push(' '); + } + result += line; + } + } + result +} + /// Returns text formatted according to RFC 3767 (format=flowed). /// /// This function accepts text separated by LF, but returns text @@ -52,20 +81,12 @@ fn format_line_flowed(line: &str) -> String { /// RFC 2646 technique is used to insert soft line breaks, so DelSp /// SHOULD be set to "no" when sending. pub fn format_flowed(text: &str) -> String { - let mut result = String::new(); + format_flowed_prefix(text, "") +} - for line in text.split('\n') { - if !result.is_empty() { - result += "\r\n"; - } - let line = line.trim_end(); - if line.len() > 78 { - result += &format_line_flowed(line); - } else { - result += line; - } - } - result +/// Same as format_flowed(), but adds "> " prefix to each line. +pub fn format_flowed_quote(text: &str) -> String { + format_flowed_prefix(text, "> ") } /// Joins lines in format=flowed text. @@ -122,6 +143,9 @@ mod tests { To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\ client and enter the setup code presented on the generating device."; assert_eq!(format_flowed(text), expected); + + let text = "> Not a quote"; + assert_eq!(format_flowed(text), " > Not a quote"); } #[test] @@ -133,4 +157,21 @@ mod tests { unwrapped on the receiver"; assert_eq!(unformat_flowed(text, false), expected); } + + #[test] + fn test_format_flowed_quote() { + let quote = "this is a quoted line"; + let expected = "> this is a quoted line"; + assert_eq!(format_flowed_quote(quote), expected); + + let quote = "> foo bar baz"; + let expected = "> > foo bar baz"; + assert_eq!(format_flowed_quote(quote), expected); + + let quote = "this is a very long quote that should be wrapped using format=flowed and unwrapped on the receiver"; + let expected = + "> this is a very long quote that should be wrapped using format=flowed and \r\n\ + > unwrapped on the receiver"; + assert_eq!(format_flowed_quote(quote), expected); + } } diff --git a/src/message.rs b/src/message.rs index c3a1ce669..671446f56 100644 --- a/src/message.rs +++ b/src/message.rs @@ -741,6 +741,55 @@ impl Message { self.update_param(context).await; } + /// Sets message quote. + /// + /// Message-Id is used to set Reply-To field, message text is used for quote. + /// + /// Encryption is required if quoted message was encrypted. + /// + /// The message itself is not required to exist in the database, + /// it may even be deleted from the database by the time the message is prepared. + pub fn set_quote(&mut self, quote: &Message) -> Result<(), Error> { + ensure!( + !quote.rfc724_mid.is_empty(), + "Message without Message-Id cannot be quoted" + ); + self.in_reply_to = Some(quote.rfc724_mid.clone()); + + if quote + .param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or_default() + { + self.param.set(Param::GuaranteeE2ee, "1"); + } + + self.param + .set(Param::Quote, quote.get_text().unwrap_or_default()); + + Ok(()) + } + + pub fn quoted_text(&self) -> Option { + self.param.get(Param::Quote).map(|s| s.to_string()) + } + + pub async fn quoted_message(&self, context: &Context) -> Result, Error> { + if self.param.get(Param::Quote).is_some() { + if let Some(in_reply_to) = &self.in_reply_to { + if let Some((_folder, _uid, msg_id)) = rfc724_mid_exists( + context, + in_reply_to.trim_start_matches('<').trim_end_matches('>'), + ) + .await? + { + return Ok(Some(Message::load_from_db(context, msg_id).await?)); + } + } + } + Ok(None) + } + pub async fn update_param(&mut self, context: &Context) -> bool { context .sql @@ -2014,4 +2063,43 @@ mod tests { } assert!(has_image); } + + #[async_std::test] + async fn test_quote() { + use crate::config::Config; + + let d = test::TestContext::new().await; + let ctx = &d.ctx; + + let contact = Contact::create(ctx, "", "dest@example.com") + .await + .expect("failed to create contact"); + + let res = ctx + .set_config(Config::ConfiguredAddr, Some("self@example.com")) + .await; + assert!(res.is_ok()); + + let chat = chat::create_by_contact_id(ctx, contact).await.unwrap(); + + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("Quoted message".to_string())); + + // Prepare message for sending, so it gets a Message-Id. + assert!(msg.rfc724_mid.is_empty()); + let msg_id = chat::prepare_msg(ctx, chat, &mut msg).await.unwrap(); + let msg = Message::load_from_db(ctx, msg_id).await.unwrap(); + assert!(!msg.rfc724_mid.is_empty()); + + let mut msg2 = Message::new(Viewtype::Text); + msg2.set_quote(&msg).expect("can't set quote"); + assert!(msg2.quoted_text() == msg.get_text()); + + let quoted_msg = msg2 + .quoted_message(ctx) + .await + .expect("error while retrieving quoted message") + .expect("quoted message not found"); + assert!(quoted_msg.get_text() == msg2.quoted_text()); + } } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 1fac00535..24434f603 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -11,7 +11,7 @@ use crate::dc_tools::*; use crate::e2ee::*; use crate::ephemeral::Timer as EphemeralTimer; use crate::error::{bail, ensure, format_err, Error}; -use crate::format_flowed::format_flowed; +use crate::format_flowed::{format_flowed, format_flowed_quote}; use crate::location; use crate::message::{self, Message}; use crate::mimeparser::SystemMessage; @@ -917,12 +917,17 @@ impl<'a, 'b> MimeFactory<'a, 'b> { } }; + let quoted_text = self + .msg + .quoted_text() + .map(|quote| format_flowed_quote("e) + "\r\n"); let flowed_text = format_flowed(final_text); let footer = &self.selfstatus; let message_text = format!( - "{}{}{}{}{}", + "{}{}{}{}{}{}", fwdhint.unwrap_or_default(), + quoted_text.unwrap_or_default(), escape_message_footer_marks(&flowed_text), if !final_text.is_empty() && !footer.is_empty() { "\r\n\r\n" diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 4ec6daefa..dff6d16b4 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -704,8 +704,8 @@ impl MimeMessage { } }; - let (simplified_txt, is_forwarded) = if decoded_data.is_empty() { - ("".into(), false) + let (simplified_txt, is_forwarded, top_quote) = if decoded_data.is_empty() { + ("".to_string(), false, None) } else { let is_html = mime_type == mime::TEXT_HTML; let out = if is_html { @@ -723,7 +723,7 @@ impl MimeMessage { false }; - let simplified_txt = if mime_type.type_() == mime::TEXT + let (simplified_txt, simplified_quote) = if mime_type.type_() == mime::TEXT && mime_type.subtype() == mime::PLAIN && is_format_flowed { @@ -732,9 +732,11 @@ impl MimeMessage { } else { false }; - unformat_flowed(&simplified_txt, delsp) + let unflowed_text = unformat_flowed(&simplified_txt, delsp); + let unflowed_quote = top_quote.map(|q| unformat_flowed(&q, delsp)); + (unflowed_text, unflowed_quote) } else { - simplified_txt + (simplified_txt, top_quote) }; if !simplified_txt.is_empty() { @@ -742,6 +744,9 @@ impl MimeMessage { part.typ = Viewtype::Text; part.mimetype = Some(mime_type); part.msg = simplified_txt; + if let Some(quote) = simplified_quote { + part.param.set(Param::Quote, quote); + } part.msg_raw = Some(decoded_data); self.do_add_single_part(part); } @@ -2119,12 +2124,39 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= assert!(test.is_empty()); } - #[test] - fn test_mime_parse_format_flowed() { - let mime_type = "text/plain; charset=utf-8; Format=Flowed; DelSp=No" - .parse::() + #[async_std::test] + async fn parse_format_flowed_quote() { + let context = TestContext::new().await; + let raw = br##"Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no +Subject: Re: swipe-to-reply +MIME-Version: 1.0 +In-Reply-To: +Date: Tue, 06 Oct 2020 00:00:00 +0000 +Chat-Version: 1.0 +Message-ID: +To: bob +From: alice + +> Long +> quote. + +Reply +"##; + + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]) + .await .unwrap(); - let format_param = mime_type.get_param("format").unwrap(); - assert_eq!(format_param.as_str().to_ascii_lowercase(), "flowed"); + assert_eq!( + message.get_subject(), + Some("Re: swipe-to-reply".to_string()) + ); + + assert_eq!(message.parts.len(), 1); + assert_eq!(message.parts[0].typ, Viewtype::Text); + assert_eq!( + message.parts[0].param.get(Param::Quote).unwrap(), + "Long quote." + ); + assert_eq!(message.parts[0].msg, "Reply"); } } diff --git a/src/param.rs b/src/param.rs index 0f901f7b8..ef452774e 100644 --- a/src/param.rs +++ b/src/param.rs @@ -53,6 +53,9 @@ pub enum Param { /// For Messages Forwarded = b'a', + /// For Messages: quoted text. + Quote = b'q', + /// For Messages Cmd = b'S', diff --git a/src/simplify.rs b/src/simplify.rs index 88c613835..aa95ee0fb 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -1,3 +1,5 @@ +use itertools::Itertools; + // protect lines starting with `--` against being treated as a footer. // for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B); // this should be invisible on most systems and there is no need to unescape it again @@ -64,7 +66,7 @@ fn split_lines(buf: &str) -> Vec<&str> { /// Simplify message text for chat display. /// Remove quotes, signatures, trailing empty lines etc. -pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) { +pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool, Option) { input.retain(|c| c != '\r'); let lines = split_lines(&input); let (lines, is_forwarded) = skip_forward_header(&lines); @@ -72,25 +74,25 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) { let original_lines = &lines; let lines = remove_message_footer(lines); + let (lines, top_quote) = remove_top_quote(lines); let text = if is_chat_message { render_message(lines, false, false) } else { let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines); let (lines, has_bottom_quote) = remove_bottom_quote(lines); - let (lines, has_top_quote) = remove_top_quote(lines); if lines.iter().all(|it| it.trim().is_empty()) { render_message(original_lines, false, false) } else { render_message( lines, - has_top_quote, + top_quote.is_some(), has_nonstandard_footer || has_bottom_quote, ) } }; - (text, is_forwarded) + (text, is_forwarded, top_quote) } /// Skips "forwarded message" header. @@ -134,11 +136,15 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) { } #[allow(clippy::indexing_slicing)] -fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) { +fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option) { + let mut first_quoted_line = 0; let mut last_quoted_line = None; let mut has_quoted_headline = false; for (l, line) in lines.iter().enumerate() { if is_plain_quote(line) { + if last_quoted_line.is_none() { + first_quoted_line = l; + } last_quoted_line = Some(l) } else if !is_empty_line(line) { if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() { @@ -150,9 +156,20 @@ fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) { } } if let Some(last_quoted_line) = last_quoted_line { - (&lines[last_quoted_line + 1..], true) + ( + &lines[last_quoted_line + 1..], + Some( + lines[first_quoted_line..last_quoted_line + 1] + .iter() + .map(|s| { + s.strip_prefix(">") + .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u)) + }) + .join("\n"), + ), + ) } else { - (lines, false) + (lines, None) } } @@ -231,7 +248,7 @@ mod tests { #[test] // proptest does not support [[:graphical:][:space:]] regex. fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") { - let (output, _is_forwarded) = simplify(input, true); + let (output, _is_forwarded, _) = simplify(input, true); assert!(output.split('\n').all(|s| s != "-- ")); } } @@ -239,7 +256,7 @@ mod tests { #[test] fn test_dont_remove_whole_message() { let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string(); - let (plain, is_forwarded) = simplify(input, false); + let (plain, is_forwarded, _) = simplify(input, false); assert_eq!( plain, "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text" @@ -250,7 +267,7 @@ mod tests { #[test] fn test_chat_message() { let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string(); - let (plain, is_forwarded) = simplify(input, true); + let (plain, is_forwarded, _) = simplify(input, true); assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good."); assert!(!is_forwarded); } @@ -258,7 +275,7 @@ mod tests { #[test] fn test_simplify_trim() { let input = "line1\n\r\r\rline2".to_string(); - let (plain, is_forwarded) = simplify(input, false); + let (plain, is_forwarded, _) = simplify(input, false); assert_eq!(plain, "line1\nline2"); assert!(!is_forwarded); @@ -267,7 +284,7 @@ mod tests { #[test] fn test_simplify_forwarded_message() { let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string(); - let (plain, is_forwarded) = simplify(input, false); + let (plain, is_forwarded, _) = simplify(input, false); assert_eq!(plain, "Forwarded message"); assert!(is_forwarded); @@ -287,17 +304,17 @@ mod tests { #[test] fn test_remove_top_quote() { - let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]); + let (lines, top_quote) = remove_top_quote(&["> first", "> second"]); assert!(lines.is_empty()); - assert!(has_top_quote); + assert_eq!(top_quote.unwrap(), "first\nsecond"); - let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]); + let (lines, top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]); assert_eq!(lines, &["not a quote"]); - assert!(has_top_quote); + assert_eq!(top_quote.unwrap(), "first\nsecond"); - let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]); + let (lines, top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]); assert_eq!(lines, &["not a quote", "> first", "> second"]); - assert!(!has_top_quote); + assert!(top_quote.is_none()); } #[test] @@ -312,41 +329,41 @@ mod tests { #[test] fn test_remove_message_footer() { let input = "text\n--\nno footer".to_string(); - let (plain, _) = simplify(input, true); + let (plain, _, _) = simplify(input, true); assert_eq!(plain, "text\n--\nno footer"); let input = "text\n\n--\n\nno footer".to_string(); - let (plain, _) = simplify(input, true); + let (plain, _, _) = simplify(input, true); assert_eq!(plain, "text\n\n--\n\nno footer"); let input = "text\n\n-- no footer\n\n".to_string(); - let (plain, _) = simplify(input, true); + let (plain, _, _) = simplify(input, true); assert_eq!(plain, "text\n\n-- no footer"); let input = "text\n\n--\nno footer\n-- \nfooter".to_string(); - let (plain, _) = simplify(input, true); + let (plain, _, _) = simplify(input, true); assert_eq!(plain, "text\n\n--\nno footer"); let input = "text\n\n--\ntreated as footer when unescaped".to_string(); - let (plain, _) = simplify(input.clone(), true); + let (plain, _, _) = simplify(input.clone(), true); assert_eq!(plain, "text"); // see remove_message_footer() for some explanations let escaped = escape_message_footer_marks(&input); - let (plain, _) = simplify(escaped, true); + let (plain, _, _) = simplify(escaped, true); assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped"); // Nonstandard footer sent by https://siju.es/ let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string(); - let (plain, _) = simplify(input.clone(), false); + let (plain, _, _) = simplify(input.clone(), false); assert_eq!(plain, "Message text here [...]"); - let (plain, _) = simplify(input.clone(), true); + let (plain, _, _) = simplify(input.clone(), true); assert_eq!(plain, input); let input = "--\ntreated as footer when unescaped".to_string(); - let (plain, _) = simplify(input.clone(), true); + let (plain, _, _) = simplify(input.clone(), true); assert_eq!(plain, ""); // see remove_message_footer() for some explanations let escaped = escape_message_footer_marks(&input); - let (plain, _) = simplify(escaped, true); + let (plain, _, _) = simplify(escaped, true); assert_eq!(plain, "--\ntreated as footer when unescaped"); } }