From bfdd6f36e213a271ec09afc8624921f1be68d4d5 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 27 Apr 2020 12:24:12 +0200 Subject: [PATCH 1/3] regard line with ony '--' as footer mark partly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the footer mark normally used in email-conversations is `-- `, note the trailing space, see RFC 3676, §4.3 unfortunately, the final space is removed by some providers, which lead to footers showing up on delta-to-delta-conversations (on nondc-to-delta, this is not an issue as we cannot be sure anyway and show a [...] therefore) this change accepts lines with only `--` as a footer separator if there is no other footer separator and if the line before is empty and the line after is not. as there is still some chance to remove text accidentally, see tests, some protection against that is needed in another commit. --- src/simplify.rs | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index cf61dc8ae..985639cf1 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -1,13 +1,25 @@ /// Remove standard (RFC 3676, §4.3) footer if it is found. fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] { + let mut nearly_standard_footer = None; for (ix, &line) in lines.iter().enumerate() { - // quoted-printable may encode `-- ` to `-- =20` which is converted - // back to `-- ` match line { + // some providers encode `-- ` to `-- =20` which results in `-- ` "-- " | "-- " => return &lines[..ix], + // some providers encode `-- ` to `=2D-` which results in only `--`; + // use that only when no other footer is found + // and if the line before is empty and the line after is not empty + "--" => { + if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != "" + { + nearly_standard_footer = Some(ix); + } + } _ => (), } } + if let Some(ix) = nearly_standard_footer { + return &lines[..ix]; + } lines } @@ -268,4 +280,31 @@ mod tests { assert_eq!(lines, &["not a quote", "> first", "> second"]); assert!(!has_top_quote); } + + #[test] + fn test_remove_message_footer() { + let input = "text\n--\nno footer".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n--\nno footer"); + + let input = "text\n\n--\n\nno footer".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n\n--\n\nno footer"); + + let input = "text\n\n-- no footer\n\n".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n\n-- no footer"); + + let input = "text\n\n--\nno footer\n-- \nfooter".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n\n--\nno footer"); + + let input = "text\n\n--\ntreated as footer when unescaped".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text"); // see remove_message_footer() for some explanations + + let input = "--\ntreated as footer when unescaped".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, ""); + } } From 459fec56db7b851025427640ad5f016ca3fc9424 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 27 Apr 2020 16:02:45 +0200 Subject: [PATCH 2/3] protect '--' in message from being treated as a footer-beginning --- src/mimefactory.rs | 3 ++- src/simplify.rs | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 127a23111..078e608e5 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -15,6 +15,7 @@ use crate::message::{self, Message}; use crate::mimeparser::SystemMessage; use crate::param::*; use crate::peerstate::{Peerstate, PeerstateVerifiedStatus}; +use crate::simplify::escape_message_footer_marks; use crate::stock::StockMessage; // attachments of 25 mb brutto should work on the majority of providers @@ -807,7 +808,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> { let message_text = format!( "{}{}{}{}{}", fwdhint.unwrap_or_default(), - &final_text, + escape_message_footer_marks(final_text), if !final_text.is_empty() && !footer.is_empty() { "\r\n\r\n" } else { diff --git a/src/simplify.rs b/src/simplify.rs index 985639cf1..eb8bc93c6 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -1,3 +1,19 @@ +// protect lines starting with `--` against being treated as a footer. +// for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B); +// this should be invisible on most systems and there is no need to unescape it again +// (which won't be done by non-deltas anyway) +// +// this escapes a bit more than actually needed by delta (eg. also lines as "-- footer"), +// but for non-delta-compatibility, that seems to be better. +// (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced) +pub fn escape_message_footer_marks(text: &str) -> String { + if text.starts_with("--") { + "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-") + } else { + text.replace("\n--", "\n-\u{200B}-") + } +} + /// Remove standard (RFC 3676, §4.3) footer if it is found. fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] { let mut nearly_standard_footer = None; @@ -281,6 +297,15 @@ mod tests { assert!(!has_top_quote); } + #[test] + fn test_escape_message_footer_marks() { + let esc = escape_message_footer_marks("--\n--text --in line"); + assert_eq!(esc, "-\u{200B}-\n-\u{200B}-text --in line"); + + let esc = escape_message_footer_marks("--\r\n--text"); + assert_eq!(esc, "-\u{200B}-\r\n-\u{200B}-text"); + } + #[test] fn test_remove_message_footer() { let input = "text\n--\nno footer".to_string(); @@ -300,11 +325,20 @@ mod tests { assert_eq!(plain, "text\n\n--\nno footer"); let input = "text\n\n--\ntreated as footer when unescaped".to_string(); - let (plain, _) = simplify(input, true); + let (plain, _) = simplify(input.clone(), true); assert_eq!(plain, "text"); // see remove_message_footer() for some explanations + let escaped = escape_message_footer_marks(&input); + let (plain, _) = simplify(escaped, true); + assert_eq!( + plain, + "text\n\n-\u{200B}-\ntreated as footer when unescaped" + ); let input = "--\ntreated as footer when unescaped".to_string(); - let (plain, _) = simplify(input, true); - assert_eq!(plain, ""); + let (plain, _) = simplify(input.clone(), true); + assert_eq!(plain, ""); // see remove_message_footer() for some explanations + let escaped = escape_message_footer_marks(&input); + let (plain, _) = simplify(escaped, true); + assert_eq!(plain, "-\u{200B}-\ntreated as footer when unescaped"); } } From 2a9b967d2d52a6367970c83022588ffb77498cfc Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 27 Apr 2020 16:32:16 +0200 Subject: [PATCH 3/3] remove footer-escape-character from message texts --- src/simplify.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index eb8bc93c6..79d6f6605 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -184,7 +184,8 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> if is_cut_at_end && (!is_cut_at_begin || !empty_body) { ret += " [...]"; } - ret + // redo escaping done by escape_message_footer_marks() + ret.replace("\u{200B}", "") } /** @@ -329,16 +330,13 @@ mod tests { assert_eq!(plain, "text"); // see remove_message_footer() for some explanations let escaped = escape_message_footer_marks(&input); let (plain, _) = simplify(escaped, true); - assert_eq!( - plain, - "text\n\n-\u{200B}-\ntreated as footer when unescaped" - ); + assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped"); let input = "--\ntreated as footer when unescaped".to_string(); let (plain, _) = simplify(input.clone(), true); assert_eq!(plain, ""); // see remove_message_footer() for some explanations let escaped = escape_message_footer_marks(&input); let (plain, _) = simplify(escaped, true); - assert_eq!(plain, "-\u{200B}-\ntreated as footer when unescaped"); + assert_eq!(plain, "--\ntreated as footer when unescaped"); } }