Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-05-09 01:46:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add SimplifiedText structure

Browse Source 
Return structure from `simplify` instead of 5-tuple.
This commit is contained in:
link2xt
2022-06-12 21:08:32 +00:00
parent 8a4e07c83e
commit d947479a60
3 changed files with 159 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/dehtml.rs
View File

@@ -287,7 +287,7 @@ pub fn dehtml_manually(buf: &str) -> String {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::simplify::simplify; use crate::simplify::{simplify, SimplifiedText};
#[test] #[test]
fn test_dehtml() { fn test_dehtml() {
@@ -311,7 +311,7 @@ mod tests {
("<!some invalid html code>\n<b>some text</b>", "some text"), ("<!some invalid html code>\n<b>some text</b>", "some text"),
]; ];
for (input, output) in cases { for (input, output) in cases {
assert_eq!(simplify(dehtml(input).unwrap(), true).0, output); assert_eq!(simplify(dehtml(input).unwrap(), true).text, output);
} }
let none_cases = vec!["<html> </html>", ""]; let none_cases = vec!["<html> </html>", ""];
for input in none_cases { for input in none_cases {
@@ -387,10 +387,16 @@ mod tests {
let input = include_str!("../test-data/message/gmx-quote-body.eml"); let input = include_str!("../test-data/message/gmx-quote-body.eml");
let dehtml = dehtml(input).unwrap(); let dehtml = dehtml(input).unwrap();
println!("{}", dehtml); println!("{}", dehtml);
let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false); let SimplifiedText {
assert_eq!(msg, "Test"); text,
assert_eq!(forwarded, false); is_forwarded,
assert_eq!(cut, false); is_cut,
top_quote,
footer,
} = simplify(dehtml, false);
assert_eq!(text, "Test");
assert_eq!(is_forwarded, false);
assert_eq!(is_cut, false);
assert_eq!(top_quote.as_deref(), Some("test")); assert_eq!(top_quote.as_deref(), Some("test"));
assert_eq!(footer, None); assert_eq!(footer, None);
} }

35
src/mimeparser.rs
View File

@@ -27,7 +27,7 @@ use crate::location;
use crate::message::{self, Viewtype}; use crate::message::{self, Viewtype};
use crate::param::{Param, Params}; use crate::param::{Param, Params};
use crate::peerstate::Peerstate; use crate::peerstate::Peerstate;
use crate::simplify::simplify; use crate::simplify::{simplify, SimplifiedText};
use crate::stock_str; use crate::stock_str;
use crate::sync::SyncItems; use crate::sync::SyncItems;
@@ -927,22 +927,27 @@ impl MimeMessage {
let mut dehtml_failed = false; let mut dehtml_failed = false;
let (simplified_txt, is_forwarded, is_cut, top_quote, footer) = let SimplifiedText {
if decoded_data.is_empty() { text: simplified_txt,
("".to_string(), false, false, None, None) is_forwarded,
} else { is_cut,
let is_html = mime_type == mime::TEXT_HTML; top_quote,
let out = if is_html { footer,
self.is_mime_modified = true; } = if decoded_data.is_empty() {
dehtml(&decoded_data).unwrap_or_else(|| { Default::default()
dehtml_failed = true; } else {
decoded_data.clone() let is_html = mime_type == mime::TEXT_HTML;
}) let out = if is_html {
} else { self.is_mime_modified = true;
dehtml(&decoded_data).unwrap_or_else(|| {
dehtml_failed = true;
decoded_data.clone() decoded_data.clone()
}; })
simplify(out, self.has_chat_version()) } else {
decoded_data.clone()
}; };
simplify(out, self.has_chat_version())
};
self.is_mime_modified = self.is_mime_modified self.is_mime_modified = self.is_mime_modified
|| ((is_forwarded || is_cut || top_quote.is_some()) || ((is_forwarded || is_cut || top_quote.is_some())

164
src/simplify.rs
View File

@@ -71,14 +71,27 @@ pub(crate) fn split_lines(buf: &str) -> Vec<&str> {
buf.split('\n').collect() buf.split('\n').collect()
} }
/// Simplified text and some additional information gained from the input.
#[derive(Debug, Default)]
pub(crate) struct SimplifiedText {
pub text: String,
/// True if the message is forwarded.
pub is_forwarded: bool,
/// True if nonstandard footer was removed.
pub is_cut: bool,
/// Top quote, if any.
pub top_quote: Option<String>,
/// Footer, if any.
pub footer: Option<String>,
}
/// Simplify message text for chat display. /// Simplify message text for chat display.
/// Remove quotes, signatures, trailing empty lines etc. /// Remove quotes, signatures, trailing empty lines etc.
/// Returns `(text, is_forwarded, is_cut, quote, footer)` tuple, pub(crate) fn simplify(mut input: String, is_chat_message: bool) -> SimplifiedText {
/// returning the simplified text and some additional information gained from the input.
pub fn simplify(
mut input: String,
is_chat_message: bool,
) -> (String, bool, bool, Option<String>, Option<String>) {
let mut is_cut = false; let mut is_cut = false;
input.retain(|c| c != '\r'); input.retain(|c| c != '\r');
@@ -118,7 +131,14 @@ pub fn simplify(
render_message(quote_lines, false) render_message(quote_lines, false)
}); });
} }
(text, is_forwarded, is_cut, top_quote, footer)
SimplifiedText {
text,
is_forwarded,
is_cut,
top_quote,
footer,
}
} }
/// Skips "forwarded message" header. /// Skips "forwarded message" header.
@@ -273,17 +293,25 @@ mod tests {
#[test] #[test]
// proptest does not support [[:graphical:][:space:]] regex. // proptest does not support [[:graphical:][:space:]] regex.
fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") { fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
let (output, _is_forwarded, _, _, _) = simplify(input, true); let SimplifiedText {
assert!(output.split('\n').all(|s| s != "-- ")); text,
..
} = simplify(input, true);
assert!(text.split('\n').all(|s| s != "-- "));
} }
} }
#[test] #[test]
fn test_dont_remove_whole_message() { fn test_dont_remove_whole_message() {
let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string(); let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
let (plain, is_forwarded, is_cut, _, _) = simplify(input, false); let SimplifiedText {
text,
is_forwarded,
is_cut,
..
} = simplify(input, false);
assert_eq!( assert_eq!(
plain, text,
"------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text" "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
); );
assert!(!is_forwarded); assert!(!is_forwarded);
@@ -293,8 +321,14 @@ mod tests {
#[test] #[test]
fn test_chat_message() { fn test_chat_message() {
let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string(); let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
let (plain, is_forwarded, is_cut, _, footer) = simplify(input, true); let SimplifiedText {
assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good."); text,
is_forwarded,
is_cut,
footer,
..
} = simplify(input, true);
assert_eq!(text, "Hi! How are you?\n\n---\n\nI am good.");
assert!(!is_forwarded); assert!(!is_forwarded);
assert!(is_cut); assert!(is_cut);
assert_eq!( assert_eq!(
@@ -306,9 +340,14 @@ mod tests {
#[test] #[test]
fn test_simplify_trim() { fn test_simplify_trim() {
let input = "line1\n\r\r\rline2".to_string(); let input = "line1\n\r\r\rline2".to_string();
let (plain, is_forwarded, is_cut, _, _) = simplify(input, false); let SimplifiedText {
text,
is_forwarded,
is_cut,
..
} = simplify(input, false);
assert_eq!(plain, "line1\nline2"); assert_eq!(text, "line1\nline2");
assert!(!is_forwarded); assert!(!is_forwarded);
assert!(!is_cut); assert!(!is_cut);
} }
@@ -316,9 +355,15 @@ mod tests {
#[test] #[test]
fn test_simplify_forwarded_message() { fn test_simplify_forwarded_message() {
let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string(); let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
let (plain, is_forwarded, is_cut, _, footer) = simplify(input, false); let SimplifiedText {
text,
is_forwarded,
is_cut,
footer,
..
} = simplify(input, false);
assert_eq!(plain, "Forwarded message"); assert_eq!(text, "Forwarded message");
assert!(is_forwarded); assert!(is_forwarded);
assert!(is_cut); assert!(is_cut);
assert_eq!(footer.unwrap(), "Signature goes here"); assert_eq!(footer.unwrap(), "Signature goes here");
@@ -363,59 +408,104 @@ mod tests {
#[test] #[test]
fn test_remove_message_footer() { fn test_remove_message_footer() {
let input = "text\n--\nno footer".to_string(); let input = "text\n--\nno footer".to_string();
let (plain, _, is_cut, _, footer) = simplify(input, true); let SimplifiedText {
assert_eq!(plain, "text\n--\nno footer"); text,
is_cut,
footer,
..
} = simplify(input, true);
assert_eq!(text, "text\n--\nno footer");
assert_eq!(footer, None); assert_eq!(footer, None);
assert!(!is_cut); assert!(!is_cut);
let input = "text\n\n--\n\nno footer".to_string(); let input = "text\n\n--\n\nno footer".to_string();
let (plain, _, is_cut, _, footer) = simplify(input, true); let SimplifiedText {
assert_eq!(plain, "text\n\n--\n\nno footer"); text,
is_cut,
footer,
..
} = simplify(input, true);
assert_eq!(text, "text\n\n--\n\nno footer");
assert_eq!(footer, None); assert_eq!(footer, None);
assert!(!is_cut); assert!(!is_cut);
let input = "text\n\n-- no footer\n\n".to_string(); let input = "text\n\n-- no footer\n\n".to_string();
let (plain, _, _, _, footer) = simplify(input, true); let SimplifiedText { text, footer, .. } = simplify(input, true);
assert_eq!(plain, "text\n\n-- no footer"); assert_eq!(text, "text\n\n-- no footer");
assert_eq!(footer, None); assert_eq!(footer, None);
let input = "text\n\n--\nno footer\n-- \nfooter".to_string(); let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
let (plain, _, is_cut, _, footer) = simplify(input, true); let SimplifiedText {
assert_eq!(plain, "text\n\n--\nno footer"); text,
is_cut,
footer,
..
} = simplify(input, true);
assert_eq!(text, "text\n\n--\nno footer");
assert!(is_cut); assert!(is_cut);
assert_eq!(footer.unwrap(), "footer"); assert_eq!(footer.unwrap(), "footer");
let input = "text\n\n--\ntreated as footer when unescaped".to_string(); let input = "text\n\n--\ntreated as footer when unescaped".to_string();
let (plain, _, is_cut, _, footer) = simplify(input.clone(), true); let SimplifiedText {
assert_eq!(plain, "text"); // see remove_message_footer() for some explanations text,
is_cut,
footer,
..
} = simplify(input.clone(), true);
assert_eq!(text, "text"); // see remove_message_footer() for some explanations
assert!(is_cut); assert!(is_cut);
assert_eq!(footer.unwrap(), "treated as footer when unescaped"); assert_eq!(footer.unwrap(), "treated as footer when unescaped");
let escaped = escape_message_footer_marks(&input); let escaped = escape_message_footer_marks(&input);
let (plain, _, is_cut, _, footer) = simplify(escaped, true); let SimplifiedText {
assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped"); text,
is_cut,
footer,
..
} = simplify(escaped, true);
assert_eq!(text, "text\n\n--\ntreated as footer when unescaped");
assert!(!is_cut); assert!(!is_cut);
assert_eq!(footer, None); assert_eq!(footer, None);
// Nonstandard footer sent by <https://siju.es/> // Nonstandard footer sent by <https://siju.es/>
let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string(); let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
let (plain, _, is_cut, _, footer) = simplify(input.clone(), false); let SimplifiedText {
assert_eq!(plain, "Message text here [...]"); text,
is_cut,
footer,
..
} = simplify(input.clone(), false);
assert_eq!(text, "Message text here [...]");
assert!(is_cut); assert!(is_cut);
assert_eq!(footer, None); assert_eq!(footer, None);
let (plain, _, is_cut, _, footer) = simplify(input.clone(), true); let SimplifiedText {
assert_eq!(plain, input); text,
is_cut,
footer,
..
} = simplify(input.clone(), true);
assert_eq!(text, input);
assert!(!is_cut); assert!(!is_cut);
assert_eq!(footer, None); assert_eq!(footer, None);
let input = "--\ntreated as footer when unescaped".to_string(); let input = "--\ntreated as footer when unescaped".to_string();
let (plain, _, is_cut, _, footer) = simplify(input.clone(), true); let SimplifiedText {
assert_eq!(plain, ""); // see remove_message_footer() for some explanations text,
is_cut,
footer,
..
} = simplify(input.clone(), true);
assert_eq!(text, ""); // see remove_message_footer() for some explanations
assert!(is_cut); assert!(is_cut);
assert_eq!(footer.unwrap(), "treated as footer when unescaped"); assert_eq!(footer.unwrap(), "treated as footer when unescaped");
let escaped = escape_message_footer_marks(&input); let escaped = escape_message_footer_marks(&input);
let (plain, _, is_cut, _, footer) = simplify(escaped, true); let SimplifiedText {
assert_eq!(plain, "--\ntreated as footer when unescaped"); text,
is_cut,
footer,
..
} = simplify(escaped, true);
assert_eq!(text, "--\ntreated as footer when unescaped");
assert!(!is_cut); assert!(!is_cut);
assert_eq!(footer, None); assert_eq!(footer, None);
} }