//! De-HTML
//!
//! A module to remove HTML tags from the email text
use once_cell::sync::Lazy;
use quick_xml::events::{BytesEnd, BytesStart, BytesText};
static LINE_RE: Lazy = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
struct Dehtml {
strbuilder: String,
add_text: AddText,
last_href: Option,
}
#[derive(Debug, PartialEq)]
enum AddText {
No,
YesRemoveLineEnds,
YesPreserveLineEnds,
}
/// dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
/// the newlines are typically removed in further processing by the caller
pub fn dehtml(buf: &str) -> Option {
let s = dehtml_quick_xml(buf);
if !s.trim().is_empty() {
return Some(s);
}
let s = dehtml_manually(buf);
if !s.trim().is_empty() {
return Some(s);
}
None
}
pub fn dehtml_quick_xml(buf: &str) -> String {
let buf = buf.trim().trim_start_matches("");
let mut dehtml = Dehtml {
strbuilder: String::with_capacity(buf.len()),
add_text: AddText::YesRemoveLineEnds,
last_href: None,
};
let mut reader = quick_xml::Reader::from_str(buf);
reader.check_end_names(false);
let mut buf = Vec::new();
loop {
match reader.read_event(&mut buf) {
Ok(quick_xml::events::Event::Start(ref e)) => {
dehtml_starttag_cb(e, &mut dehtml, &reader)
}
Ok(quick_xml::events::Event::End(ref e)) => dehtml_endtag_cb(e, &mut dehtml),
Ok(quick_xml::events::Event::Text(ref e)) => dehtml_text_cb(e, &mut dehtml),
Ok(quick_xml::events::Event::CData(ref e)) => dehtml_cdata_cb(e, &mut dehtml),
Ok(quick_xml::events::Event::Empty(ref e)) => {
// Handle empty tags as a start tag immediately followed by end tag.
// For example, `` is treated as ``.
dehtml_starttag_cb(e, &mut dehtml, &reader);
dehtml_endtag_cb(&BytesEnd::borrowed(e.name()), &mut dehtml);
}
Err(e) => {
eprintln!(
"Parse html error: Error at position {}: {:?}",
reader.buffer_position(),
e
);
}
Ok(quick_xml::events::Event::Eof) => break,
_ => (),
}
buf.clear();
}
dehtml.strbuilder
}
fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
if dehtml.add_text == AddText::YesPreserveLineEnds
|| dehtml.add_text == AddText::YesRemoveLineEnds
{
let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
if dehtml.add_text == AddText::YesRemoveLineEnds {
dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
} else {
dehtml.strbuilder += &last_added;
}
}
}
fn dehtml_cdata_cb(event: &BytesText, dehtml: &mut Dehtml) {
if dehtml.add_text == AddText::YesPreserveLineEnds
|| dehtml.add_text == AddText::YesRemoveLineEnds
{
let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
if dehtml.add_text == AddText::YesRemoveLineEnds {
dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
} else {
dehtml.strbuilder += &last_added;
}
}
}
fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
match tag.as_str() {
"p" | "div" | "table" | "td" | "style" | "script" | "title" | "pre" => {
dehtml.strbuilder += "\n\n";
dehtml.add_text = AddText::YesRemoveLineEnds;
}
"a" => {
if let Some(ref last_href) = dehtml.last_href.take() {
dehtml.strbuilder += "](";
dehtml.strbuilder += last_href;
dehtml.strbuilder += ")";
}
}
"b" | "strong" => {
dehtml.strbuilder += "*";
}
"i" | "em" => {
dehtml.strbuilder += "_";
}
_ => {}
}
}
fn dehtml_starttag_cb(
event: &BytesStart,
dehtml: &mut Dehtml,
reader: &quick_xml::Reader,
) {
let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
match tag.as_str() {
"p" | "div" | "table" | "td" => {
dehtml.strbuilder += "\n\n";
dehtml.add_text = AddText::YesRemoveLineEnds;
}
"br" => {
dehtml.strbuilder += "\n";
dehtml.add_text = AddText::YesRemoveLineEnds;
}
"style" | "script" | "title" => {
dehtml.add_text = AddText::No;
}
"pre" => {
dehtml.strbuilder += "\n\n";
dehtml.add_text = AddText::YesPreserveLineEnds;
}
"a" => {
if let Some(href) = event
.html_attributes()
.filter_map(|attr| attr.ok())
.find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
{
let href = href
.unescape_and_decode_value(reader)
.unwrap_or_default()
.to_lowercase();
if !href.is_empty() {
dehtml.last_href = Some(href);
dehtml.strbuilder += "[";
}
}
}
"b" | "strong" => {
dehtml.strbuilder += "*";
}
"i" | "em" => {
dehtml.strbuilder += "_";
}
_ => {}
}
}
pub fn dehtml_manually(buf: &str) -> String {
// Just strip out everything between "<" and ">"
let mut strbuilder = String::new();
let mut show_next_chars = true;
for c in buf.chars() {
match c {
'<' => show_next_chars = false,
'>' => show_next_chars = true,
_ => {
if show_next_chars {
strbuilder.push(c)
}
}
}
}
strbuilder
}
#[cfg(test)]
mod tests {
use super::*;
use crate::simplify::simplify;
#[test]
fn test_dehtml() {
let cases = vec",
),
(" bar ", "* bar *"),
(" bar foo", "* bar _ foo"),
("& bar", "& bar"),
// Despite missing ', this should be shown:
("",
"[](https://get.delta.chat/)",
),
("\nfat text", "*fat text*"),
// Invalid html (at least DC should show the text if the html is invalid):
("\nsome text", "some text"),
];
for (input, output) in cases {
assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
}
let none_cases = vec![" ", ""];
for input in none_cases {
assert_eq!(dehtml(input), None);
}
}
#[test]
fn test_dehtml_parse_br() {
let html = "\r\r\nline1
\r\n\r\n\r\rline2
line3\n\r";
let plain = dehtml(html).unwrap();
assert_eq!(plain, "line1\n\r\r\rline2\nline3");
}
#[test]
fn test_dehtml_parse_href() {
let html = "text");
}
#[test]
fn test_dehtml_html_encoded() {
let html =
"<>"'& äÄöÖüÜß fooÆçÇ ♦&noent;";
let plain = dehtml(html).unwrap();
assert_eq!(
plain,
"<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
);
}
#[test]
fn test_unclosed_tags() {
let input = r##"
Hi
lots of text
"##;
let txt = dehtml(input).unwrap();
assert_eq!(txt.trim(), "lots of text");
}
}
![](/foo.png>Hi</a> ", "Hi "),
(
"<a href=){kind=link}