From 4fa0edcfbfde9eac8342b18f72b4b9738d75e210 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 8 Jul 2021 21:24:13 +0200 Subject: [PATCH 1/4] add hop info add hop info creation Save it to the db format and fix clippy replace `and_then` with simple check cargo fmt chage table to get info from --- src/dc_receive_imf.rs | 7 ++++--- src/dc_tools.rs | 46 +++++++++++++++++++++++++++++++++++++++++++ src/message.rs | 8 ++++++++ src/mimeparser.rs | 6 +++++- src/sql/migrations.rs | 4 ++++ 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index fbcf9524c..e49bb32a9 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -965,7 +965,7 @@ INSERT INTO msgs txt, subject, txt_raw, param, bytes, hidden, mime_headers, mime_in_reply_to, mime_references, mime_modified, error, ephemeral_timer, - ephemeral_timestamp + ephemeral_timestamp, hop_info ) VALUES ( ?, ?, ?, ?, @@ -974,7 +974,7 @@ INSERT INTO msgs ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ? + ?, ? ); "#, )?; @@ -1050,7 +1050,8 @@ INSERT INTO msgs mime_modified, part.error.take().unwrap_or_default(), ephemeral_timer, - ephemeral_timestamp + ephemeral_timestamp, + mime_parser.hop_info ])?; let row_id = conn.last_insert_rowid(); diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 6f3356326..848f2e60b 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -5,6 +5,7 @@ use core::cmp::{max, min}; use std::borrow::Cow; use std::fmt; use std::io::Cursor; +use std::str::from_utf8; use std::str::FromStr; use std::time::{Duration, SystemTime}; @@ -14,6 +15,10 @@ use async_std::{fs, io}; use anyhow::{bail, Error}; use chrono::{Local, TimeZone}; +use itertools::Itertools; +use mailparse::dateparse; +use mailparse::headers::Headers; +use mailparse::MailHeaderMap; use rand::{thread_rng, Rng}; use crate::chat::{add_device_msg, add_device_msg_with_importance}; @@ -670,6 +675,47 @@ pub fn remove_subject_prefix(last_subject: &str) -> String { .to_string() } +// Types and methods to create hop-info for message-info + +fn extract_address_from_receive_header<'a>(header: &'a str, start: &str) -> Option<&'a str> { + let header_len = header.len(); + header.find(start).and_then(|mut begin| { + begin += start.len(); + let end = header.get(begin..)?.find(' ').unwrap_or(header_len); + header.get(begin..begin + end) + }) +} + +pub(crate) fn parse_receive_header(header: &str) -> String { + let mut hop_info = String::from("Hop:\n"); + + if let Ok(date) = dateparse(header) { + let date_obj = Local.timestamp(date, 0); + hop_info.push_str(&format!("Date: {}\n", date_obj.to_rfc2822())); + }; + + if let Some(from) = extract_address_from_receive_header(header, "from ") { + hop_info.push_str(&format!("From: {}\n", from)); + } + + if let Some(by) = extract_address_from_receive_header(header, "by ") { + hop_info.push_str(&format!("By: {}\n", by)); + } + hop_info +} + +/// parses "receive"-headers +pub(crate) fn parse_receive_headers(headers: &Headers) -> String { + let headers = headers + .get_all_headers("Received") + .iter() + .filter_map(|header_map_item| from_utf8(header_map_item.get_value_raw()).ok()) + .map(|header_value| parse_receive_header(header_value)) + .collect::>(); + + headers.iter().map(|a| a.to_string()).join("\n\n") +} + #[cfg(test)] mod tests { #![allow(clippy::indexing_slicing)] diff --git a/src/message.rs b/src/message.rs index 3c7ca9c29..8f92a4cf1 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1263,6 +1263,14 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { ret += &format!("\nLast seen as: {}/{}", server_folder, msg.server_uid); } } + let hop_info: Option = context + .sql + .query_get_value("SELECT hop_info FROM msgs WHERE id=?;", paramsv![msg_id]) + .await?; + + if hop_info.is_some() { + ret.push_str(&hop_info.unwrap_or_else(|| "No Hop info".to_owned())); + } Ok(ret) } diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 595580ecb..09a7c6df7 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -13,7 +13,7 @@ use crate::blob::BlobObject; use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSE}; use crate::contact::addr_normalize; use crate::context::Context; -use crate::dc_tools::{dc_get_filemeta, dc_truncate}; +use crate::dc_tools::{dc_get_filemeta, dc_truncate, parse_receive_headers}; use crate::dehtml::dehtml; use crate::e2ee; use crate::events::EventType; @@ -76,6 +76,8 @@ pub struct MimeMessage { /// This is non-empty only if the message was actually encrypted. It is used /// for e.g. late-parsing HTML. pub decoded_data: Vec, + + pub(crate) hop_info: String, } #[derive(Debug, PartialEq)] @@ -271,6 +273,8 @@ impl MimeMessage { footer: None, is_mime_modified: false, decoded_data: Vec::new(), + + hop_info: parse_receive_headers(&mail.get_headers()), }; parser.parse_mime_recursive(context, &mail, false).await?; parser.maybe_remove_bad_parts(); diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 734b8fbc3..f2ed69ace 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -473,6 +473,10 @@ paramsv![] // this way, the app looks familiar after the contact request upgrade. info!(context, "[migration] v78"); sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78) + } + if dbversion < 79 { + info!(context, "[migration] v79"); + sql.execute_migration("ALTER TABLE msgs ADD COLUMN hop_info TEXT DEFAULT '';", 79) .await?; } From 48a36439279a5be48bfcada244b30b91954e0937 Mon Sep 17 00:00:00 2001 From: Septias <=> Date: Thu, 29 Jul 2021 13:05:48 +0200 Subject: [PATCH 2/4] small fix --- src/sql/migrations.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index f2ed69ace..590a6e805 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -472,7 +472,7 @@ paramsv![] // move requests to "Archived Chats", // this way, the app looks familiar after the contact request upgrade. info!(context, "[migration] v78"); - sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78) + sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78).await?; } if dbversion < 79 { info!(context, "[migration] v79"); From d8b8a3839ccbe9c0b188f259121481a858dc8df4 Mon Sep 17 00:00:00 2001 From: Septias <=> Date: Thu, 29 Jul 2021 17:26:32 +0200 Subject: [PATCH 3/4] add tests --- src/dc_tools.rs | 72 +++++++++++++++++++++++++++++++++++++++---- src/message.rs | 2 +- src/sql/migrations.rs | 3 +- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 848f2e60b..85145d329 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -681,7 +681,10 @@ fn extract_address_from_receive_header<'a>(header: &'a str, start: &str) -> Opti let header_len = header.len(); header.find(start).and_then(|mut begin| { begin += start.len(); - let end = header.get(begin..)?.find(' ').unwrap_or(header_len); + let end = header + .get(begin..)? + .find(|c| c == ' ' || c == '\n') + .unwrap_or(header_len); header.get(begin..begin + end) }) } @@ -695,11 +698,11 @@ pub(crate) fn parse_receive_header(header: &str) -> String { }; if let Some(from) = extract_address_from_receive_header(header, "from ") { - hop_info.push_str(&format!("From: {}\n", from)); + hop_info.push_str(&format!("From: {}\n", from.trim())); } if let Some(by) = extract_address_from_receive_header(header, "by ") { - hop_info.push_str(&format!("By: {}\n", by)); + hop_info.push_str(&format!("By: {}\n", by.trim())); } hop_info } @@ -709,11 +712,13 @@ pub(crate) fn parse_receive_headers(headers: &Headers) -> String { let headers = headers .get_all_headers("Received") .iter() + .rev() .filter_map(|header_map_item| from_utf8(header_map_item.get_value_raw()).ok()) - .map(|header_value| parse_receive_header(header_value)) + .enumerate() + .map(|(i, header_value)| (i + 1).to_string() + ". " + &parse_receive_header(header_value)) .collect::>(); - headers.iter().map(|a| a.to_string()).join("\n\n") + headers.iter().map(|a| a.to_string()).join("\n") } #[cfg(test)] @@ -722,7 +727,62 @@ mod tests { use super::*; - use crate::test_utils::TestContext; + use crate::{ + chat::ChatItem, config::Config, dc_receive_imf::dc_receive_imf, message::get_msg_info, + test_utils::TestContext, + }; + + #[test] + fn test_parse_receive_headers() { + let raw = include_bytes!("../test-data/message/mail_with_cc.txt"); + let mail = mailparse::parse_mail(&raw[..]).unwrap(); + let hop_info = parse_receive_headers(&mail.get_headers()); + let expected = concat!( + "1. Hop:\n", + "Date: Sat, 14 Sep 2019 19:00:22 +0200\n", + "From: localhost\n", + "By: hq5.merlinux.eu\n", + "\n", + "2. Hop:\n", + "Date: Sat, 14 Sep 2019 19:00:25 +0200\n", + "From: hq5.merlinux.eu\n", + "By: hq5.merlinux.eu\n", + ); + assert_eq!(&hop_info, expected) + } + + #[async_std::test] + async fn test_parse_receive_headers_integration() -> anyhow::Result<()> { + let t = TestContext::new_alice().await; + t.set_config(Config::ShowEmails, Some("2")).await?; + let raw = include_bytes!("../test-data/message/mail_with_cc.txt"); + dc_receive_imf(&t, raw, "INBOX", 1, false).await.unwrap(); + let g = t.get_last_msg().await; + + let expected = r"State: Fresh + +hi + +Message-ID: 2dfdbde7@example.org +Last seen as: INBOX/1 +1. Hop: +Date: Sat, 14 Sep 2019 19:00:22 +0200 +From: localhost +By: hq5.merlinux.eu + +2. Hop: +Date: Sat, 14 Sep 2019 19:00:25 +0200 +From: hq5.merlinux.eu +By: hq5.merlinux.eu +"; + let result = get_msg_info(&t, g.id).await.unwrap(); + // little hack to ignore the first row of a parsed email because it contains a + // send time that depends and the test runtime which makes it impossible to + // compare with a static string + let capped_result = &result[result.find("State").unwrap()..]; + assert_eq!(expected, capped_result); + Ok(()) + } #[test] fn test_rust_ftoa() { diff --git a/src/message.rs b/src/message.rs index 8f92a4cf1..19ba87afc 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1260,7 +1260,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { } if let Some(ref server_folder) = msg.server_folder { if !server_folder.is_empty() { - ret += &format!("\nLast seen as: {}/{}", server_folder, msg.server_uid); + ret += &format!("\nLast seen as: {}/{}\n", server_folder, msg.server_uid); } } let hop_info: Option = context diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 590a6e805..51e983e7b 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -472,7 +472,8 @@ paramsv![] // move requests to "Archived Chats", // this way, the app looks familiar after the contact request upgrade. info!(context, "[migration] v78"); - sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78).await?; + sql.execute_migration("UPDATE chats SET archived=1 WHERE blocked=2;", 78) + .await?; } if dbversion < 79 { info!(context, "[migration] v79"); From 3f8878fe7a756257dc6170712bc034d5b411ec3d Mon Sep 17 00:00:00 2001 From: Septias <=> Date: Thu, 29 Jul 2021 17:34:31 +0200 Subject: [PATCH 4/4] cargo fix --- src/dc_tools.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 85145d329..df0e7c197 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -728,7 +728,7 @@ mod tests { use super::*; use crate::{ - chat::ChatItem, config::Config, dc_receive_imf::dc_receive_imf, message::get_msg_info, + config::Config, dc_receive_imf::dc_receive_imf, message::get_msg_info, test_utils::TestContext, };