From 564d681bca1375330749be2278e1290a32b5c866 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 26 Jan 2021 01:20:35 +0300 Subject: [PATCH] Try to extract group ID from In-Reply-To and References as a last resort This should help if parent message can't be found because messages were reordered or deleted. This does not prevent group IDs from being removed from Message-IDs in the future, in which case it will become dead code. --- src/dc_receive_imf.rs | 75 +++++++++++++++++++++++++++++++++++++++++-- src/dc_tools.rs | 63 ++++++++++++++++++++++++++++++++++++ src/imap/mod.rs | 18 ++++++++++- 3 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 03e036a74..1380b9fa0 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -12,7 +12,9 @@ use crate::constants::{ }; use crate::contact::{addr_cmp, normalize_name, Contact, Origin, VerifiedStatus}; use crate::context::Context; -use crate::dc_tools::{dc_create_smeared_timestamp, dc_smeared_time, time}; +use crate::dc_tools::{ + dc_create_smeared_timestamp, dc_extract_grpid_from_rfc724_mid, dc_smeared_time, time, +}; use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer}; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; @@ -1129,8 +1131,8 @@ async fn create_or_lookup_group( set_better_msg(mime_parser, &better_msg); } - let grpid = if let Some(grpid) = mime_parser.get(HeaderDef::ChatGroupId) { - grpid.clone() + let grpid = if let Some(grpid) = try_getting_grpid(mime_parser) { + grpid } else { let mut member_ids: Vec = to_ids.iter().copied().collect(); if !member_ids.contains(&from_id) { @@ -1443,6 +1445,37 @@ async fn create_or_lookup_group( Ok((chat_id, chat_id_blocked)) } +fn try_getting_grpid(mime_parser: &MimeMessage) -> Option { + if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupId) { + return Some(optional_field.clone()); + } + + // Useful for undecipherable messages sent to known group. + if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::MessageId) { + return Some(extracted_grpid.to_string()); + } + + if !mime_parser.has_chat_version() { + if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) { + return Some(extracted_grpid.to_string()); + } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) { + return Some(extracted_grpid.to_string()); + } + } + + None +} + +/// try extract a grpid from a message-id list header value +fn extract_grpid(mime_parser: &MimeMessage, headerdef: HeaderDef) -> Option<&str> { + let header = mime_parser.get(headerdef)?; + let parts = header + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()); + parts.filter_map(dc_extract_grpid_from_rfc724_mid).next() +} + /// Creates ad-hoc group and returns chat ID on success. async fn create_adhoc_group( context: &Context, @@ -1861,6 +1894,42 @@ mod tests { assert_eq!(res, "b94d27b9934d3e08"); } + #[async_std::test] + async fn test_grpid_simple() { + let context = TestContext::new().await; + let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: hello\n\ + Subject: outer-subject\n\ + In-Reply-To: \n\ + References: \n\ + \n\ + hello\x00"; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) + .await + .unwrap(); + assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), None); + let grpid = Some("HcxyMARjyJy"); + assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid); + } + + #[async_std::test] + async fn test_grpid_from_multiple() { + let context = TestContext::new().await; + let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: hello\n\ + Subject: outer-subject\n\ + In-Reply-To: \n\ + References: , \n\ + \n\ + hello\x00"; + let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) + .await + .unwrap(); + let grpid = Some("HcxyMARjyJy"); + assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), grpid); + assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid); + } + #[test] fn test_dc_create_incoming_rfc724_mid() { let mut members = ContactIds::new(); diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 203307537..19503b0df 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -279,6 +279,31 @@ pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str } } +/// Extract the group id (grpid) from a message id (mid) +/// +/// # Arguments +/// +/// * `mid` - A string that holds the message id. Leading/Trailing <> +/// characters are automatically stripped. +pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> { + let mid = mid.trim_start_matches('<').trim_end_matches('>'); + + if mid.len() < 9 || !mid.starts_with("Gr.") { + return None; + } + + if let Some(mid_without_offset) = mid.get(3..) { + if let Some(grpid_len) = mid_without_offset.find('.') { + /* strict length comparison, the 'Gr.' magic is weak enough */ + if grpid_len == 11 || grpid_len == 16 { + return Some(mid_without_offset.get(0..grpid_len).unwrap()); + } + } + } + + None +} + // the returned suffix is lower-case pub fn dc_get_filesuffix_lc(path_filename: impl AsRef) -> Option { Path::new(path_filename.as_ref()) @@ -760,18 +785,56 @@ mod tests { ); } + #[test] + fn test_dc_extract_grpid_from_rfc724_mid() { + // Should return None if we pass invalid mid + let mid = "foobar"; + let grpid = dc_extract_grpid_from_rfc724_mid(mid); + assert_eq!(grpid, None); + + // Should return None if grpid has a length which is not 11 or 16 + let mid = "Gr.12345678.morerandom@domain.de"; + let grpid = dc_extract_grpid_from_rfc724_mid(mid); + assert_eq!(grpid, None); + + // Should return extracted grpid for grpid with length of 11 + let mid = "Gr.12345678901.morerandom@domain.de"; + let grpid = dc_extract_grpid_from_rfc724_mid(mid); + assert_eq!(grpid, Some("12345678901")); + + // Should return extracted grpid for grpid with length of 11 + let mid = "Gr.1234567890123456.morerandom@domain.de"; + let grpid = dc_extract_grpid_from_rfc724_mid(mid); + assert_eq!(grpid, Some("1234567890123456")); + + // Should return extracted grpid for grpid with length of 11 + let mid = ""; + let grpid = dc_extract_grpid_from_rfc724_mid(mid); + assert_eq!(grpid, Some("12345678901")); + + // Should return extracted grpid for grpid with length of 11 + let mid = ""; + let grpid = dc_extract_grpid_from_rfc724_mid(mid); + assert_eq!(grpid, Some("1234567890123456")); + } + #[test] fn test_dc_create_outgoing_rfc724_mid() { // create a normal message-id let mid = dc_create_outgoing_rfc724_mid(None, "foo@bar.de"); assert!(mid.starts_with("Mr.")); assert!(mid.ends_with("bar.de")); + assert!(dc_extract_grpid_from_rfc724_mid(mid.as_str()).is_none()); // create a message-id containing a group-id let grpid = dc_create_id(); let mid = dc_create_outgoing_rfc724_mid(Some(&grpid), "foo@bar.de"); assert!(mid.starts_with("Gr.")); assert!(mid.ends_with("bar.de")); + assert_eq!( + dc_extract_grpid_from_rfc724_mid(mid.as_str()), + Some(grpid.as_str()) + ); } #[test] diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 4fbc30662..c59534d6d 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -29,7 +29,9 @@ use crate::mimeparser; use crate::oauth2::dc_get_oauth2_access_token; use crate::param::Params; use crate::provider::Socket; -use crate::{chat, scheduler::InterruptInfo, stock::StockMessage}; +use crate::{ + chat, dc_tools::dc_extract_grpid_from_rfc724_mid, scheduler::InterruptInfo, stock::StockMessage, +}; use crate::{config::Config, dc_receive_imf::dc_receive_imf_inner}; mod client; @@ -38,6 +40,7 @@ pub mod scan_folders; pub mod select_folder; mod session; +use chat::get_chat_id_by_grpid; use client::Client; use mailparse::SingleInfo; use message::Message; @@ -1611,6 +1614,19 @@ pub(crate) async fn prefetch_should_download( } } + // Same as previous check, but using group IDs embedded into + // Message-IDs as a last resort, in case parent message was + // deleted from the database or has not arrived yet. + if let Some(rfc724_mid) = headers.get_header_value(HeaderDef::MessageId) { + if let Some(group_id) = dc_extract_grpid_from_rfc724_mid(&rfc724_mid) { + if let Ok((chat_id, _, _)) = get_chat_id_by_grpid(context, group_id).await { + if !chat_id.is_unset() { + return Ok(true); + } + } + } + } + let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) { let from = from.to_ascii_lowercase(); from.contains("mailer-daemon") || from.contains("mail-daemon")