Try to extract group ID from In-Reply-To and References as a last resort

This should help if parent message can't be found because messages were
reordered or deleted.

This does not prevent group IDs from being removed from Message-IDs in
the future, in which case it will become dead code.

src/dc_receive_imf.rs

@@ -12,7 +12,9 @@ use crate::constants::{
 };
 use crate::contact::{addr_cmp, normalize_name, Contact, Origin, VerifiedStatus};
 use crate::context::Context;
-use crate::dc_tools::{dc_create_smeared_timestamp, dc_smeared_time, time};
+use crate::dc_tools::{
+    dc_create_smeared_timestamp, dc_extract_grpid_from_rfc724_mid, dc_smeared_time, time,
+};
 use crate::ephemeral::{stock_ephemeral_timer_changed, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
@@ -1129,8 +1131,8 @@ async fn create_or_lookup_group(
         set_better_msg(mime_parser, &better_msg);
     }
 
-    let grpid = if let Some(grpid) = mime_parser.get(HeaderDef::ChatGroupId) {
-        grpid.clone()
+    let grpid = if let Some(grpid) = try_getting_grpid(mime_parser) {
+        grpid
     } else {
         let mut member_ids: Vec<u32> = to_ids.iter().copied().collect();
         if !member_ids.contains(&from_id) {
@@ -1443,6 +1445,37 @@ async fn create_or_lookup_group(
     Ok((chat_id, chat_id_blocked))
 }
 
+fn try_getting_grpid(mime_parser: &MimeMessage) -> Option<String> {
+    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupId) {
+        return Some(optional_field.clone());
+    }
+
+    // Useful for undecipherable messages sent to known group.
+    if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::MessageId) {
+        return Some(extracted_grpid.to_string());
+    }
+
+    if !mime_parser.has_chat_version() {
+        if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
+            return Some(extracted_grpid.to_string());
+        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
+            return Some(extracted_grpid.to_string());
+        }
+    }
+
+    None
+}
+
+/// try extract a grpid from a message-id list header value
+fn extract_grpid(mime_parser: &MimeMessage, headerdef: HeaderDef) -> Option<&str> {
+    let header = mime_parser.get(headerdef)?;
+    let parts = header
+        .split(',')
+        .map(str::trim)
+        .filter(|part| !part.is_empty());
+    parts.filter_map(dc_extract_grpid_from_rfc724_mid).next()
+}
+
 /// Creates ad-hoc group and returns chat ID on success.
 async fn create_adhoc_group(
     context: &Context,
@@ -1861,6 +1894,42 @@ mod tests {
         assert_eq!(res, "b94d27b9934d3e08");
     }
 
+    #[async_std::test]
+    async fn test_grpid_simple() {
+        let context = TestContext::new().await;
+        let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                    From: hello\n\
+                    Subject: outer-subject\n\
+                    In-Reply-To: <lqkjwelq123@123123>\n\
+                    References: <Gr.HcxyMARjyJy.9-uvzWPTLtV@nauta.cu>\n\
+                    \n\
+                    hello\x00";
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), None);
+        let grpid = Some("HcxyMARjyJy");
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid);
+    }
+
+    #[async_std::test]
+    async fn test_grpid_from_multiple() {
+        let context = TestContext::new().await;
+        let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
+                    From: hello\n\
+                    Subject: outer-subject\n\
+                    In-Reply-To: <Gr.HcxyMARjyJy.9-qweqwe@asd.net>\n\
+                    References: <qweqweqwe>, <Gr.HcxyMARjyJy.9-uvzWPTLtV@nau.ca>\n\
+                    \n\
+                    hello\x00";
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
+            .await
+            .unwrap();
+        let grpid = Some("HcxyMARjyJy");
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), grpid);
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid);
+    }
+
     #[test]
     fn test_dc_create_incoming_rfc724_mid() {
         let mut members = ContactIds::new();

src/dc_tools.rs

@@ -279,6 +279,31 @@ pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str
     }
 }
 
+/// Extract the group id (grpid) from a message id (mid)
+///
+/// # Arguments
+///
+/// * `mid` - A string that holds the message id.  Leading/Trailing <>
+/// characters are automatically stripped.
+pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> {
+    let mid = mid.trim_start_matches('<').trim_end_matches('>');
+
+    if mid.len() < 9 || !mid.starts_with("Gr.") {
+        return None;
+    }
+
+    if let Some(mid_without_offset) = mid.get(3..) {
+        if let Some(grpid_len) = mid_without_offset.find('.') {
+            /* strict length comparison, the 'Gr.' magic is weak enough */
+            if grpid_len == 11 || grpid_len == 16 {
+                return Some(mid_without_offset.get(0..grpid_len).unwrap());
+            }
+        }
+    }
+
+    None
+}
+
 // the returned suffix is lower-case
 pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
     Path::new(path_filename.as_ref())
@@ -760,18 +785,56 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_dc_extract_grpid_from_rfc724_mid() {
+        // Should return None if we pass invalid mid
+        let mid = "foobar";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, None);
+
+        // Should return None if grpid has a length which is not 11 or 16
+        let mid = "Gr.12345678.morerandom@domain.de";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, None);
+
+        // Should return extracted grpid for grpid with length of 11
+        let mid = "Gr.12345678901.morerandom@domain.de";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, Some("12345678901"));
+
+        // Should return extracted grpid for grpid with length of 11
+        let mid = "Gr.1234567890123456.morerandom@domain.de";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, Some("1234567890123456"));
+
+        // Should return extracted grpid for grpid with length of 11
+        let mid = "<Gr.12345678901.morerandom@domain.de>";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, Some("12345678901"));
+
+        // Should return extracted grpid for grpid with length of 11
+        let mid = "<Gr.1234567890123456.morerandom@domain.de>";
+        let grpid = dc_extract_grpid_from_rfc724_mid(mid);
+        assert_eq!(grpid, Some("1234567890123456"));
+    }
+
     #[test]
     fn test_dc_create_outgoing_rfc724_mid() {
         // create a normal message-id
         let mid = dc_create_outgoing_rfc724_mid(None, "foo@bar.de");
         assert!(mid.starts_with("Mr."));
         assert!(mid.ends_with("bar.de"));
+        assert!(dc_extract_grpid_from_rfc724_mid(mid.as_str()).is_none());
 
         // create a message-id containing a group-id
         let grpid = dc_create_id();
         let mid = dc_create_outgoing_rfc724_mid(Some(&grpid), "foo@bar.de");
         assert!(mid.starts_with("Gr."));
         assert!(mid.ends_with("bar.de"));
+        assert_eq!(
+            dc_extract_grpid_from_rfc724_mid(mid.as_str()),
+            Some(grpid.as_str())
+        );
     }
 
     #[test]

src/imap/mod.rs

@@ -29,7 +29,9 @@ use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::provider::Socket;
-use crate::{chat, scheduler::InterruptInfo, stock::StockMessage};
+use crate::{
+    chat, dc_tools::dc_extract_grpid_from_rfc724_mid, scheduler::InterruptInfo, stock::StockMessage,
+};
 use crate::{config::Config, dc_receive_imf::dc_receive_imf_inner};
 
 mod client;
@@ -38,6 +40,7 @@ pub mod scan_folders;
 pub mod select_folder;
 mod session;
 
+use chat::get_chat_id_by_grpid;
 use client::Client;
 use mailparse::SingleInfo;
 use message::Message;
@@ -1611,6 +1614,19 @@ pub(crate) async fn prefetch_should_download(
         }
     }
 
+    // Same as previous check, but using group IDs embedded into
+    // Message-IDs as a last resort, in case parent message was
+    // deleted from the database or has not arrived yet.
+    if let Some(rfc724_mid) = headers.get_header_value(HeaderDef::MessageId) {
+        if let Some(group_id) = dc_extract_grpid_from_rfc724_mid(&rfc724_mid) {
+            if let Ok((chat_id, _, _)) = get_chat_id_by_grpid(context, group_id).await {
+                if !chat_id.is_unset() {
+                    return Ok(true);
+                }
+            }
+        }
+    }
+
     let maybe_ndn = if let Some(from) = headers.get_header_value(HeaderDef::From_) {
         let from = from.to_ascii_lowercase();
         from.contains("mailer-daemon") || from.contains("mail-daemon")