use itertools::join; use num_traits::FromPrimitive; use sha2::{Digest, Sha256}; use crate::chat::{self, Chat, ChatId}; use crate::config::Config; use crate::constants::*; use crate::contact::*; use crate::context::Context; use crate::dc_tools::*; use crate::error::Result; use crate::events::Event; use crate::headerdef::HeaderDef; use crate::job::{self, Action}; use crate::message::{self, MessageState, MessengerMessage, MsgId}; use crate::mimeparser::*; use crate::param::*; use crate::peerstate::*; use crate::securejoin::{self, handle_securejoin_handshake}; use crate::stock::StockMessage; use crate::{contact, location}; // IndexSet is like HashSet but maintains order of insertion type ContactIds = indexmap::IndexSet; #[derive(Debug, PartialEq, Eq)] enum CreateEvent { MsgsChanged, IncomingMsg, } /// Receive a message and add it to the database. pub async fn dc_receive_imf( context: &Context, imf_raw: &[u8], server_folder: impl AsRef, server_uid: u32, seen: bool, ) -> Result<()> { info!( context, "Receiving message {}/{}...", if !server_folder.as_ref().is_empty() { server_folder.as_ref() } else { "?" }, server_uid, ); if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" { info!(context, "dc_receive_imf: incoming message mime-body:"); println!("{}", String::from_utf8_lossy(imf_raw)); } let mut mime_parser = MimeMessage::from_bytes(context, imf_raw).await?; // we can not add even an empty record if we have no info whatsoever ensure!(mime_parser.has_headers(), "No Headers Found"); // the function returns the number of created messages in the database let mut chat_id = ChatId::new(0); let mut hidden = false; let mut needs_delete_job = false; let mut insert_msg_id = MsgId::new_unset(); let mut sent_timestamp = 0; let mut created_db_entries = Vec::new(); let mut create_event_to_send = Some(CreateEvent::MsgsChanged); // helper method to handle early exit and memory cleanup let cleanup = |context: &Context, create_event_to_send: &Option, created_db_entries: Vec<(ChatId, MsgId)>| { if let Some(create_event_to_send) = create_event_to_send { for (chat_id, msg_id) in created_db_entries { let event = match create_event_to_send { CreateEvent::MsgsChanged => Event::MsgsChanged { msg_id, chat_id }, CreateEvent::IncomingMsg => Event::IncomingMsg { msg_id, chat_id }, }; context.call_cb(event); } } }; if let Some(value) = mime_parser.get(HeaderDef::Date) { // is not yet checked against bad times! we do this later if we have the database information. sent_timestamp = mailparse::dateparse(value).unwrap_or_default(); } // get From: (it can be an address list!) and check if it is known (for known From:'s we add // the other To:/Cc: in the 3rd pass) // or if From: is equal to SELF (in this case, it is any outgoing messages, // we do not check Return-Path any more as this is unreliable, see // https://github.com/deltachat/deltachat-core/issues/150) let (from_id, from_id_blocked, incoming_origin) = if let Some(field_from) = mime_parser.get(HeaderDef::From_) { from_field_to_contact_id(context, field_from).await? } else { (0, false, Origin::Unknown) }; let incoming = from_id != DC_CONTACT_ID_SELF; let mut to_ids = ContactIds::new(); for header_def in &[HeaderDef::To, HeaderDef::Cc] { if let Some(field) = mime_parser.get(header_def.clone()) { to_ids.extend( &dc_add_or_lookup_contacts_by_address_list( context, &field, if !incoming { Origin::OutgoingTo } else if incoming_origin.is_known() { Origin::IncomingTo } else { Origin::IncomingUnknownTo }, ) .await?, ); } } // Add parts let rfc724_mid = match mime_parser.get_rfc724_mid() { Some(x) => x, None => { // missing Message-IDs may come if the mail was set from this account with another // client that relies in the SMTP server to generate one. // true eg. for the Webmailer used in all-inkl-KAS match dc_create_incoming_rfc724_mid(sent_timestamp, from_id, &to_ids) { Some(x) => x, None => { bail!("No Message-Id found and could not create incoming rfc724_mid"); } } } }; if mime_parser.parts.last().is_some() { if let Err(err) = add_parts( context, &mut mime_parser, imf_raw, incoming, incoming_origin, server_folder.as_ref(), server_uid, &to_ids, &rfc724_mid, &mut sent_timestamp, from_id, from_id_blocked, &mut hidden, &mut chat_id, seen, &mut needs_delete_job, &mut insert_msg_id, &mut created_db_entries, &mut create_event_to_send, ) .await { cleanup(context, &create_event_to_send, created_db_entries); bail!("add_parts error: {:?}", err); } } else { // there are parts in this message, do some basic calculations so that the variables // are correct in the further processing if sent_timestamp > time() { sent_timestamp = time() } } if mime_parser.location_kml.is_some() || mime_parser.message_kml.is_some() { save_locations( context, &mime_parser, chat_id, from_id, insert_msg_id, hidden, ) .await; } if let Some(avatar_action) = &mime_parser.user_avatar { match contact::set_profile_image(&context, from_id, avatar_action).await { Ok(()) => { context.call_cb(Event::ChatModified(chat_id)); } Err(err) => { warn!(context, "reveive_imf cannot update profile image: {}", err); } }; } // if we delete we don't need to try moving messages if needs_delete_job && !created_db_entries.is_empty() { job::add( context, Action::DeleteMsgOnImap, created_db_entries[0].1.to_u32() as i32, Params::new(), 0, ) .await; } else { context .do_heuristics_moves(server_folder.as_ref(), insert_msg_id) .await; } info!( context, "received message {} has Message-Id: {}", server_uid, rfc724_mid ); cleanup(context, &create_event_to_send, created_db_entries); mime_parser .handle_reports(context, from_id, sent_timestamp, &server_folder, server_uid) .await; Ok(()) } /// Converts "From" field to contact id. /// /// Also returns whether it is blocked or not and its origin. pub async fn from_field_to_contact_id( context: &Context, field_from: &str, ) -> Result<(u32, bool, Origin)> { let from_ids = dc_add_or_lookup_contacts_by_address_list( context, &field_from, Origin::IncomingUnknownFrom, ) .await?; if from_ids.contains(&DC_CONTACT_ID_SELF) { Ok((DC_CONTACT_ID_SELF, false, Origin::OutgoingBcc)) } else if !from_ids.is_empty() { if from_ids.len() > 1 { warn!( context, "mail has more than one From address, only using first: {:?}", field_from ); } let from_id = from_ids.get_index(0).cloned().unwrap_or_default(); let mut from_id_blocked = false; let mut incoming_origin = Origin::Unknown; if let Ok(contact) = Contact::load_from_db(context, from_id).await { from_id_blocked = contact.blocked; incoming_origin = contact.origin; } Ok((from_id, from_id_blocked, incoming_origin)) } else { warn!(context, "mail has an empty From header: {:?}", field_from); // if there is no from given, from_id stays 0 which is just fine. These messages // are very rare, however, we have to add them to the database (they go to the // "deaddrop" chat) to avoid a re-download from the server. See also [**] Ok((0, false, Origin::Unknown)) } } #[allow(clippy::too_many_arguments, clippy::cognitive_complexity)] async fn add_parts( context: &Context, mut mime_parser: &mut MimeMessage, imf_raw: &[u8], incoming: bool, incoming_origin: Origin, server_folder: impl AsRef, server_uid: u32, to_ids: &ContactIds, rfc724_mid: &str, sent_timestamp: &mut i64, from_id: u32, from_id_blocked: bool, hidden: &mut bool, chat_id: &mut ChatId, seen: bool, needs_delete_job: &mut bool, insert_msg_id: &mut MsgId, created_db_entries: &mut Vec<(ChatId, MsgId)>, create_event_to_send: &mut Option, ) -> Result<()> { let mut state: MessageState; let mut chat_id_blocked = Blocked::Not; let mut sort_timestamp = 0; let mut rcvd_timestamp = 0; let mut mime_in_reply_to = String::new(); let mut mime_references = String::new(); let mut incoming_origin = incoming_origin; // check, if the mail is already in our database - if so, just update the folder/uid // (if the mail was moved around) and finish. (we may get a mail twice eg. if it is // moved between folders. make sure, this check is done eg. before securejoin-processing) */ if let Ok((old_server_folder, old_server_uid, _)) = message::rfc724_mid_exists(context, &rfc724_mid).await { if old_server_folder != server_folder.as_ref() || old_server_uid != server_uid { message::update_server_uid(context, &rfc724_mid, server_folder.as_ref(), server_uid) .await; } bail!("Message already in DB"); } let mut msgrmsg = if mime_parser.has_chat_version() { MessengerMessage::Yes } else if is_reply_to_messenger_message(context, mime_parser).await { MessengerMessage::Reply } else { MessengerMessage::No }; // incoming non-chat messages may be discarded let mut allow_creation = true; let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await).unwrap_or_default(); if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage && msgrmsg == MessengerMessage::No { // this message is a classic email not a chat-message nor a reply to one match show_emails { ShowEmails::Off => { *chat_id = ChatId::new(DC_CHAT_ID_TRASH); allow_creation = false; } ShowEmails::AcceptedContacts => allow_creation = false, ShowEmails::All => {} } } // check if the message introduces a new chat: // - outgoing messages introduce a chat with the first to: address if they are sent by a messenger // - incoming messages introduce a chat only for known contacts if they are sent by a messenger // (of course, the user can add other chats manually later) let to_id: u32; if incoming { state = if seen { MessageState::InSeen } else { MessageState::InFresh }; to_id = DC_CONTACT_ID_SELF; // handshake messages must be processed _before_ chats are created // (eg. contacs may be marked as verified) if mime_parser.get(HeaderDef::SecureJoin).is_some() { // avoid discarding by show_emails setting msgrmsg = MessengerMessage::Yes; *chat_id = ChatId::new(0); allow_creation = true; match handle_securejoin_handshake(context, mime_parser, from_id).await { Ok(securejoin::HandshakeMessage::Done) => { *hidden = true; *needs_delete_job = true; state = MessageState::InSeen; } Ok(securejoin::HandshakeMessage::Ignore) => { *hidden = true; state = MessageState::InSeen; } Ok(securejoin::HandshakeMessage::Propagate) => { // Message will still be processed as "member // added" or similar system message. } Err(err) => { *hidden = true; context.bob.write().await.status = 0; // secure-join failed context.stop_ongoing().await; error!(context, "Error in Secure-Join message handling: {}", err); } } } let (test_normal_chat_id, test_normal_chat_id_blocked) = chat::lookup_by_contact_id(context, from_id) .await .unwrap_or_default(); // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list, // it might also be blocked and displayed in the deaddrop as a result if chat_id.is_unset() { // try to create a group // (groups appear automatically only if the _sender_ is known, see core issue #54) let create_blocked = if !test_normal_chat_id.is_unset() && test_normal_chat_id_blocked == Blocked::Not { Blocked::Not } else { Blocked::Deaddrop }; let (new_chat_id, new_chat_id_blocked) = create_or_lookup_group( context, &mut mime_parser, allow_creation, create_blocked, from_id, to_ids, ) .await?; *chat_id = new_chat_id; chat_id_blocked = new_chat_id_blocked; if !chat_id.is_unset() && chat_id_blocked != Blocked::Not && create_blocked == Blocked::Not { new_chat_id.unblock(context).await; chat_id_blocked = Blocked::Not; } } if chat_id.is_unset() { // check if the message belongs to a mailing list if mime_parser.is_mailinglist_message() { *chat_id = ChatId::new(DC_CHAT_ID_TRASH); info!(context, "Message belongs to a mailing list and is ignored.",); } } if chat_id.is_unset() { // try to create a normal chat let create_blocked = if from_id == to_id { Blocked::Not } else { Blocked::Deaddrop }; if !test_normal_chat_id.is_unset() { *chat_id = test_normal_chat_id; chat_id_blocked = test_normal_chat_id_blocked; } else if allow_creation { let (id, bl) = chat::create_or_lookup_by_contact_id(context, from_id, create_blocked) .await .unwrap_or_default(); *chat_id = id; chat_id_blocked = bl; } if !chat_id.is_unset() && Blocked::Not != chat_id_blocked { if Blocked::Not == create_blocked { chat_id.unblock(context).await; chat_id_blocked = Blocked::Not; } else if is_reply_to_known_message(context, mime_parser).await { // we do not want any chat to be created implicitly. Because of the origin-scale-up, // the contact requests will pop up and this should be just fine. Contact::scaleup_origin_by_id(context, from_id, Origin::IncomingReplyTo).await; info!( context, "Message is a reply to a known message, mark sender as known.", ); if !incoming_origin.is_known() { incoming_origin = Origin::IncomingReplyTo; } } } } if chat_id.is_unset() { // maybe from_id is null or sth. else is suspicious, move message to trash *chat_id = ChatId::new(DC_CHAT_ID_TRASH); } // if the chat_id is blocked, // for unknown senders and non-delta-messages set the state to NOTICED // to not result in a chatlist-contact-request (this would require the state FRESH) if Blocked::Not != chat_id_blocked && state == MessageState::InFresh && !incoming_origin.is_known() && msgrmsg == MessengerMessage::No && show_emails != ShowEmails::All { state = MessageState::InNoticed; } } else { // Outgoing // the mail is on the IMAP server, probably it is also delivered. // We cannot recreate other states (read, error). state = MessageState::OutDelivered; to_id = to_ids.get_index(0).cloned().unwrap_or_default(); if !to_ids.is_empty() { if chat_id.is_unset() { let (new_chat_id, new_chat_id_blocked) = create_or_lookup_group( context, &mut mime_parser, allow_creation, Blocked::Not, from_id, to_ids, ) .await?; *chat_id = new_chat_id; chat_id_blocked = new_chat_id_blocked; // automatically unblock chat when the user sends a message if !chat_id.is_unset() && chat_id_blocked != Blocked::Not { new_chat_id.unblock(context).await; chat_id_blocked = Blocked::Not; } } if chat_id.is_unset() && allow_creation { let create_blocked = if MessengerMessage::No != msgrmsg && !Contact::is_blocked_load(context, to_id).await { Blocked::Not } else { Blocked::Deaddrop }; let (id, bl) = chat::create_or_lookup_by_contact_id(context, to_id, create_blocked) .await .unwrap_or_default(); *chat_id = id; chat_id_blocked = bl; if !chat_id.is_unset() && Blocked::Not != chat_id_blocked && Blocked::Not == create_blocked { chat_id.unblock(context).await; chat_id_blocked = Blocked::Not; } } } let self_sent = from_id == DC_CONTACT_ID_SELF && to_ids.len() == 1 && to_ids.contains(&DC_CONTACT_ID_SELF); if chat_id.is_unset() && self_sent { // from_id==to_id==DC_CONTACT_ID_SELF - this is a self-sent messages, // maybe an Autocrypt Setup Message let (id, bl) = chat::create_or_lookup_by_contact_id(context, DC_CONTACT_ID_SELF, Blocked::Not) .await .unwrap_or_default(); *chat_id = id; chat_id_blocked = bl; if !chat_id.is_unset() && Blocked::Not != chat_id_blocked { chat_id.unblock(context).await; chat_id_blocked = Blocked::Not; } } if chat_id.is_unset() { *chat_id = ChatId::new(DC_CHAT_ID_TRASH); } } // correct message_timestamp, it should not be used before, // however, we cannot do this earlier as we need from_id to be set calc_timestamps( context, *chat_id, from_id, *sent_timestamp, !seen, &mut sort_timestamp, sent_timestamp, &mut rcvd_timestamp, ) .await; // unarchive chat chat_id.unarchive(context).await?; // if the mime-headers should be saved, find out its size // (the mime-header ends with an empty line) let save_mime_headers = context.get_config_bool(Config::SaveMimeHeaders).await; if let Some(raw) = mime_parser.get(HeaderDef::InReplyTo) { mime_in_reply_to = raw.clone(); } if let Some(raw) = mime_parser.get(HeaderDef::References) { mime_references = raw.clone(); } // fine, so far. now, split the message into simple parts usable as "short messages" // and add them to the database (mails sent by other messenger clients should result // into only one message; mails sent by other clients may result in several messages // (eg. one per attachment)) let icnt = mime_parser.parts.len(); context .sql .with_conn(|mut conn| { let subject = mime_parser.get_subject().unwrap_or_default(); let mut txt_raw = None; for part in mime_parser.parts.iter_mut() { let mut stmt = conn.prepare_cached( "INSERT INTO msgs \ (rfc724_mid, server_folder, server_uid, chat_id, from_id, to_id, timestamp, \ timestamp_sent, timestamp_rcvd, type, state, msgrmsg, txt, txt_raw, param, \ bytes, hidden, mime_headers, mime_in_reply_to, mime_references) \ VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);", )?; if mime_parser.location_kml.is_some() && icnt == 1 && (part.msg == "-location-" || part.msg.is_empty()) { *hidden = true; if state == MessageState::InFresh { state = MessageState::InNoticed; } } if part.typ == Viewtype::Text { let msg_raw = part.msg_raw.as_ref().cloned().unwrap_or_default(); txt_raw = Some(format!("{}\n\n{}", subject, msg_raw)); } if mime_parser.is_system_message != SystemMessage::Unknown { part.param .set_int(Param::Cmd, mime_parser.is_system_message as i32); } stmt.execute(paramsv![ rfc724_mid, server_folder.as_ref().to_string(), server_uid as i32, *chat_id, from_id as i32, to_id as i32, sort_timestamp, *sent_timestamp, rcvd_timestamp, part.typ, state, msgrmsg, part.msg, // txt_raw might contain invalid utf8 txt_raw.unwrap_or_default(), part.param.to_string(), part.bytes as isize, *hidden, if save_mime_headers { Some(String::from_utf8_lossy(imf_raw)) } else { None }, mime_in_reply_to, mime_references, ])?; txt_raw = None; // This is okay, as we use a cached prepared statement. drop(stmt); let row_id = crate::sql::get_rowid(context, &mut conn, "msgs", "rfc724_mid", &rfc724_mid)?; *insert_msg_id = MsgId::new(row_id); created_db_entries.push((*chat_id, *insert_msg_id)); } Ok(()) }) .await?; info!( context, "Message has {} parts and is assigned to chat #{}.", icnt, *chat_id, ); // check event to send if chat_id.is_trash() { *create_event_to_send = None; } else if incoming && state == MessageState::InFresh { if from_id_blocked { *create_event_to_send = None; } else if Blocked::Not != chat_id_blocked { *create_event_to_send = Some(CreateEvent::MsgsChanged); } else { *create_event_to_send = Some(CreateEvent::IncomingMsg); } } Ok(()) } async fn save_locations( context: &Context, mime_parser: &MimeMessage, chat_id: ChatId, from_id: u32, insert_msg_id: MsgId, hidden: bool, ) { if chat_id.is_special() { return; } let mut location_id_written = false; let mut send_event = false; if mime_parser.message_kml.is_some() { let locations = &mime_parser.message_kml.as_ref().unwrap().locations; let newest_location_id = location::save(context, chat_id, from_id, locations, true) .await .unwrap_or_default(); if 0 != newest_location_id && !hidden && location::set_msg_location_id(context, insert_msg_id, newest_location_id) .await .is_ok() { location_id_written = true; send_event = true; } } if mime_parser.location_kml.is_some() { if let Some(ref addr) = mime_parser.location_kml.as_ref().unwrap().addr { if let Ok(contact) = Contact::get_by_id(context, from_id).await { if contact.get_addr().to_lowercase() == addr.to_lowercase() { let locations = &mime_parser.location_kml.as_ref().unwrap().locations; let newest_location_id = location::save(context, chat_id, from_id, locations, false) .await .unwrap_or_default(); if newest_location_id != 0 && !hidden && !location_id_written { if let Err(err) = location::set_msg_location_id( context, insert_msg_id, newest_location_id, ) .await { error!(context, "Failed to set msg_location_id: {:?}", err); } } send_event = true; } } } } if send_event { context.call_cb(Event::LocationChanged(Some(from_id))); } } #[allow(clippy::too_many_arguments)] async fn calc_timestamps( context: &Context, chat_id: ChatId, from_id: u32, message_timestamp: i64, is_fresh_msg: bool, sort_timestamp: &mut i64, sent_timestamp: &mut i64, rcvd_timestamp: &mut i64, ) { *rcvd_timestamp = time(); *sent_timestamp = message_timestamp; if *sent_timestamp > *rcvd_timestamp { *sent_timestamp = *rcvd_timestamp } *sort_timestamp = message_timestamp; if is_fresh_msg { let last_msg_time: Option = context .sql .query_get_value( context, "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?", paramsv![chat_id, from_id as i32, *sort_timestamp], ) .await; if let Some(last_msg_time) = last_msg_time { if last_msg_time > 0 && *sort_timestamp <= last_msg_time { *sort_timestamp = last_msg_time + 1; } } } if *sort_timestamp >= dc_smeared_time(context).await { *sort_timestamp = dc_create_smeared_timestamp(context).await; } } /// This function tries extracts the group-id from the message and returns the /// corresponding chat_id. If the chat_id is not existent, it is created. /// If the message contains groups commands (name, profile image, changed members), /// they are executed as well. /// /// if no group-id could be extracted from the message, create_or_lookup_adhoc_group() is called /// which tries to create or find out the chat_id by: /// - is there a group with the same recipients? if so, use this (if there are multiple, use the most recent one) /// - create an ad-hoc group based on the recipient list /// /// on success the function returns the found/created (chat_id, chat_blocked) tuple . #[allow(non_snake_case, clippy::cognitive_complexity)] async fn create_or_lookup_group( context: &Context, mime_parser: &mut MimeMessage, allow_creation: bool, create_blocked: Blocked, from_id: u32, to_ids: &ContactIds, ) -> Result<(ChatId, Blocked)> { let mut chat_id_blocked = Blocked::Not; let mut recreate_member_list = false; let mut send_EVENT_CHAT_MODIFIED = false; let mut X_MrRemoveFromGrp = None; let mut X_MrAddToGrp = None; let mut X_MrGrpNameChanged = false; let mut better_msg: String = From::from(""); if mime_parser.is_system_message == SystemMessage::LocationStreamingEnabled { better_msg = context .stock_system_msg(StockMessage::MsgLocationEnabled, "", "", from_id as u32) .await; set_better_msg(mime_parser, &better_msg); } let mut grpid = "".to_string(); if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupId) { grpid = optional_field.clone(); } if grpid.is_empty() { if let Some(value) = mime_parser.get(HeaderDef::MessageId) { if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(&value) { grpid = extracted_grpid.to_string(); } } if grpid.is_empty() { if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) { grpid = extracted_grpid.to_string(); } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) { grpid = extracted_grpid.to_string(); } else { return create_or_lookup_adhoc_group( context, mime_parser, allow_creation, create_blocked, from_id, to_ids, ) .await .map_err(|err| { info!(context, "could not create adhoc-group: {:?}", err); err }); } } } // now we have a grpid that is non-empty // but we might not know about this group let grpname = mime_parser.get(HeaderDef::ChatGroupName).cloned(); if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() { X_MrRemoveFromGrp = Some(optional_field); mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup; let left_group = Contact::lookup_id_by_addr(context, X_MrRemoveFromGrp.as_ref().unwrap()) .await == from_id as u32; better_msg = context .stock_system_msg( if left_group { StockMessage::MsgGroupLeft } else { StockMessage::MsgDelMember }, X_MrRemoveFromGrp.as_ref().unwrap(), "", from_id as u32, ) .await } else { let field = mime_parser.get(HeaderDef::ChatGroupMemberAdded).cloned(); if let Some(optional_field) = field { mime_parser.is_system_message = SystemMessage::MemberAddedToGroup; better_msg = context .stock_system_msg( StockMessage::MsgAddMember, &optional_field, "", from_id as u32, ) .await; X_MrAddToGrp = Some(optional_field); } else { let field = mime_parser.get(HeaderDef::ChatGroupNameChanged); if let Some(field) = field { X_MrGrpNameChanged = true; better_msg = context .stock_system_msg( StockMessage::MsgGrpName, field, if let Some(ref name) = grpname { name } else { "" }, from_id as u32, ) .await; mime_parser.is_system_message = SystemMessage::GroupNameChanged; } else if let Some(value) = mime_parser.get(HeaderDef::ChatContent) { if value == "group-avatar-changed" { if let Some(avatar_action) = &mime_parser.group_avatar { // this is just an explicit message containing the group-avatar, // apart from that, the group-avatar is send along with various other messages mime_parser.is_system_message = SystemMessage::GroupImageChanged; better_msg = context .stock_system_msg( match avatar_action { AvatarAction::Delete => StockMessage::MsgGrpImgDeleted, AvatarAction::Change(_) => StockMessage::MsgGrpImgChanged, }, "", "", from_id as u32, ) .await } } } } } set_better_msg(mime_parser, &better_msg); // check, if we have a chat with this group ID let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid) .await .unwrap_or((ChatId::new(0), false, Blocked::Not)); if !chat_id.is_error() { if chat_id_verified { if let Err(err) = check_verified_properties(context, mime_parser, from_id as u32, to_ids).await { warn!(context, "verification problem: {}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(s); } } if !chat::is_contact_in_chat(context, chat_id, from_id as u32).await { // The From-address is not part of this group. // It could be a new user or a DSN from a mailer-daemon. // in any case we do not want to recreate the member list // but still show the message as part of the chat. // After all, the sender has a reference/in-reply-to that // points to this chat. let s = context.stock_str(StockMessage::UnknownSenderForChat).await; mime_parser.repl_msg_by_error(s.to_string()); } } // check if the group does not exist but should be created let group_explicitly_left = chat::is_group_explicitly_left(context, &grpid) .await .unwrap_or_default(); let self_addr = context .get_config(Config::ConfiguredAddr) .await .unwrap_or_default(); if chat_id.is_error() && !mime_parser.is_mailinglist_message() && !grpid.is_empty() && grpname.is_some() // otherwise, a pending "quit" message may pop up && X_MrRemoveFromGrp.is_none() // re-create explicitly left groups only if ourself is re-added && (!group_explicitly_left || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap())) { // group does not exist but should be created let create_verified = if mime_parser.get(HeaderDef::ChatVerified).is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id as u32, to_ids).await { warn!(context, "verification problem: {}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } VerifiedStatus::Verified } else { VerifiedStatus::Unverified }; if !allow_creation { info!(context, "creating group forbidden by caller"); return Ok((ChatId::new(0), Blocked::Not)); } chat_id = create_group_record( context, &grpid, grpname.as_ref().unwrap(), create_blocked, create_verified, ) .await; chat_id_blocked = create_blocked; recreate_member_list = true; } // again, check chat_id if chat_id.is_special() { return if group_explicitly_left { Ok((ChatId::new(DC_CHAT_ID_TRASH), chat_id_blocked)) } else { create_or_lookup_adhoc_group( context, mime_parser, allow_creation, create_blocked, from_id, to_ids, ) .await .map_err(|err| { warn!(context, "failed to create ad-hoc group: {:?}", err); err }) }; } // We have a valid chat_id > DC_CHAT_ID_LAST_SPECIAL. // // However, it's possible that we got a non-DC message // and the user hit "reply" instead of "reply-all". // We heuristically detect this case and show // a placeholder-system-message to warn about this // and refer to "message-info" to see the message. // This is similar to how we show messages arriving // in verified chat using an un-verified key or cleartext. // XXX insert code in a different PR :) // execute group commands if X_MrAddToGrp.is_some() { recreate_member_list = true; } else if X_MrGrpNameChanged { if let Some(ref grpname) = grpname { if grpname.len() < 200 { info!(context, "updating grpname for chat {}", chat_id); if context .sql .execute( "UPDATE chats SET name=? WHERE id=?;", paramsv![grpname.to_string(), chat_id], ) .await .is_ok() { context.call_cb(Event::ChatModified(chat_id)); } } } } if let Some(avatar_action) = &mime_parser.group_avatar { info!(context, "group-avatar change for {}", chat_id); if let Ok(mut chat) = Chat::load_from_db(context, chat_id).await { match avatar_action { AvatarAction::Change(profile_image) => { chat.param.set(Param::ProfileImage, profile_image); } AvatarAction::Delete => { chat.param.remove(Param::ProfileImage); } }; chat.update_param(context).await?; send_EVENT_CHAT_MODIFIED = true; } } // add members to group/check members if recreate_member_list { if !chat::is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF).await { chat::add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await; } if from_id > DC_CONTACT_ID_LAST_SPECIAL && !Contact::addr_equals_contact(context, &self_addr, from_id as u32).await && !chat::is_contact_in_chat(context, chat_id, from_id).await { chat::add_to_chat_contacts_table(context, chat_id, from_id as u32).await; } for &to_id in to_ids.iter() { info!(context, "adding to={:?} to chat id={}", to_id, chat_id); if !Contact::addr_equals_contact(context, &self_addr, to_id).await && !chat::is_contact_in_chat(context, chat_id, to_id).await { chat::add_to_chat_contacts_table(context, chat_id, to_id).await; } } send_EVENT_CHAT_MODIFIED = true; } else if let Some(removed_addr) = X_MrRemoveFromGrp { let contact_id = Contact::lookup_id_by_addr(context, removed_addr).await; if contact_id != 0 { info!(context, "remove {:?} from chat id={}", contact_id, chat_id); chat::remove_from_chat_contacts_table(context, chat_id, contact_id).await; } send_EVENT_CHAT_MODIFIED = true; } if send_EVENT_CHAT_MODIFIED { context.call_cb(Event::ChatModified(chat_id)); } Ok((chat_id, chat_id_blocked)) } /// try extract a grpid from a message-id list header value fn extract_grpid(mime_parser: &MimeMessage, headerdef: HeaderDef) -> Option<&str> { let header = mime_parser.get(headerdef)?; let parts = header .split(',') .map(str::trim) .filter(|part| !part.is_empty()); parts.filter_map(dc_extract_grpid_from_rfc724_mid).next() } /// Handle groups for received messages, return chat_id/Blocked status on success async fn create_or_lookup_adhoc_group( context: &Context, mime_parser: &MimeMessage, allow_creation: bool, create_blocked: Blocked, from_id: u32, to_ids: &ContactIds, ) -> Result<(ChatId, Blocked)> { // if we're here, no grpid was found, check if there is an existing // ad-hoc group matching the to-list or if we should and can create one // (we do not want to heuristically look at the likely mangled Subject) if mime_parser.is_mailinglist_message() { // XXX we could parse List-* headers and actually create and // manage a mailing list group, eventually info!( context, "not creating ad-hoc group for mailing list message" ); return Ok((ChatId::new(0), Blocked::Not)); } let mut member_ids: Vec = to_ids.iter().copied().collect(); if !member_ids.contains(&from_id) { member_ids.push(from_id); } if !member_ids.contains(&DC_CONTACT_ID_SELF) { member_ids.push(DC_CONTACT_ID_SELF); } if member_ids.len() < 3 { info!(context, "not creating ad-hoc group: too few contacts"); return Ok((ChatId::new(0), Blocked::Not)); } let chat_ids = search_chat_ids_by_contact_ids(context, &member_ids).await?; if !chat_ids.is_empty() { let chat_ids_str = join(chat_ids.iter().map(|x| x.to_string()), ","); let res = context .sql .query_row( format!( "SELECT c.id, c.blocked FROM chats c LEFT JOIN msgs m ON m.chat_id=c.id WHERE c.id IN({}) ORDER BY m.timestamp DESC, m.id DESC LIMIT 1;", chat_ids_str ), paramsv![], |row| { Ok(( row.get::<_, ChatId>(0)?, row.get::<_, Option>(1)?.unwrap_or_default(), )) }, ) .await; if let Ok((id, id_blocked)) = res { /* success, chat found */ return Ok((id, id_blocked)); } } if !allow_creation { info!(context, "creating ad-hoc group prevented from caller"); return Ok((ChatId::new(0), Blocked::Not)); } // we do not check if the message is a reply to another group, this may result in // chats with unclear member list. instead we create a new group in the following lines ... // create a new ad-hoc group // - there is no need to check if this group exists; otherwise we would have caught it above let grpid = create_adhoc_grp_id(context, &member_ids).await; if grpid.is_empty() { warn!( context, "failed to create ad-hoc grpid for {:?}", member_ids ); return Ok((ChatId::new(0), Blocked::Not)); } // use subject as initial chat name let default_name = context .stock_string_repl_int(StockMessage::Member, member_ids.len() as i32) .await; let grpname = mime_parser.get_subject().unwrap_or_else(|| default_name); // create group record let new_chat_id: ChatId = create_group_record( context, &grpid, grpname, create_blocked, VerifiedStatus::Unverified, ) .await; for &member_id in &member_ids { chat::add_to_chat_contacts_table(context, new_chat_id, member_id).await; } context.call_cb(Event::ChatModified(new_chat_id)); Ok((new_chat_id, create_blocked)) } async fn create_group_record( context: &Context, grpid: impl AsRef, grpname: impl AsRef, create_blocked: Blocked, create_verified: VerifiedStatus, ) -> ChatId { if context.sql.execute( "INSERT INTO chats (type, name, grpid, blocked, created_timestamp) VALUES(?, ?, ?, ?, ?);", paramsv![ if VerifiedStatus::Unverified != create_verified { Chattype::VerifiedGroup } else { Chattype::Group }, grpname.as_ref(), grpid.as_ref(), create_blocked, time(), ], ).await .is_err() { warn!( context, "Failed to create group '{}' for grpid={}", grpname.as_ref(), grpid.as_ref() ); return ChatId::new(0); } let row_id = context .sql .get_rowid(context, "chats", "grpid", grpid.as_ref()) .await .unwrap_or_default(); let chat_id = ChatId::new(row_id); info!( context, "Created group '{}' grpid={} as {}", grpname.as_ref(), grpid.as_ref(), chat_id ); chat_id } async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String { /* algorithm: - sort normalized, lowercased, e-mail addresses alphabetically - put all e-mail addresses into a single string, separate the address by a single comma - sha-256 this string (without possibly terminating null-characters) - encode the first 64 bits of the sha-256 output as lowercase hex (results in 16 characters from the set [0-9a-f]) */ let member_ids_str = join(member_ids.iter().map(|x| x.to_string()), ","); let member_cs = context .get_config(Config::ConfiguredAddr) .await .unwrap_or_else(|| "no-self".to_string()) .to_lowercase(); let members = context .sql .query_map( format!( "SELECT addr FROM contacts WHERE id IN({}) AND id!=1", // 1=DC_CONTACT_ID_SELF member_ids_str ), paramsv![], |row| row.get::<_, String>(0), |rows| { let mut addrs = rows.collect::, _>>()?; addrs.sort(); let mut acc = member_cs.clone(); for addr in &addrs { acc += ","; acc += &addr.to_lowercase(); } Ok(acc) }, ) .await .unwrap_or_else(|_| member_cs); hex_hash(&members) } fn hex_hash(s: impl AsRef) -> String { let bytes = s.as_ref().as_bytes(); let result = Sha256::digest(bytes); hex::encode(&result[..8]) } async fn search_chat_ids_by_contact_ids( context: &Context, unsorted_contact_ids: &[u32], ) -> Result> { /* searches chat_id's by the given contact IDs, may return zero, one or more chat_id's */ let mut contact_ids = Vec::with_capacity(23); let mut chat_ids = Vec::with_capacity(23); /* copy array, remove duplicates and SELF, sort by ID */ if !unsorted_contact_ids.is_empty() { for &curr_id in unsorted_contact_ids { if curr_id != 1 && !contact_ids.contains(&curr_id) { contact_ids.push(curr_id); } } if !contact_ids.is_empty() { contact_ids.sort(); let contact_ids_str = join(contact_ids.iter().map(|x| x.to_string()), ","); context.sql.query_map( format!( "SELECT DISTINCT cc.chat_id, cc.contact_id FROM chats_contacts cc LEFT JOIN chats c ON c.id=cc.chat_id WHERE cc.chat_id IN(SELECT chat_id FROM chats_contacts WHERE contact_id IN({})) AND c.type=120 AND cc.contact_id!=1 ORDER BY cc.chat_id, cc.contact_id;", // 1=DC_CONTACT_ID_SELF contact_ids_str ), paramsv![], |row| Ok((row.get::<_, ChatId>(0)?, row.get::<_, u32>(1)?)), |rows| { let mut last_chat_id = ChatId::new(0); let mut matches = 0; let mut mismatches = 0; for row in rows { let (chat_id, contact_id) = row?; if chat_id != last_chat_id { if matches == contact_ids.len() && mismatches == 0 { chat_ids.push(last_chat_id); } last_chat_id = chat_id; matches = 0; mismatches = 0; } if matches < contact_ids.len() && contact_id == contact_ids[matches] { matches += 1; } else { mismatches += 1; } } if matches == contact_ids.len() && mismatches == 0 { chat_ids.push(last_chat_id); } Ok(()) } ).await?; } } Ok(chat_ids) } async fn check_verified_properties( context: &Context, mimeparser: &MimeMessage, from_id: u32, to_ids: &ContactIds, ) -> Result<()> { let contact = Contact::load_from_db(context, from_id).await?; ensure!(mimeparser.was_encrypted(), "This message is not encrypted."); // ensure, the contact is verified // and the message is signed with a verified key of the sender. // this check is skipped for SELF as there is no proper SELF-peerstate // and results in group-splits otherwise. if from_id != DC_CONTACT_ID_SELF { let peerstate = Peerstate::from_addr(context, &context.sql, contact.get_addr()).await; if peerstate.is_none() || contact.is_verified_ex(context, peerstate.as_ref()).await != VerifiedStatus::BidirectVerified { bail!( "Sender of this message is not verified: {}", contact.get_addr() ); } if let Some(peerstate) = peerstate { ensure!( peerstate.has_verified_key(&mimeparser.signatures), "The message was sent with non-verified encryption." ); } } // we do not need to check if we are verified with ourself let mut to_ids = to_ids.clone(); to_ids.remove(&DC_CONTACT_ID_SELF); if to_ids.is_empty() { return Ok(()); } let to_ids_str = join(to_ids.iter().map(|x| x.to_string()), ","); let rows = context .sql .query_map( format!( "SELECT c.addr, LENGTH(ps.verified_key_fingerprint) FROM contacts c \ LEFT JOIN acpeerstates ps ON c.addr=ps.addr WHERE c.id IN({}) ", to_ids_str ), paramsv![], |row| Ok((row.get::<_, String>(0)?, row.get::<_, i32>(1).unwrap_or(0))), |rows| { rows.collect::, _>>() .map_err(Into::into) }, ) .await?; for (to_addr, _is_verified) in rows.into_iter() { info!( context, "check_verified_properties: {:?} self={:?}", to_addr, context.is_self_addr(&to_addr).await ); let mut is_verified = _is_verified != 0; let peerstate = Peerstate::from_addr(context, &context.sql, &to_addr).await; // mark gossiped keys (if any) as verified if mimeparser.gossipped_addr.contains(&to_addr) { if let Some(mut peerstate) = peerstate { // if we're here, we know the gossip key is verified: // - use the gossip-key as verified-key if there is no verified-key // - OR if the verified-key does not match public-key or gossip-key // (otherwise a verified key can _only_ be updated through QR scan which might be annoying, // see https://github.com/nextleap-project/countermitm/issues/46 for a discussion about this point) if !is_verified || peerstate.verified_key_fingerprint != peerstate.public_key_fingerprint && peerstate.verified_key_fingerprint != peerstate.gossip_key_fingerprint { info!(context, "{} has verified {}.", contact.get_addr(), to_addr,); let fp = peerstate.gossip_key_fingerprint.clone(); if let Some(fp) = fp { peerstate.set_verified( PeerstateKeyType::GossipKey, &fp, PeerstateVerifiedStatus::BidirectVerified, ); peerstate.save_to_db(&context.sql, false).await?; is_verified = true; } } } } if !is_verified { bail!( "{} is not a member of this verified group", to_addr.to_string() ); } } Ok(()) } fn set_better_msg(mime_parser: &mut MimeMessage, better_msg: impl AsRef) { let msg = better_msg.as_ref(); if !msg.is_empty() && !mime_parser.parts.is_empty() { let part = &mut mime_parser.parts[0]; if part.typ == Viewtype::Text { part.msg = msg.to_string(); } } } async fn is_reply_to_known_message(context: &Context, mime_parser: &MimeMessage) -> bool { /* check if the message is a reply to a known message; the replies are identified by the Message-ID from `In-Reply-To`/`References:` (to support non-Delta-Clients) */ if let Some(field) = mime_parser.get(HeaderDef::InReplyTo) { if is_known_rfc724_mid_in_list(context, &field).await { return true; } } if let Some(field) = mime_parser.get(HeaderDef::References) { if is_known_rfc724_mid_in_list(context, &field).await { return true; } } false } async fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool { if mid_list.is_empty() { return false; } if let Ok(ids) = mailparse::msgidparse(mid_list) { for id in ids.iter() { if is_known_rfc724_mid(context, id).await { return true; } } } false } /// Check if a message is a reply to a known message (messenger or non-messenger). async fn is_known_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool { context .sql .exists( "SELECT m.id FROM msgs m \ LEFT JOIN chats c ON m.chat_id=c.id \ WHERE m.rfc724_mid=? \ AND m.chat_id>9 AND c.blocked=0;", paramsv![rfc724_mid], ) .await .unwrap_or_default() } /// Checks if the message defined by mime_parser references a message send by us from Delta Chat. /// This is similar to is_reply_to_known_message() but /// - checks also if any of the referenced IDs are send by a messenger /// - it is okay, if the referenced messages are moved to trash here /// - no check for the Chat-* headers (function is only called if it is no messenger message itself) async fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) -> bool { if let Some(value) = mime_parser.get(HeaderDef::InReplyTo) { if is_msgrmsg_rfc724_mid_in_list(context, &value).await { return true; } } if let Some(value) = mime_parser.get(HeaderDef::References) { if is_msgrmsg_rfc724_mid_in_list(context, &value).await { return true; } } false } pub(crate) async fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool { if let Ok(ids) = mailparse::msgidparse(mid_list) { for id in ids.iter() { if is_msgrmsg_rfc724_mid(context, id).await { return true; } } } false } /// Check if a message is a reply to any messenger message. async fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool { context .sql .exists( "SELECT id FROM msgs WHERE rfc724_mid=? AND msgrmsg!=0 AND chat_id>9;", paramsv![rfc724_mid], ) .await .unwrap_or_default() } async fn dc_add_or_lookup_contacts_by_address_list( context: &Context, addr_list_raw: &str, origin: Origin, ) -> Result { let addrs = match mailparse::addrparse(addr_list_raw) { Ok(addrs) => addrs, Err(err) => { bail!("could not parse {:?}: {:?}", addr_list_raw, err); } }; let mut contact_ids = ContactIds::new(); for addr in addrs.iter() { match addr { mailparse::MailAddr::Single(info) => { contact_ids.insert( add_or_lookup_contact_by_addr(context, &info.display_name, &info.addr, origin) .await?, ); } mailparse::MailAddr::Group(infos) => { for info in &infos.addrs { contact_ids.insert( add_or_lookup_contact_by_addr( context, &info.display_name, &info.addr, origin, ) .await?, ); } } } } Ok(contact_ids) } /// Add contacts to database on receiving messages. async fn add_or_lookup_contact_by_addr( context: &Context, display_name: &Option, addr: &str, origin: Origin, ) -> Result { if context.is_self_addr(addr).await? { return Ok(DC_CONTACT_ID_SELF); } let display_name_normalized = display_name .as_ref() .map(normalize_name) .unwrap_or_default(); let (row_id, _modified) = Contact::add_or_lookup(context, display_name_normalized, addr, origin).await?; ensure!(row_id > 0, "could not add contact: {:?}", addr); Ok(row_id) } fn dc_create_incoming_rfc724_mid( message_timestamp: i64, contact_id_from: u32, contact_ids_to: &ContactIds, ) -> Option { /* create a deterministic rfc724_mid from input such that repeatedly calling it with the same input results in the same Message-id */ let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default(); let result = format!( "{}-{}-{}@stub", message_timestamp, contact_id_from, largest_id_to ); Some(result) } #[cfg(test)] mod tests { use super::*; use crate::test_utils::dummy_context; #[test] fn test_hex_hash() { let data = "hello world"; let res = hex_hash(data); assert_eq!(res, "b94d27b9934d3e08"); } #[async_std::test] async fn test_grpid_simple() { let context = dummy_context().await; let raw = b"From: hello\n\ Subject: outer-subject\n\ In-Reply-To: \n\ References: \n\ \n\ hello\x00"; let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), None); let grpid = Some("HcxyMARjyJy"); assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid); } #[async_std::test] async fn test_grpid_from_multiple() { let context = dummy_context().await; let raw = b"From: hello\n\ Subject: outer-subject\n\ In-Reply-To: \n\ References: , \n\ \n\ hello\x00"; let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]) .await .unwrap(); let grpid = Some("HcxyMARjyJy"); assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), grpid); assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid); } #[test] fn test_dc_create_incoming_rfc724_mid() { let mut members = ContactIds::new(); assert_eq!( dc_create_incoming_rfc724_mid(123, 45, &members), Some("123-45-0@stub".into()) ); members.insert(7); members.insert(3); assert_eq!( dc_create_incoming_rfc724_mid(123, 45, &members), Some("123-45-7@stub".into()) ); members.insert(9); assert_eq!( dc_create_incoming_rfc724_mid(123, 45, &members), Some("123-45-9@stub".into()) ); } }