diff --git a/Cargo.lock b/Cargo.lock index 5a147a4c5..f12ffb684 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,7 +681,7 @@ dependencies = [ "lettre_email 0.9.2 (git+https://github.com/deltachat/lettre)", "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "mailparse 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", + "mailparse 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "num-derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1504,7 +1504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "mailparse" -version = "0.10.4" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3332,7 +3332,7 @@ dependencies = [ "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" "checksum lzw 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d947cbb889ed21c2a84be6ffbaebf5b4e0f4340638cba0444907e38b56be084" -"checksum mailparse 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6c03df7fe4bab038aaa2f313baae7600de0afd606f8244860801c46f53babdd8" +"checksum mailparse 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7181507a68fef921f011b0c0f143efca20871da5ab3963bdc064537278469cd2" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" "checksum md-5 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8" diff --git a/Cargo.toml b/Cargo.toml index e25fa69e3..a40e166cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ bitflags = "1.1.0" debug_stub_derive = "0.3.0" sanitize-filename = "0.2.1" stop-token = { version = "0.1.1", features = ["unstable"] } -mailparse = "0.10.2" +mailparse = "0.12.0" encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" } native-tls = "0.2.3" image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] } diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 4493bb633..8adcf3a8c 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -380,6 +380,14 @@ char* dc_get_blobdir (const dc_context_t* context); * - `save_mime_headers` = 1=save mime headers * and make dc_get_mime_headers() work for subsequent calls, * 0=do not save mime headers (default) + * - `delete_device_after` = 0=do not delete messages from device automatically (default), + * >=1=seconds, after which messages are deleted automatically from the device. + * Messages in the "saved messages" chat (see dc_chat_is_self_talk()) are skipped. + * Messages are deleted whether they were seen or not, the UI should clearly point that out. + * - `delete_server_after` = 0=do not delete messages from server automatically (default), + * >=1=seconds, after which messages are deleted automatically from the server. + * "Saved messages" are deleted from the server as well as + * emails matching the `show_emails` settings above, the UI should clearly point that out. * * If you want to retrieve a value, use dc_get_config(). * @@ -1012,6 +1020,21 @@ int dc_get_msg_cnt (dc_context_t* context, uint32_t ch int dc_get_fresh_msg_cnt (dc_context_t* context, uint32_t chat_id); + +/** + * Estimate the number of messages that will be deleted + * by the dc_set_config()-options `delete_device_after` or `delete_server_after`. + * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages. + * + * @param context The context object as returned from dc_context_new(). + * @param from_server 1=Estimate deletion count for server, 0=Estimate deletion count for device + * @param seconds Count messages older than the given number of seconds. + * @return Number of messages that are older than the given number of seconds. + * This includes emails downloaded due to the `show_emails` option. + * Messages in the "saved messages" folder are not counted as they will not be deleted automatically. + */ +int dc_estimate_deletion_cnt (dc_context_t* context, int from_server, int64_t seconds); + /** * Returns the message IDs of all _fresh_ messages of any chat. * Typically used for implementing notification summaries. @@ -1375,8 +1398,9 @@ char* dc_get_mime_headers (dc_context_t* context, uint32_t ms */ void dc_delete_msgs (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt); -/** +/* * Empty IMAP server folder: delete all messages. + * Deprecated, use dc_set_config() with the key "delete_server_after" instead. * * @memberof dc_context_t * @param context The context object as created by dc_context_new() @@ -3843,28 +3867,8 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); */ -/** - * @defgroup DC_EMPTY DC_EMPTY - * - * These constants configure emptying imap folders with dc_empty_server() - * - * @addtogroup DC_EMPTY - * @{ - */ - -/** - * Clear all mvbox messages. - */ -#define DC_EMPTY_MVBOX 0x01 - -/** - * Clear all INBOX messages. - */ -#define DC_EMPTY_INBOX 0x02 - -/** - * @} - */ +#define DC_EMPTY_MVBOX 0x01 // Deprecated, flag for dc_empty_server(): Clear all mvbox messages +#define DC_EMPTY_INBOX 0x02 // Deprecated, flag for dc_empty_server(): Clear all INBOX messages /** diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 73c08dcc5..5b65267aa 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -401,7 +401,7 @@ pub unsafe extern "C" fn dc_set_config_from_qr( pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char { if context.is_null() { eprintln!("ignoring careless call to dc_get_info()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_context = &*context; @@ -1061,6 +1061,25 @@ pub unsafe extern "C" fn dc_get_fresh_msg_cnt( .unwrap_or(0) } +#[no_mangle] +pub unsafe extern "C" fn dc_estimate_deletion_cnt( + context: *mut dc_context_t, + from_server: libc::c_int, + seconds: i64, +) -> libc::c_int { + if context.is_null() || seconds < 0 { + eprintln!("ignoring careless call to dc_estimate_deletion_cnt()"); + return 0; + } + let ffi_context = &*context; + ffi_context + .with_inner(|ctx| { + message::estimate_deletion_cnt(ctx, from_server != 0, seconds).unwrap_or(0) + as libc::c_int + }) + .unwrap_or(0) +} + #[no_mangle] pub unsafe extern "C" fn dc_get_fresh_msgs( context: *mut dc_context_t, @@ -1488,7 +1507,7 @@ pub unsafe extern "C" fn dc_get_msg_info( ) -> *mut libc::c_char { if context.is_null() { eprintln!("ignoring careless call to dc_get_msg_info()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_context = &*context; @@ -2486,7 +2505,7 @@ pub unsafe extern "C" fn dc_chat_get_type(chat: *mut dc_chat_t) -> libc::c_int { pub unsafe extern "C" fn dc_chat_get_name(chat: *mut dc_chat_t) -> *mut libc::c_char { if chat.is_null() { eprintln!("ignoring careless call to dc_chat_get_name()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_chat = &*chat; ffi_chat.chat.get_name().strdup() @@ -2810,7 +2829,7 @@ pub unsafe extern "C" fn dc_msg_get_sort_timestamp(msg: *mut dc_msg_t) -> i64 { pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_char { if msg.is_null() { eprintln!("ignoring careless call to dc_msg_get_text()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_msg = &*msg; ffi_msg.message.get_text().unwrap_or_default().strdup() @@ -2829,8 +2848,7 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha ffi_msg .message .get_file(ctx) - .and_then(|p| p.to_c_string().ok()) - .map(|cs| dc_strdup(cs.as_ptr())) + .map(|p| p.strdup()) .unwrap_or_else(|| "".strdup()) }) .unwrap_or_else(|_| "".strdup()) @@ -2840,7 +2858,7 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c_char { if msg.is_null() { eprintln!("ignoring careless call to dc_msg_get_filename()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_msg = &*msg; ffi_msg.message.get_filename().unwrap_or_default().strdup() @@ -2850,13 +2868,13 @@ pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c pub unsafe extern "C" fn dc_msg_get_filemime(msg: *mut dc_msg_t) -> *mut libc::c_char { if msg.is_null() { eprintln!("ignoring careless call to dc_msg_get_filemime()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_msg = &*msg; if let Some(x) = ffi_msg.message.get_filemime() { x.strdup() } else { - dc_strdup(ptr::null()) + "".strdup() } } @@ -3180,7 +3198,7 @@ pub unsafe extern "C" fn dc_contact_get_id(contact: *mut dc_contact_t) -> u32 { pub unsafe extern "C" fn dc_contact_get_addr(contact: *mut dc_contact_t) -> *mut libc::c_char { if contact.is_null() { eprintln!("ignoring careless call to dc_contact_get_addr()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_contact = &*contact; ffi_contact.contact.get_addr().strdup() @@ -3190,7 +3208,7 @@ pub unsafe extern "C" fn dc_contact_get_addr(contact: *mut dc_contact_t) -> *mut pub unsafe extern "C" fn dc_contact_get_name(contact: *mut dc_contact_t) -> *mut libc::c_char { if contact.is_null() { eprintln!("ignoring careless call to dc_contact_get_name()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_contact = &*contact; ffi_contact.contact.get_name().strdup() @@ -3202,7 +3220,7 @@ pub unsafe extern "C" fn dc_contact_get_display_name( ) -> *mut libc::c_char { if contact.is_null() { eprintln!("ignoring careless call to dc_contact_get_display_name()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_contact = &*contact; ffi_contact.contact.get_display_name().strdup() @@ -3214,7 +3232,7 @@ pub unsafe extern "C" fn dc_contact_get_name_n_addr( ) -> *mut libc::c_char { if contact.is_null() { eprintln!("ignoring careless call to dc_contact_get_name_n_addr()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_contact = &*contact; ffi_contact.contact.get_name_n_addr().strdup() @@ -3226,7 +3244,7 @@ pub unsafe extern "C" fn dc_contact_get_first_name( ) -> *mut libc::c_char { if contact.is_null() { eprintln!("ignoring careless call to dc_contact_get_first_name()"); - return dc_strdup(ptr::null()); + return "".strdup(); } let ffi_contact = &*contact; ffi_contact.contact.get_first_name().strdup() diff --git a/deltachat-ffi/src/string.rs b/deltachat-ffi/src/string.rs index 7cfd1c357..256b110b0 100644 --- a/deltachat-ffi/src/string.rs +++ b/deltachat-ffi/src/string.rs @@ -9,7 +9,7 @@ use std::ptr; /// # Examples /// /// ```rust,norun -/// use deltachat::dc_tools::{dc_strdup, to_string_lossy}; +/// use crate::string::{dc_strdup, to_string_lossy}; /// unsafe { /// let str_a = b"foobar\x00" as *const u8 as *const libc::c_char; /// let str_a_copy = dc_strdup(str_a); @@ -17,7 +17,7 @@ use std::ptr; /// assert_ne!(str_a, str_a_copy); /// } /// ``` -pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char { +unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char { let ret: *mut libc::c_char; if !s.is_null() { ret = libc::strdup(s); @@ -32,7 +32,7 @@ pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char { /// Error type for the [OsStrExt] trait #[derive(Debug, Fail, PartialEq)] -pub enum CStringError { +pub(crate) enum CStringError { /// The string contains an interior null byte #[fail(display = "String contains an interior null byte")] InteriorNullByte, @@ -66,7 +66,7 @@ pub enum CStringError { /// let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr()); /// } /// ``` -pub trait OsStrExt { +pub(crate) trait OsStrExt { /// Convert a [std::ffi::OsStr] to an [std::ffi::CString] /// /// This is useful to convert e.g. a [std::path::Path] to @@ -131,15 +131,16 @@ fn os_str_to_c_string_unicode( } /// Convenience methods/associated functions for working with [CString] -/// -/// This is helps transitioning from unsafe code. -pub trait CStringExt { - /// Create a new [CString], yolo style +trait CStringExt { + /// Create a new [CString], best effort /// - /// This unwrap the result, panicking when there are embedded NULL - /// bytes. - fn yolo>>(t: T) -> CString { - CString::new(t).expect("String contains null byte, can not be CString") + /// Like the [to_string_lossy] this doesn't give up in the face of + /// bad input (embedded null bytes in this case) instead it does + /// the best it can by stripping the embedded null bytes. + fn new_lossy>>(t: T) -> CString { + let mut s = t.into(); + s.retain(|&c| c != 0); + CString::new(s).unwrap_or_default() } } @@ -151,7 +152,7 @@ impl CStringExt for CString {} /// Rust strings to raw C strings. This can be clumsy to do correctly /// and the compiler sometimes allows it in an unsafe way. These /// methods make it more succinct and help you get it right. -pub trait StrExt { +pub(crate) trait Strdup { /// Allocate a new raw C `*char` version of this string. /// /// This allocates a new raw C string which must be freed using @@ -168,35 +169,44 @@ pub trait StrExt { unsafe fn strdup(&self) -> *mut libc::c_char; } -impl> StrExt for T { +impl> Strdup for T { unsafe fn strdup(&self) -> *mut libc::c_char { - let tmp = CString::yolo(self.as_ref()); + let tmp = CString::new_lossy(self.as_ref()); + dc_strdup(tmp.as_ptr()) + } +} + +// We can not implement for AsRef because we already implement +// AsRev and this conflicts. So implement for Path directly. +impl Strdup for std::path::Path { + unsafe fn strdup(&self) -> *mut libc::c_char { + let tmp = self.to_c_string().unwrap_or_else(|_| CString::default()); dc_strdup(tmp.as_ptr()) } } /// Convenience methods to turn optional strings into C strings. /// -/// This is the same as the [StrExt] trait but a different trait name -/// to work around the type system not allowing to implement [StrExt] -/// for `Option` When we already have an [StrExt] impl +/// This is the same as the [Strdup] trait but a different trait name +/// to work around the type system not allowing to implement [Strdup] +/// for `Option` When we already have an [Strdup] impl /// for `AsRef<&str>`. /// /// When the [Option] is [Option::Some] this behaves just like -/// [StrExt::strdup], when it is [Option::None] a null pointer is +/// [Strdup::strdup], when it is [Option::None] a null pointer is /// returned. -pub trait OptStrExt { +pub(crate) trait OptStrdup { /// Allocate a new raw C `*char` version of this string, or NULL. /// - /// See [StrExt::strdup] for details. + /// See [Strdup::strdup] for details. unsafe fn strdup(&self) -> *mut libc::c_char; } -impl> OptStrExt for Option { +impl> OptStrdup for Option { unsafe fn strdup(&self) -> *mut libc::c_char { match self { Some(s) => { - let tmp = CString::yolo(s.as_ref()); + let tmp = CString::new_lossy(s.as_ref()); dc_strdup(tmp.as_ptr()) } None => ptr::null_mut(), @@ -204,7 +214,7 @@ impl> OptStrExt for Option { } } -pub fn to_string_lossy(s: *const libc::c_char) -> String { +pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String { if s.is_null() { return "".into(); } @@ -214,7 +224,7 @@ pub fn to_string_lossy(s: *const libc::c_char) -> String { cstr.to_string_lossy().to_string() } -pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option { +pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option { if s.is_null() { return None; } @@ -235,7 +245,7 @@ pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option { /// /// [Path]: std::path::Path #[cfg(not(target_os = "windows"))] -pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path { +pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path { assert!(!s.is_null(), "cannot be used on null pointers"); use std::os::unix::ffi::OsStrExt; unsafe { @@ -247,7 +257,7 @@ pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path { // as_path() implementation for windows, documented above. #[cfg(target_os = "windows")] -pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path { +pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path { as_path_unicode(s) } @@ -354,8 +364,14 @@ mod tests { } #[test] - fn test_cstring_yolo() { - assert_eq!(CString::new("hello").unwrap(), CString::yolo("hello")); + fn test_cstring_new_lossy() { + assert!(CString::new("hel\x00lo").is_err()); + assert!(CString::new(String::from("hel\x00o")).is_err()); + let r = CString::new("hello").unwrap(); + assert_eq!(CString::new_lossy("hello"), r); + assert_eq!(CString::new_lossy("hel\x00lo"), r); + assert_eq!(CString::new_lossy(String::from("hello")), r); + assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r); } #[test] diff --git a/draft/group-sync.rst b/draft/group-sync.rst new file mode 100644 index 000000000..286d19cb6 --- /dev/null +++ b/draft/group-sync.rst @@ -0,0 +1,126 @@ + +Problem: missing eventual group consistency +-------------------------------------------- + +If group members are concurrently adding new members, +the new members will miss each other's additions, example: + +- Alice and Bob are in a two-member group + +- Alice adds Carol, concurrently Bob adds Doris + +- Carol will see a three-member group (Alice, Bob, Carol), + Doris will see a different three-member group (Alice, Bob, Doris), + and only Alice and Bob will have all four members. + +Note that for verified groups any mitigation mechanism likely +needs to make all clients to know who originally added a member. + + +solution: memorize+attach (possible encrypted) chat-meta mime messages +---------------------------------------------------------------------- + +For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped. + + +- All Chat-Group-Member-Added/Removed messages are recorded in their + full raw (signed and encrypted) mime-format in the DB + +- If an incoming member-add/member-delete messages has a member list + which is, apart from the added/removed member, not consistent + with our own view, broadcast a "Chat-Group-Member-Correction" message to + all members, attaching the original added/removed mime-message for all mismatching + contacts. If we have no relevant add/del information, don't send a + correction message out. + +- Upong receiving added/removed attachments we don't do the + check_consistency+correction message dance. + This avoids recursion problems and hard-to-reason-about chatter. + +Notes: + +- mechanism works for both encrypted and unencrypted add/del messages + +- we already have a "mime_headers" column in the DB for each incoming message. + We could extend it to also include the payload and store mime unconditionally + for member-added/removed messages. + +- multiple member-added/removed messages can be attached in a single + correction message + +- it is minimal on the number of overall messages to reach group consistency + (best-case: no extra messages, the ABCD case above: max two extra messages) + +- somewhat backward compatible: older clients will probably ignore + messages which are signed by someone who is not the outer From-address. + +- the correction-protocol also helps with dropped messages. If a member + did not see a member-added/removed message, the next member add/removed + message in the group will likely heal group consistency for this member. + +- we can quite easily extend the mechanism to also provide the group-avatar or + other meta-information. + +Discussions of variants +++++++++++++++++++++++++ + +- instead of acting on MemberAdded/Removed message we could send + corrections for any received message that addresses inconsistent group members but + a) this would delay group-membership healing + b) could lead to a lot of members sending corrections + +- instead of broadcasting correction messages we could only send it to + the sender of the inconsistent member-added/removed message. + A receiver of such a correction message would then need to forward + the message to the members it thinks also have an inconsistent view. + This sounds complicated and error-prone. Concretely, if Alice + receives Bob's "Member-added: Doris" message, then Alice + broadcasting the correction message with "Member-added: Carol" + would reach all four members, healing group consistency in one step. + If Bob meanwhile receives Alice's "Member-Added: Carol" message, + Bob would broadcast a correction message to all four members as well. + (Imagine a situation where Alice/Bob added Carol/Doris + while both being in an offline or bad-connection situation). + + +solution2: repeat member-added/removed messages +--------------------------------------------------- + +Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed +but keep sending out the old headers until the new protocol is sufficiently deployed. + +The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL) +which controls repetition of the signed "add/del e-mail address" payload. + +Example:: + + Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org + owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h + bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi + ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR + iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP + FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup + b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk + mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM + clKdskprY+4LY0EBwLD3SQ7dPkTITCRD + =P6GG + +TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message. +Receivers will apply the add/del change to the group-membership, +decrease the TTL by 1, and if TTL>0 re-sent the header. + +The "add|del e-mail address" payload is pgp-signed and repeated verbatim. +This allows to propagate, in a cryptographically secured way, +who added a member. This is particularly important for allowing +to show in verified groups who added a member (planned). + +Disadvantage to solution 1: + +- requires to specify encoding and precise rules for what/how is signed. + +- causes O(N^2) extra messages + +- Not easily extendable for other things (without introducing a new + header / encoding) + + diff --git a/examples/repl/cmdline.rs b/examples/repl/cmdline.rs index 7b135aaa4..8bdffd0b6 100644 --- a/examples/repl/cmdline.rs +++ b/examples/repl/cmdline.rs @@ -394,6 +394,7 @@ pub async fn cmdline( providerinfo \n\ event \n\ fileinfo \n\ + estimatedeletion \n\ emptyserver (1=MVBOX 2=INBOX)\n\ clear -- clear screen\n\ exit or quit\n\ @@ -1049,6 +1050,16 @@ pub async fn cmdline( bail!("Command failed."); } } + "estimatedeletion" => { + ensure!(!arg1.is_empty(), "Argument missing"); + let seconds = arg1.parse()?; + let device_cnt = message::estimate_deletion_cnt(context, false, seconds)?; + let server_cnt = message::estimate_deletion_cnt(context, true, seconds)?; + println!( + "estimated count of messages older than {} seconds:\non device: {}\non server: {}", + seconds, device_cnt, server_cnt + ); + } "emptyserver" => { ensure!(!arg1.is_empty(), "Argument missing"); diff --git a/examples/repl/main.rs b/examples/repl/main.rs index 3b4f47ab5..96a39cec3 100644 --- a/examples/repl/main.rs +++ b/examples/repl/main.rs @@ -207,8 +207,17 @@ const CONTACT_COMMANDS: [&str; 6] = [ "delcontact", "cleanupcontacts", ]; -const MISC_COMMANDS: [&str; 9] = [ - "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help", +const MISC_COMMANDS: [&str; 10] = [ + "getqr", + "getbadqr", + "checkqr", + "event", + "fileinfo", + "clear", + "exit", + "quit", + "help", + "estimatedeletion", ]; impl Hinter for DcHelper { diff --git a/src/aheader.rs b/src/aheader.rs index a87f7be65..628de89f7 100644 --- a/src/aheader.rs +++ b/src/aheader.rs @@ -75,7 +75,7 @@ impl Aheader { wanted_from: &str, headers: &[mailparse::MailHeader<'_>], ) -> Option { - if let Ok(Some(value)) = headers.get_header_value(HeaderDef::Autocrypt) { + if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) { match Self::from_str(&value) { Ok(header) => { if addr_cmp(&header.addr, wanted_from) { diff --git a/src/chat.rs b/src/chat.rs index 531cda89c..12c8eccf6 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -301,10 +301,7 @@ impl ChatId { /// Returns `true`, if message was deleted, `false` otherwise. async fn maybe_delete_draft(self, context: &Context) -> bool { match self.get_draft_msg_id(context).await { - Some(msg_id) => { - Message::delete_from_db(context, msg_id).await; - true - } + Some(msg_id) => msg_id.delete_from_db(context).await.is_ok(), None => false, } } @@ -382,6 +379,26 @@ impl ChatId { .unwrap_or_default() as usize } + pub(crate) async fn get_param(self, context: &Context) -> Result { + let res: Option = context + .sql + .query_get_value_result("SELECT param FROM chats WHERE id=?", paramsv![self]) + .await?; + Ok(res + .map(|s| s.parse().unwrap_or_default()) + .unwrap_or_default()) + } + + // Returns true if chat is a saved messages chat. + pub async fn is_self_talk(self, context: &Context) -> Result { + Ok(self.get_param(context).await?.exists(Param::Selftalk)) + } + + /// Returns true if chat is a device chat. + pub async fn is_device_talk(self, context: &Context) -> Result { + Ok(self.get_param(context).await?.exists(Param::Devicetalk)) + } + /// Bad evil escape hatch. /// /// Avoid using this, eventually types should be cleaned up enough @@ -1518,6 +1535,18 @@ pub async fn get_chat_msgs( flags: u32, marker1before: Option, ) -> Vec { + match hide_device_expired_messages(context).await { + Err(err) => warn!(context, "Failed to delete expired messages: {}", err), + Ok(messages_deleted) => { + if messages_deleted { + context.call_cb(Event::MsgsChanged { + msg_id: MsgId::new(0), + chat_id: ChatId::new(0), + }) + } + } + } + let process_row = |row: &rusqlite::Row| Ok((row.get::<_, MsgId>("id")?, row.get::<_, i64>("timestamp")?)); let process_rows = |rows: rusqlite::MappedRows<_>| { @@ -1671,6 +1700,52 @@ pub async fn marknoticed_all_chats(context: &Context) -> Result<(), Error> { Ok(()) } +/// Hides messages which are expired according to "delete_device_after" setting. +/// +/// Returns true if any message is hidden, so event can be emitted. If nothing +/// has been hidden, returns false. +pub async fn hide_device_expired_messages(context: &Context) -> Result { + if let Some(delete_device_after) = context.get_config_delete_device_after().await { + let threshold_timestamp = time() - delete_device_after; + + let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF) + .await + .unwrap_or_default() + .0; + let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE) + .await + .unwrap_or_default() + .0; + + // Hide expired messages + // + // Only update the rows that have to be updated, to avoid emitting + // unnecessary "chat modified" events. + let rows_modified = context + .sql + .execute( + "UPDATE msgs \ + SET txt = 'DELETED', hidden = 1 \ + WHERE timestamp < ? \ + AND chat_id > ? \ + AND chat_id != ? \ + AND chat_id != ? \ + AND NOT hidden", + paramsv![ + threshold_timestamp, + DC_CHAT_ID_LAST_SPECIAL, + self_chat_id, + device_chat_id + ], + ) + .await?; + + Ok(rows_modified > 0) + } else { + Ok(false) + } +} + pub async fn get_chat_media( context: &Context, chat_id: ChatId, @@ -2182,7 +2257,7 @@ pub async fn remove_contact_from_chat( "Cannot remove contact from chat; self not in group.".into() ) ); - } else { + } else if remove_from_chat_contacts_table(context, chat_id, contact_id).await { /* we should respect this - whatever we send to the group, it gets discarded anyway! */ if let Ok(contact) = Contact::get_by_id(context, contact_id).await { if chat.is_promoted() { @@ -2220,10 +2295,8 @@ pub async fn remove_contact_from_chat( }); } } - if remove_from_chat_contacts_table(context, chat_id, contact_id).await { - context.call_cb(Event::ChatModified(chat_id)); - success = true; - } + context.call_cb(Event::ChatModified(chat_id)); + success = true; } } } diff --git a/src/chatlist.rs b/src/chatlist.rs index 1d56b2056..2eee6caf8 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -74,7 +74,8 @@ impl Chatlist { /// if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and /// the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived /// chats - /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist, + /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist + /// and hides the device-chat, // typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added /// to the list (may be used eg. for selecting chats on forwarding, the flag is @@ -91,6 +92,12 @@ impl Chatlist { query: Option<&str>, query_contact_id: Option, ) -> Result { + // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some + // messages get hidden to avoid reloading the same chatlist. + if let Err(err) = hide_device_expired_messages(context).await { + warn!(context, "Failed to hide expired messages: {}", err); + } + let mut add_archived_link_item = false; let process_row = |row: &rusqlite::Row| { @@ -104,6 +111,15 @@ impl Chatlist { .map_err(Into::into) }; + let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING { + chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE) + .await + .unwrap_or_default() + .0 + } else { + ChatId::new(0) + }; + // select with left join and minimum: // // - the inner select must use `hidden` and _not_ `m.hidden` @@ -142,6 +158,9 @@ impl Chatlist { ).await? } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY { // show archived chats + // (this includes the archived device-chat; we could skip it, + // however, then the number of archived chats do not match, which might be even more irritating. + // and adapting the number requires larger refactorings and seems not to be worth the effort) context .sql .query_map( @@ -186,13 +205,13 @@ impl Chatlist { SELECT MAX(timestamp) FROM msgs WHERE chat_id=c.id - AND (hidden=0 OR state=?)) - WHERE c.id>9 + AND (hidden=0 OR state=?1)) + WHERE c.id>9 AND c.id!=?2 AND c.blocked=0 - AND c.name LIKE ? + AND c.name LIKE ?3 GROUP BY c.id ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;", - paramsv![MessageState::OutDraft, str_like_cmd], + paramsv![MessageState::OutDraft, skip_id, str_like_cmd], process_row, process_rows, ) @@ -217,12 +236,12 @@ impl Chatlist { FROM msgs WHERE chat_id=c.id AND (hidden=0 OR state=?1)) - WHERE c.id>9 + WHERE c.id>9 AND c.id!=?2 AND c.blocked=0 - AND NOT c.archived=?2 + AND NOT c.archived=?3 GROUP BY c.id - ORDER BY c.id=?3 DESC, c.archived=?4 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;", - paramsv![MessageState::OutDraft, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned], + ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;", + paramsv![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned], process_row, process_rows, ).await?; @@ -446,16 +465,21 @@ mod tests { async fn test_sort_self_talk_up_on_forward() { let t = dummy_context().await; t.ctx.update_device_chats().await.unwrap(); + create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat") + .await + .unwrap(); let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap(); - assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0)) + assert!(chats.len() == 3); + assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0)) .await .unwrap() - .is_device_talk()); + .is_self_talk()); let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None) .await .unwrap(); + assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0)) .await .unwrap() diff --git a/src/config.rs b/src/config.rs index a39b54bca..45ad152ba 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,9 +4,12 @@ use strum::{EnumProperty, IntoEnumIterator}; use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString}; use crate::blob::BlobObject; +use crate::chat::ChatId; use crate::constants::DC_VERSION_STR; use crate::context::Context; use crate::dc_tools::*; +use crate::events::Event; +use crate::message::MsgId; use crate::mimefactory::RECOMMENDED_FILE_SIZE; use crate::stock::StockMessage; @@ -63,6 +66,25 @@ pub enum Config { #[strum(props(default = "0"))] KeyGenType, + /// Timer in seconds after which the message is deleted from the + /// server. + /// + /// Equals to 0 by default, which means the message is never + /// deleted. + /// + /// Value 1 is treated as "delete at once": messages are deleted + /// immediately, without moving to DeltaChat folder. + #[strum(props(default = "0"))] + DeleteServerAfter, + + /// Timer in seconds after which the message is deleted from the + /// device. + /// + /// Equals to 0 by default, which means the message is never + /// deleted. + #[strum(props(default = "0"))] + DeleteDeviceAfter, + SaveMimeHeaders, ConfiguredAddr, ConfiguredMailServer, @@ -127,6 +149,29 @@ impl Context { self.get_config_int(key).await != 0 } + /// Gets configured "delete_server_after" value. + /// + /// `None` means never delete the message, `Some(0)` means delete + /// at once, `Some(x)` means delete after `x` seconds. + pub async fn get_config_delete_server_after(&self) -> Option { + match self.get_config_int(Config::DeleteServerAfter).await { + 0 => None, + 1 => Some(0), + x => Some(x as i64), + } + } + + /// Gets configured "delete_device_after" value. + /// + /// `None` means never delete the message, `Some(x)` means delete + /// after `x` seconds. + pub async fn get_config_delete_device_after(&self) -> Option { + match self.get_config_int(Config::DeleteDeviceAfter).await { + 0 => None, + x => Some(x as i64), + } + } + /// Set the given config key. /// If `None` is passed as a value the value is cleared and set to the default if there is one. pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> { @@ -174,6 +219,15 @@ impl Context { self.sql.set_raw_config(self, key, val).await } + Config::DeleteDeviceAfter => { + let ret = self.sql.set_raw_config(self, key, value).await; + // Force chatlist reload to delete old messages immediately. + self.call_cb(Event::MsgsChanged { + msg_id: MsgId::new(0), + chat_id: ChatId::new(0), + }); + ret + } _ => self.sql.set_raw_config(self, key, value).await, } } diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 491978c26..040038aea 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -16,7 +16,7 @@ use crate::message::{self, MessageState, MessengerMessage, MsgId}; use crate::mimeparser::*; use crate::param::*; use crate::peerstate::*; -use crate::securejoin::{self, handle_securejoin_handshake}; +use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device}; use crate::stock::StockMessage; use crate::{contact, location}; @@ -196,20 +196,27 @@ pub async fn dc_receive_imf( }; } - // if we delete we don't need to try moving messages - if needs_delete_job && !created_db_entries.is_empty() { - job::add( - context, - Action::DeleteMsgOnImap, - created_db_entries[0].1.to_u32() as i32, - Params::new(), - 0, - ) - .await; - } else { - context - .do_heuristics_moves(server_folder.as_ref(), insert_msg_id) - .await; + // Get user-configured server deletion + let delete_server_after = context.get_config_delete_server_after().await; + + if !created_db_entries.is_empty() { + if needs_delete_job || delete_server_after == Some(0) { + for db_entry in &created_db_entries { + job::add( + context, + Action::DeleteMsgOnImap, + db_entry.1.to_u32() as i32, + Params::new(), + 0, + ) + .await; + } + } else { + // Move message if we don't delete it immediately. + context + .do_heuristics_moves(server_folder.as_ref(), insert_msg_id) + .await; + } } info!( @@ -220,7 +227,7 @@ pub async fn dc_receive_imf( cleanup(context, &create_event_to_send, created_db_entries); mime_parser - .handle_reports(context, from_id, sent_timestamp, &server_folder, server_uid) + .handle_reports(context, from_id, sent_timestamp) .await; Ok(()) @@ -351,11 +358,9 @@ async fn add_parts( }; to_id = DC_CONTACT_ID_SELF; - // handshake messages must be processed _before_ chats are created - // (eg. contacs may be marked as verified) + // handshake may mark contacts as verified and must be processed before chats are created if mime_parser.get(HeaderDef::SecureJoin).is_some() { - // avoid discarding by show_emails setting - msgrmsg = MessengerMessage::Yes; + msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting *chat_id = ChatId::new(0); allow_creation = true; match handle_securejoin_handshake(context, mime_parser, from_id).await { @@ -369,8 +374,7 @@ async fn add_parts( state = MessageState::InSeen; } Ok(securejoin::HandshakeMessage::Propagate) => { - // Message will still be processed as "member - // added" or similar system message. + // process messages as "member added" normally } Err(err) => { *hidden = true; @@ -491,6 +495,27 @@ async fn add_parts( // We cannot recreate other states (read, error). state = MessageState::OutDelivered; to_id = to_ids.get_index(0).cloned().unwrap_or_default(); + + // handshake may mark contacts as verified and must be processed before chats are created + if mime_parser.get(HeaderDef::SecureJoin).is_some() { + msgrmsg = MessengerMessage::Yes; // avoid discarding by show_emails setting + *chat_id = ChatId::new(0); + allow_creation = true; + match observe_securejoin_on_other_device(context, mime_parser, to_id) { + Ok(securejoin::HandshakeMessage::Done) + | Ok(securejoin::HandshakeMessage::Ignore) => { + *hidden = true; + } + Ok(securejoin::HandshakeMessage::Propagate) => { + // process messages as "member added" normally + } + Err(err) => { + *hidden = true; + error!(context, "Error in Secure-Join watching: {}", err); + } + } + } + if !to_ids.is_empty() { if chat_id.is_unset() { let (new_chat_id, new_chat_id_blocked) = create_or_lookup_group( @@ -604,6 +629,7 @@ async fn add_parts( let sent_timestamp = *sent_timestamp; let is_hidden = *hidden; let chat_id = *chat_id; + let is_mdn = !mime_parser.reports.is_empty(); // TODO: can this clone be avoided? let rfc724_mid = rfc724_mid.to_string(); @@ -624,8 +650,11 @@ async fn add_parts( VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);", )?; - if location_kml_is && icnt == 1 && (part.msg == "-location-" || part.msg.is_empty()) - { + let is_location_kml = location_kml_is + && icnt == 1 + && (part.msg == "-location-" || part.msg.is_empty()); + + if is_mdn || is_location_kml { is_hidden = true; if state == MessageState::InFresh { state = MessageState::InNoticed; diff --git a/src/e2ee.rs b/src/e2ee.rs index 1c6c421ce..0cda08fa2 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -127,7 +127,7 @@ pub async fn try_decrypt( ) -> Result<(Option>, HashSet)> { let from = mail .headers - .get_header_value(HeaderDef::From_)? + .get_header_value(HeaderDef::From_) .and_then(|from_addr| mailparse::addrparse(&from_addr).ok()) .and_then(|from| from.extract_single_info()) .map(|from| from.addr) @@ -425,7 +425,6 @@ Sent with my Delta Chat Messenger: https://delta.chat"; } #[async_std::test] - #[ignore] // generating keys is expensive async fn test_generate() { let t = dummy_context().await; let addr = "alice@example.org"; @@ -437,7 +436,6 @@ Sent with my Delta Chat Messenger: https://delta.chat"; } #[async_std::test] - #[ignore] async fn test_generate_concurrent() { use std::sync::Arc; diff --git a/src/headerdef.rs b/src/headerdef.rs index df75adec7..af8343949 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -1,5 +1,5 @@ use crate::strum::AsStaticRef; -use mailparse::{MailHeader, MailHeaderMap, MailParseError}; +use mailparse::{MailHeader, MailHeaderMap}; #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)] #[strum(serialize_all = "kebab_case")] @@ -52,11 +52,11 @@ impl HeaderDef { } pub trait HeaderDefMap { - fn get_header_value(&self, headerdef: HeaderDef) -> Result, MailParseError>; + fn get_header_value(&self, headerdef: HeaderDef) -> Option; } impl HeaderDefMap for [MailHeader<'_>] { - fn get_header_value(&self, headerdef: HeaderDef) -> Result, MailParseError> { + fn get_header_value(&self, headerdef: HeaderDef) -> Option { self.get_first_value(headerdef.get_headername()) } } @@ -79,18 +79,13 @@ mod tests { let (headers, _) = mailparse::parse_headers(b"fRoM: Bob\naUtoCryPt-SeTup-MessAge: v99").unwrap(); assert_eq!( - headers - .get_header_value(HeaderDef::AutocryptSetupMessage) - .unwrap(), + headers.get_header_value(HeaderDef::AutocryptSetupMessage), Some("v99".to_string()) ); assert_eq!( - headers.get_header_value(HeaderDef::From_).unwrap(), + headers.get_header_value(HeaderDef::From_), Some("Bob".to_string()) ); - assert_eq!( - headers.get_header_value(HeaderDef::Autocrypt).unwrap(), - None - ); + assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None); } } diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 86aadc6e9..44bbdda0f 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -823,7 +823,6 @@ impl Imap { folder: &str, uid: u32, dest_folder: &str, - dest_uid: &mut u32, ) -> ImapActionResult { if folder == dest_folder { info!( @@ -839,10 +838,6 @@ impl Imap { return imapresult; } // we are connected, and the folder is selected - - // XXX Rust-Imap provides no target uid on mv, so just set it to 0 - *dest_uid = 0; - let set = format!("{}", uid); let display_folder_id = format!("{}/{}", folder, uid); @@ -1025,10 +1020,10 @@ impl Imap { context: &Context, message_id: &str, folder: &str, - uid: &mut u32, + uid: u32, ) -> ImapActionResult { if let Some(imapresult) = self - .prepare_imap_operation_on_msg(context, folder, *uid) + .prepare_imap_operation_on_msg(context, folder, uid) .await { return imapresult; @@ -1052,7 +1047,7 @@ impl Imap { display_imap_id, message_id, ); - return ImapActionResult::Failed; + return ImapActionResult::AlreadyDone; }; let remote_message_id = get_fetch_headers(&fetch) @@ -1067,26 +1062,26 @@ impl Imap { remote_message_id, message_id, ); - *uid = 0; + return ImapActionResult::Failed; } } Err(err) => { warn!( context, - "Cannot delete {} on IMAP: {}", display_imap_id, err + "Cannot delete on IMAP, {}: {}", display_imap_id, err, ); - *uid = 0; + return ImapActionResult::RetryLater; } } } // mark the message for deletion - if !self.add_flag_finalized(context, *uid, "\\Deleted").await { + if !self.add_flag_finalized(context, uid, "\\Deleted").await { warn!( context, "Cannot mark message {} as \"Deleted\".", display_imap_id ); - ImapActionResult::Failed + ImapActionResult::RetryLater } else { emit_event!( context, @@ -1232,11 +1227,6 @@ impl Imap { .set_raw_config_int(context, "folders_configured", DC_FOLDERS_CONFIGURED_VERSION) .await?; } - context - .sql - .set_raw_config_int(context, "folders_configured", 3) - .await?; - info!(context, "FINISHED configuring IMAP-folders."); Ok(()) } @@ -1262,14 +1252,6 @@ impl Imap { return; } - if !self - .add_flag_finalized_with_set(context, SELECT_ALL, "\\Deleted") - .await - { - error!(context, "Cannot mark messages for deletion {}", folder); - return; - } - // we now trigger expunge to actually delete messages self.config.selected_folder_needs_expunge = true; match self.select_folder::(context, None).await { @@ -1347,20 +1329,55 @@ async fn precheck_imf( message::rfc724_mid_exists(context, &rfc724_mid).await { if old_server_folder.is_empty() && old_server_uid == 0 { - info!(context, "[move] detected bcc-self {}", rfc724_mid,); - context - .do_heuristics_moves(server_folder.as_ref(), msg_id) - .await; - job::add( + info!( context, - Action::MarkseenMsgOnImap, - msg_id.to_u32() as i32, - Params::new(), - 0, - ) - .await; + "[move] detected bcc-self {} as {}/{}", rfc724_mid, server_folder, server_uid + ); + + let delete_server_after = context.get_config_delete_server_after().await; + + if delete_server_after != Some(0) { + context + .do_heuristics_moves(server_folder.as_ref(), msg_id) + .await; + job::add( + context, + Action::MarkseenMsgOnImap, + msg_id.to_u32() as i32, + Params::new(), + 0, + ) + .await; + } } else if old_server_folder != server_folder { - info!(context, "[move] detected moved message {}", rfc724_mid,); + info!( + context, + "[move] detected message {} moved by other device from {}/{} to {}/{}", + rfc724_mid, + old_server_folder, + old_server_uid, + server_folder, + server_uid + ); + } else if old_server_uid == 0 { + info!( + context, + "[move] detected message {} moved by us from {}/{} to {}/{}", + rfc724_mid, + old_server_folder, + old_server_uid, + server_folder, + server_uid + ); + } else if old_server_uid != server_uid { + warn!( + context, + "UID for message {} in folder {} changed from {} to {}", + rfc724_mid, + server_folder, + old_server_uid, + server_uid + ); } if old_server_folder != server_folder || old_server_uid != server_uid { @@ -1382,7 +1399,7 @@ fn get_fetch_headers(prefetch_msg: &Fetch) -> Result> } fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result { - if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId)? { + if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId) { Ok(crate::mimeparser::parse_message_id(&message_id)?) } else { Err(Error::Other("prefetch: No message ID found".to_string())) @@ -1392,20 +1409,20 @@ fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result async fn prefetch_is_reply_to_chat_message( context: &Context, headers: &[mailparse::MailHeader<'_>], -) -> Result { - if let Some(value) = headers.get_header_value(HeaderDef::InReplyTo)? { +) -> bool { + if let Some(value) = headers.get_header_value(HeaderDef::InReplyTo) { if is_msgrmsg_rfc724_mid_in_list(context, &value).await { - return Ok(true); + return true; } } - if let Some(value) = headers.get_header_value(HeaderDef::References)? { + if let Some(value) = headers.get_header_value(HeaderDef::References) { if is_msgrmsg_rfc724_mid_in_list(context, &value).await { - return Ok(true); + return true; } } - Ok(false) + false } async fn prefetch_should_download( @@ -1413,16 +1430,16 @@ async fn prefetch_should_download( headers: &[mailparse::MailHeader<'_>], show_emails: ShowEmails, ) -> Result { - let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion)?.is_some(); - let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await?; + let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some(); + let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers).await; // Autocrypt Setup Message should be shown even if it is from non-chat client. let is_autocrypt_setup_message = headers - .get_header_value(HeaderDef::AutocryptSetupMessage)? + .get_header_value(HeaderDef::AutocryptSetupMessage) .is_some(); let from_field = headers - .get_header_value(HeaderDef::From_)? + .get_header_value(HeaderDef::From_) .unwrap_or_default(); let (_contact_id, blocked_contact, origin) = diff --git a/src/imex.rs b/src/imex.rs index 04a054555..a5a1892b1 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -792,7 +792,6 @@ mod tests { assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n")); assert!(msg.contains("Passphrase-Format: numeric9x4\r\n")); assert!(msg.contains("Passphrase-Begin: he\n")); - assert!(msg.contains("==\n")); assert!(msg.contains("-----END PGP MESSAGE-----\n")); } diff --git a/src/job.rs b/src/job.rs index 6021d1949..f13dd42f7 100644 --- a/src/job.rs +++ b/src/job.rs @@ -78,10 +78,13 @@ pub enum Action { // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999 Housekeeping = 105, // low priority ... EmptyServer = 107, - DeleteMsgOnImap = 110, - MarkseenMdnOnImap = 120, + OldDeleteMsgOnImap = 110, MarkseenMsgOnImap = 130, + + // Moving message is prioritized lower than deletion so we don't + // bother moving message if it is already scheduled for deletion. MoveMsg = 200, + DeleteMsgOnImap = 210, // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999 MaybeSendLocations = 5005, // low priority ... @@ -104,9 +107,9 @@ impl From for Thread { Unknown => Thread::Unknown, Housekeeping => Thread::Imap, + OldDeleteMsgOnImap => Thread::Imap, DeleteMsgOnImap => Thread::Imap, EmptyServer => Thread::Imap, - MarkseenMdnOnImap => Thread::Imap, MarkseenMsgOnImap => Thread::Imap, MoveMsg => Thread::Imap, @@ -417,22 +420,15 @@ impl Job { if let Some(dest_folder) = dest_folder { let server_folder = msg.server_folder.as_ref().unwrap(); - let mut dest_uid = 0; match imap - .mv( - context, - server_folder, - msg.server_uid, - &dest_folder, - &mut dest_uid, - ) + .mv(context, server_folder, msg.server_uid, &dest_folder) .await { ImapActionResult::RetryLater => Status::RetryLater, ImapActionResult::Success => { - message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, dest_uid) - .await; + // XXX Rust-Imap provides no target uid on mv, so just set it to 0 + message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await; Status::Finished(Ok(())) } ImapActionResult::Failed => { @@ -445,11 +441,26 @@ impl Job { } } + /// Deletes a message on the server. + /// + /// foreign_id is a MsgId pointing to a message in the trash chat + /// or a hidden message. + /// + /// This job removes the database record. If there are no more + /// records pointing to the same message on the server, the job + /// also removes the message on the server. async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status { - let mut msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await); + let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await); if !msg.rfc724_mid.is_empty() { - if message::rfc724_mid_cnt(context, &msg.rfc724_mid).await > 1 { + let cnt = message::rfc724_mid_cnt(context, &msg.rfc724_mid).await; + info!( + context, + "Running delete job for message {} which has {} entries in the database", + &msg.rfc724_mid, + cnt + ); + if cnt > 1 { info!( context, "The message is deleted from the server when all parts are deleted.", @@ -459,15 +470,48 @@ impl Job { we delete the message from the server */ let mid = msg.rfc724_mid; let server_folder = msg.server_folder.as_ref().unwrap(); - let res = imap - .delete_msg(context, &mid, server_folder, &mut msg.server_uid) - .await; - if res == ImapActionResult::RetryLater { - // XXX RetryLater is converted to RetryNow here - return Status::RetryNow; + let res = if msg.server_uid == 0 { + // Message is already deleted on IMAP server. + ImapActionResult::AlreadyDone + } else { + imap.delete_msg(context, &mid, server_folder, msg.server_uid) + .await + }; + match res { + ImapActionResult::AlreadyDone | ImapActionResult::Success => {} + ImapActionResult::RetryLater | ImapActionResult::Failed => { + // If job has failed, for example due to some + // IMAP bug, we postpone it instead of failing + // immediately. This will prevent adding it + // immediately again if user has enabled + // automatic message deletion. Without this, + // we might waste a lot of traffic constantly + // retrying message deletion. + return Status::RetryLater; + } } } - Message::delete_from_db(context, msg.id).await; + if msg.chat_id.is_trash() || msg.hidden { + // Messages are stored in trash chat only to keep + // their server UID and Message-ID. Once message is + // deleted from the server, database record can be + // removed as well. + // + // Hidden messages are similar to trashed, but are + // related to some chat. We also delete their + // database records. + job_try!(msg.id.delete_from_db(context).await) + } else { + // Remove server UID from the database record. + // + // We have either just removed the message from the + // server, in which case UID is not valid anymore, or + // we have more refernces to the same server UID, so + // we remove UID to reduce the number of messages + // pointing to the corresponding UID. Once the counter + // reaches zero, we will remove the message. + job_try!(msg.id.unlink(context).await); + } Status::Finished(Ok(())) } else { /* eg. device messages have no Message-ID */ @@ -515,46 +559,6 @@ impl Job { } } } - - async fn markseen_mdn_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status { - let folder = self - .param - .get(Param::ServerFolder) - .unwrap_or_default() - .to_string(); - let uid = self.param.get_int(Param::ServerUid).unwrap_or_default() as u32; - - if imap.set_seen(context, &folder, uid).await == ImapActionResult::RetryLater { - return Status::RetryLater; - } - - if self.param.get_bool(Param::AlsoMove).unwrap_or_default() { - if let Err(err) = imap.ensure_configured_folders(context, true).await { - warn!(context, "configuring folders failed: {:?}", err); - return Status::RetryLater; - } - let dest_folder = context - .sql - .get_raw_config(context, "configured_mvbox_folder") - .await; - if let Some(dest_folder) = dest_folder { - let mut dest_uid = 0; - if ImapActionResult::RetryLater - == imap - .mv(context, &folder, uid, &dest_folder, &mut dest_uid) - .await - { - Status::RetryLater - } else { - Status::Finished(Ok(())) - } - } else { - Status::Finished(Err(format_err!("MVBOX is not configured"))) - } - } else { - Status::Finished(Ok(())) - } - } } /// Delete all pending jobs with the given action. @@ -628,7 +632,11 @@ pub async fn send_msg(context: &Context, msg_id: MsgId) -> Result<()> { .await .unwrap_or_default(); let lowercase_from = from.to_lowercase(); + + // Send BCC to self if it is enabled and we are not going to + // delete it immediately. if context.get_config_bool(Config::BccSelf).await + && context.get_config_delete_server_after().await != Some(0) && !recipients .iter() .any(|x| x.to_lowercase() == lowercase_from) @@ -716,6 +724,45 @@ pub enum Connection<'a> { Smtp(&'a mut Smtp), } +async fn add_imap_deletion_jobs(context: &Context) -> sql::Result<()> { + if let Some(delete_server_after) = context.get_config_delete_server_after().await { + let threshold_timestamp = time() - delete_server_after; + + // Select all expired messages which don't have a + // corresponding message deletion job yet. + let msg_ids = context + .sql + .query_map( + "SELECT id FROM msgs \ + WHERE timestamp < ? \ + AND server_uid != 0 \ + AND NOT EXISTS (SELECT 1 FROM jobs WHERE foreign_id = msgs.id \ + AND action = ?)", + paramsv![threshold_timestamp, Action::DeleteMsgOnImap], + |row| row.get::<_, MsgId>(0), + |ids| { + ids.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + // Schedule IMAP deletion for expired messages. + for msg_id in msg_ids { + add( + context, + Action::DeleteMsgOnImap, + msg_id.to_u32() as i32, + Params::new(), + 0, + ) + .await + } + } + + Ok(()) +} + impl<'a> fmt::Display for Connection<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -811,10 +858,7 @@ async fn perform_job_action( ); let try_res = match job.action { - Action::Unknown => { - warn!(context, "ignoring unknown job"); - Status::Finished(Ok(())) - } + Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))), Action::SendMsgToSmtp => job.send_msg_to_smtp(context, connection.smtp()).await, Action::SendMdn => job.send_mdn(context, connection.smtp()).await, Action::MaybeSendLocations => location::job_maybe_send_locations(context, job).await, @@ -822,9 +866,9 @@ async fn perform_job_action( location::job_maybe_send_locations_ended(context, job).await } Action::EmptyServer => job.empty_server(context, connection.inbox()).await, + Action::OldDeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await, Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await, Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await, - Action::MarkseenMdnOnImap => job.markseen_mdn_on_imap(context, connection.inbox()).await, Action::MoveMsg => job.move_msg(context, connection.inbox()).await, Action::Housekeeping => { sql::housekeeping(context).await; @@ -913,8 +957,8 @@ pub async fn add( Action::Unknown => unreachable!(), Action::Housekeeping | Action::EmptyServer + | Action::OldDeleteMsgOnImap | Action::DeleteMsgOnImap - | Action::MarkseenMdnOnImap | Action::MarkseenMsgOnImap | Action::MoveMsg => { context.interrupt_inbox().await; diff --git a/src/message.rs b/src/message.rs index 9f9ad4631..93c562a1d 100644 --- a/src/message.rs +++ b/src/message.rs @@ -83,6 +83,56 @@ impl MsgId { self.0 == DC_MSG_ID_DAYMARKER } + /// Put message into trash chat and delete message text. + /// + /// It means the message is deleted locally, but not on the server + /// yet. + pub async fn trash(self, context: &Context) -> crate::sql::Result<()> { + let chat_id = ChatId::new(DC_CHAT_ID_TRASH); + context + .sql + .execute( + "UPDATE msgs SET chat_id=?, txt='', txt_raw='' WHERE id=?", + paramsv![chat_id, self], + ) + .await?; + + Ok(()) + } + + /// Deletes a message and corresponding MDNs from the database. + pub async fn delete_from_db(self, context: &Context) -> crate::sql::Result<()> { + // We don't use transactions yet, so remove MDNs first to make + // sure they are not left while the message is deleted. + context + .sql + .execute("DELETE FROM msgs_mdns WHERE msg_id=?;", paramsv![self]) + .await?; + context + .sql + .execute("DELETE FROM msgs WHERE id=?;", paramsv![self]) + .await?; + Ok(()) + } + + /// Removes IMAP server UID and folder from the database record. + /// + /// It is used to avoid trying to remove the message from the + /// server multiple times when there are multiple message records + /// pointing to the same server UID. + pub(crate) async fn unlink(self, context: &Context) -> crate::sql::Result<()> { + context + .sql + .execute( + "UPDATE msgs \ + SET server_folder='', server_uid=0 \ + WHERE id=?", + paramsv![self], + ) + .await?; + Ok(()) + } + /// Bad evil escape hatch. /// /// Avoid using this, eventually types should be cleaned up enough @@ -305,21 +355,6 @@ impl Message { Ok(msg) } - pub async fn delete_from_db(context: &Context, msg_id: MsgId) { - if let Ok(msg) = Message::load_from_db(context, msg_id).await { - context - .sql - .execute("DELETE FROM msgs WHERE id=?;", paramsv![msg.id]) - .await - .ok(); - context - .sql - .execute("DELETE FROM msgs_mdns WHERE msg_id=?;", paramsv![msg.id]) - .await - .ok(); - } - } - pub fn get_filemime(&self) -> Option { if let Some(m) = self.param.get(Param::MimeType) { return Some(m.to_string()); @@ -982,7 +1017,9 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) { delete_poi_location(context, msg.location_id).await; } } - update_msg_chat_id(context, *msg_id, ChatId::new(DC_CHAT_ID_TRASH)).await; + if let Err(err) = msg_id.trash(context).await { + error!(context, "Unable to trash message {}: {}", msg_id, err); + } job::add( context, Action::DeleteMsgOnImap, @@ -1003,17 +1040,6 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) { } } -async fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: ChatId) -> bool { - context - .sql - .execute( - "UPDATE msgs SET chat_id=? WHERE id=?;", - paramsv![chat_id, msg_id], - ) - .await - .is_ok() -} - async fn delete_poi_location(context: &Context, location_id: u32) -> bool { context .sql @@ -1404,12 +1430,64 @@ pub async fn get_deaddrop_msg_cnt(context: &Context) -> usize { } } +pub async fn estimate_deletion_cnt( + context: &Context, + from_server: bool, + seconds: i64, +) -> Result { + let self_chat_id = chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF) + .await + .unwrap_or_default() + .0; + let threshold_timestamp = time() - seconds; + + let cnt: isize = if from_server { + context + .sql + .query_row( + "SELECT COUNT(*) + FROM msgs m + WHERE m.id > ? + AND timestamp < ? + AND chat_id != ? + AND server_uid != 0;", + paramsv![DC_MSG_ID_LAST_SPECIAL, threshold_timestamp, self_chat_id], + |row| row.get(0), + ) + .await? + } else { + context + .sql + .query_row( + "SELECT COUNT(*) + FROM msgs m + WHERE m.id > ? + AND timestamp < ? + AND chat_id != ? + AND chat_id != ? AND hidden = 0;", + paramsv![ + DC_MSG_ID_LAST_SPECIAL, + threshold_timestamp, + self_chat_id, + ChatId::new(DC_CHAT_ID_TRASH) + ], + |row| row.get(0), + ) + .await? + }; + Ok(cnt as usize) +} + +/// Counts number of database records pointing to specified +/// Message-ID. +/// +/// Unlinked messages are excluded. pub async fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> i32 { // check the number of messages with the same rfc724_mid match context .sql .query_row( - "SELECT COUNT(*) FROM msgs WHERE rfc724_mid=?;", + "SELECT COUNT(*) FROM msgs WHERE rfc724_mid=? AND NOT server_uid = 0", paramsv![rfc724_mid], |row| row.get(0), ) @@ -1456,8 +1534,9 @@ pub async fn update_server_uid( match context .sql .execute( - "UPDATE msgs SET server_folder=?, server_uid=? WHERE rfc724_mid=?;", - paramsv![server_folder.as_ref().to_string(), server_uid, rfc724_mid], + "UPDATE msgs SET server_folder=?, server_uid=? \ + WHERE rfc724_mid=?", + paramsv![server_folder.as_ref(), server_uid, rfc724_mid], ) .await { diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 7b1d8388c..69308a86e 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -9,7 +9,6 @@ use mailparse::{DispositionType, MailAddr, MailHeaderMap}; use crate::aheader::Aheader; use crate::bail; use crate::blob::BlobObject; -use crate::config::Config; use crate::constants::Viewtype; use crate::contact::*; use crate::context::Context; @@ -19,7 +18,6 @@ use crate::e2ee; use crate::error::Result; use crate::events::Event; use crate::headerdef::{HeaderDef, HeaderDefMap}; -use crate::job::{self, Action}; use crate::location; use crate::message; use crate::param::*; @@ -87,7 +85,7 @@ impl MimeMessage { let message_time = mail .headers - .get_header_value(HeaderDef::Date)? + .get_header_value(HeaderDef::Date) .and_then(|v| mailparse::dateparse(&v).ok()) .unwrap_or_default(); @@ -113,8 +111,7 @@ impl MimeMessage { // Handle any gossip headers if the mail was encrypted. See section // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf - let gossip_headers = - decrypted_mail.headers.get_all_values("Autocrypt-Gossip")?; + let gossip_headers = decrypted_mail.headers.get_all_values("Autocrypt-Gossip"); gossipped_addr = update_gossip_peerstates(context, message_time, &mail, gossip_headers) .await?; @@ -554,6 +551,16 @@ impl MimeMessage { if let Some(report) = self.process_report(context, mail)? { self.reports.push(report); } + + // Add MDN part so we can track it, avoid + // downloading the message again and + // delete if automatic message deletion is + // enabled. + let mut part = Part::default(); + part.typ = Viewtype::Unknown; + self.parts.push(part); + + any_part_added = true; } else { /* eg. `report-type=delivery-status`; maybe we should show them as a little error icon */ @@ -756,16 +763,13 @@ impl MimeMessage { fn merge_headers(headers: &mut HashMap, fields: &[mailparse::MailHeader<'_>]) { for field in fields { - if let Ok(key) = field.get_key() { - // lowercasing all headers is technically not correct, but makes things work better - let key = key.to_lowercase(); - if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers) + // lowercasing all headers is technically not correct, but makes things work better + let key = field.get_key().to_lowercase(); + if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers) is_known(&key) || key.starts_with("chat-") - { - if let Ok(value) = field.get_value() { - headers.insert(key, value); - } - } + { + let value = field.get_value(); + headers.insert(key.to_string(), value); } } } @@ -780,21 +784,13 @@ impl MimeMessage { let (report_fields, _) = mailparse::parse_headers(&report_body)?; // must be present - if let Some(_disposition) = report_fields - .get_header_value(HeaderDef::Disposition) - .ok() - .flatten() - { + if let Some(_disposition) = report_fields.get_header_value(HeaderDef::Disposition) { if let Some(original_message_id) = report_fields .get_header_value(HeaderDef::OriginalMessageId) - .ok() - .flatten() .and_then(|v| parse_message_id(&v).ok()) { let additional_message_ids = report_fields .get_header_value(HeaderDef::AdditionalMessageIds) - .ok() - .flatten() .map_or_else(Vec::new, |v| { v.split(' ') .filter_map(|s| parse_message_id(s).ok()) @@ -810,26 +806,18 @@ impl MimeMessage { warn!( context, "ignoring unknown disposition-notification, Message-Id: {:?}", - report_fields.get_header_value(HeaderDef::MessageId).ok() + report_fields.get_header_value(HeaderDef::MessageId) ); Ok(None) } /// Handle reports (only MDNs for now) - pub async fn handle_reports( - &self, - context: &Context, - from_id: u32, - sent_timestamp: i64, - server_folder: impl AsRef, - server_uid: u32, - ) { + pub async fn handle_reports(&self, context: &Context, from_id: u32, sent_timestamp: i64) { if self.reports.is_empty() { return; } - let mut mdn_recognized = false; for report in &self.reports { for original_message_id in std::iter::once(&report.original_message_id).chain(&report.additional_message_ids) @@ -839,20 +827,9 @@ impl MimeMessage { .await { context.call_cb(Event::MsgRead { chat_id, msg_id }); - mdn_recognized = true; } } } - - if self.has_chat_version() || mdn_recognized { - let mut param = Params::new(); - param.set(Param::ServerFolder, server_folder.as_ref()); - param.set_int(Param::ServerUid, server_uid as i32); - if self.has_chat_version() && context.get_config_bool(Config::MvboxMove).await { - param.set_int(Param::AlsoMove, 1); - } - job::add(context, Action::MarkseenMdnOnImap, 0, param, 0).await; - } } } @@ -871,14 +848,9 @@ async fn update_gossip_peerstates( if let Ok(ref header) = gossip_header { if recipients.is_none() { - recipients = Some(get_recipients(mail.headers.iter().filter_map(|v| { - let key = v.get_key(); - let value = v.get_value(); - if key.is_err() || value.is_err() { - return None; - } - Some((v.get_key().unwrap(), v.get_value().unwrap())) - }))); + recipients = Some(get_recipients( + mail.headers.iter().map(|v| (v.get_key(), v.get_value())), + )); } if recipients @@ -926,13 +898,8 @@ pub(crate) fn parse_message_id(value: &str) -> crate::error::Result { let ids = mailparse::msgidparse(value) .map_err(|err| format_err!("failed to parse message id {:?}", err))?; - if ids.len() == 1 { - let id = &ids[0]; - if id.starts_with('<') && id.ends_with('>') { - Ok(id.chars().skip(1).take(id.len() - 2).collect()) - } else { - bail!("message-ID {} is not enclosed in < and >", value); - } + if let Some(id) = ids.first() { + Ok(id.to_string()) } else { bail!("could not parse message_id: {}", value); } @@ -998,15 +965,12 @@ fn get_mime_type(mail: &mailparse::ParsedMail<'_>) -> Result<(Mime, Viewtype)> { } fn is_attachment_disposition(mail: &mailparse::ParsedMail<'_>) -> bool { - if let Ok(ct) = mail.get_content_disposition() { - return ct.disposition == DispositionType::Attachment - && ct - .params - .iter() - .any(|(key, _value)| key.starts_with("filename")); - } - - false + let ct = mail.get_content_disposition(); + ct.disposition == DispositionType::Attachment + && ct + .params + .iter() + .any(|(key, _value)| key.starts_with("filename")) } /// Tries to get attachment filename. @@ -1021,7 +985,7 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result = ct .params @@ -1398,7 +1362,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ Some("Chat: Message opened".to_string()) ); - assert_eq!(message.parts.len(), 0); + assert_eq!(message.parts.len(), 1); assert_eq!(message.reports.len(), 1); } @@ -1478,7 +1442,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\ Some("Chat: Message opened".to_string()) ); - assert_eq!(message.parts.len(), 0); + assert_eq!(message.parts.len(), 2); assert_eq!(message.reports.len(), 2); } @@ -1525,7 +1489,7 @@ Additional-Message-IDs: \n\ Some("Chat: Message opened".to_string()) ); - assert_eq!(message.parts.len(), 0); + assert_eq!(message.parts.len(), 1); assert_eq!(message.reports.len(), 1); assert_eq!(message.reports[0].original_message_id, "foo@example.org"); assert_eq!( diff --git a/src/param.rs b/src/param.rs index 2b64cd347..e1e55a394 100644 --- a/src/param.rs +++ b/src/param.rs @@ -88,12 +88,6 @@ pub enum Param { /// For Jobs SetLongitude = b'n', - /// For Jobs - ServerFolder = b'Z', - - /// For Jobs - ServerUid = b'z', - /// For Jobs AlsoMove = b'M', diff --git a/src/pgp.rs b/src/pgp.rs index e4f1534f9..1806b6fcb 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -153,8 +153,8 @@ pub(crate) fn create_keypair( keygen_type: KeyGenType, ) -> std::result::Result { let (secret_key_type, public_key_type) = match keygen_type { - KeyGenType::Rsa2048 | KeyGenType::Default => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)), - KeyGenType::Ed25519 => (PgpKeyType::EdDSA, PgpKeyType::ECDH), + KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)), + KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH), }; let user_id = format!("<{}>", addr); @@ -394,7 +394,6 @@ mod tests { } #[test] - #[ignore] // is too expensive fn test_create_keypair() { let keypair0 = create_keypair( EmailAddress::new("foo@bar.de").unwrap(), diff --git a/src/securejoin.rs b/src/securejoin.rs index 486f70940..38e636503 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -824,9 +824,42 @@ pub(crate) async fn handle_securejoin_handshake( } } +/// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen. +/// currently, the message is only ignored, in the future, +/// we may mark peers as verified accross devices: +/// +/// in a multi-device-setup, there may be other devices that "see" the handshake messages. +/// if the seen messages seen are self-sent messages encrypted+signed correctly with our key, +/// we can make some conclusions of it: +/// +/// - if we see the self-sent-message vg-member-added/vc-contact-confirm, +/// we know that we're an inviter-observer. +/// the inviting device has marked a peer as verified on vg-request-with-auth/vc-request-with-auth +/// before sending vg-member-added/vc-contact-confirm - so, if we observe vg-member-added/vc-contact-confirm, +/// we can mark the peer as verified as well. +/// +/// - if we see the self-sent-message vg-member-added-received +/// we know that we're an joiner-observer. +/// the joining device has marked the peer as verified on vg-member-added/vc-contact-confirm +/// before sending vg-member-added-received - so, if we observe vg-member-added-received, +/// we can mark the peer as verified as well. +/// +/// to make this work, (a) some messages must not be deleted, +/// (b) we need a vc-contact-confirm-received message if bcc_self is set, +/// (c) we should make sure, we do not only rely on the unencrypted To:-header for identifying the peer +/// (in handle_securejoin_handshake() we have the oob information for that) +pub(crate) fn observe_securejoin_on_other_device( + _context: &Context, + _mime_message: &MimeMessage, + _contact_id: u32, +) -> Result { + Ok(HandshakeMessage::Ignore) +} + async fn secure_connection_established(context: &Context, contact_chat_id: ChatId) { let contact_id: u32 = chat_id_2_contact_id(context, contact_chat_id).await; let contact = Contact::get_by_id(context, contact_id).await; + let addr = if let Ok(ref contact) = contact { contact.get_addr() } else { diff --git a/src/sql.rs b/src/sql.rs index 99dab7b06..b7a9a916e 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -11,7 +11,7 @@ use rusqlite::{Connection, Error as SqlError, OpenFlags}; use thread_local_object::ThreadLocal; use crate::chat::{update_device_icon, update_saved_messages_icon}; -use crate::constants::ShowEmails; +use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH}; use crate::context::Context; use crate::dc_tools::*; use crate::param::*; @@ -629,6 +629,13 @@ pub async fn housekeeping(context: &Context) { } } + if let Err(err) = prune_tombstones(context).await { + warn!( + context, + "Houskeeping: Cannot prune message tombstones: {}", err + ); + } + info!(context, "Housekeeping done.",); } @@ -1325,6 +1332,19 @@ async fn open( Ok(()) } +async fn prune_tombstones(context: &Context) -> Result<()> { + context + .sql + .execute( + "DELETE FROM msgs \ + WHERE (chat_id = ? OR hidden) \ + AND server_uid = 0", + paramsv![DC_CHAT_ID_TRASH], + ) + .await?; + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/test-data/key/alice-public.asc b/test-data/key/alice-public.asc index 7ee921bc8..dd9998ad7 100644 --- a/test-data/key/alice-public.asc +++ b/test-data/key/alice-public.asc @@ -1 +1 @@ -xsBNBF425W8BCADLIbltPzG1vk/V2ov2+eBeJJJnRu1kJHdo6e3oNB+HTIxVde5+7Uq8tTEDZB1O7m9NBUFrXr7UYQsA/86G2jmsyWKTzIu1O/t5kdcNDqsNcTVZAhBu2ixYsYVc3ws6kJONjpXLtD2u3P7vEXU3INiOb2JrBQDT8/ubEm1xas/UirYnP5DMaH068IHRdVEYs9ULFaD5scw1m/94buXYZ1CRt/2hT8iRrtBi6ki8kArnhsZC2Xr0+jRQNMUnG5k7Bwi6saCqVmd7IlqSM6MbfYank30Gi/UyDmyIrOk7daTg6WIqgiVOTHav65EK/aUvvjlr+awM+C+u35rQytzyTitZABEBAAHNEzxhbGljZUBleGFtcGxlLmNvbT7CwIkEEAEIADMCGQEFAl425ZQCGwMECwkIBwYVCAkKCwIDFgIBFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HiZQf+PLDxzWchkHAdQFbxxtoXj66aiknofjlRWHDWvUG4nULZ15tjDjnv3z22Meldr8kSV4r1+ejhLFHou9gTzAYk7eAxiybDd8AJOdK+ZgK/Nn7xjdO+HTZLhNdi+R7EektDyf8WDNktEaS8pZc74VKu4984ESi4PoqVxqGHRiSisH4cw4b2pQYxp32BkIdil7sWnqRUEoCpMoKdw2h0N7/lm+rS7/JR9cdjXaVzy1dYTqAVsTL1FTGy4osOKGOyQbkP+Cm6uNq7kC/Bt+fefsb+c2JycmI1uwdvnG7PoFslKv3lRnfkNSmrcIYlJHUl5z0yAgliophr5fqMfzQpO4zMc87ATQReNuVvAQgAuNjE1i+g4v25UNDPIMgXODU4WztE30074gQs5sZa0DQnDUMsdWc2g1o060YZDojMYJQAtBjlW1Dz8FEE7WsLNohGtRyUWmIgNxE5CpodjpwIZ0MdO4Aji0YM+g+WsOSS8kiHMs+dMFfQJuNKjujGFaMIciSaMMrUmPtzkQ/o8NEJs2Aftw90fpVR+M7Mue3++rcEX09ntbgqkgm8SV6OIrOY2kfILudtybocgYkCTeNVqz5VFXuxrnT4ceyFQ64JkwsZxb+X/pCm4V5Q2TbKRwtdonU8HfAz0nAd5tsNeGmf/dPLOKBCxlNEme399YmzWrT+kJBp7CIH5jlWQKyuLwARAQABwsB2BBgBCAAgBQJeNuWUAhsMFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HrEgf/Xu8eRPPdskwtyd98y64teidBpkHuIjuZKJpNyy2HhdGXQwYbNIzwINg0EJ+u2nkreNF/h2Lu+/saqI8Dai02dpYXjvxJIlCgP2os7sNhVaZSaS4XmmJjkHCfZuIKblZypKDJVc5AceZxrtvUbgG+94+H3zeRWVAA30S5ep6YPvxigvhmQah/sdzY7708/jd9uXcCbkP47PBaXCpuPiYLb3t7z8mOteJb7LOZUmSI1efiLDLTGj7ofkdDfA7E6/nF/1+nq+UIDWqljwiUzeNIJsFlZRa/9/uDEjcQbaDe9/knBs7k9pEDZX5u8SSwSED75L+OvRpFWenp4SSKvd2BUw== \ No newline at end of file +mDMEXlh13RYJKwYBBAHaRw8BAQdAzfVIAleCXMJrq8VeLlEVof6ITCviMktKjmcBKAu4m5C0GUFsaWNlIDxhbGljZUBleGFtcGxlLm9yZz6IkAQTFggAOBYhBC5vossjtTLXKGNLWGSwj2Gp7ZRDBQJeWHXdAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEGSwj2Gp7ZRDE3oA/i4MCyDMTsjWqDZoQwX/A/GoTO2/V0wKPhjJJy/8m2pMAPkBjOnGOtx2SZpQvJGTa9h804RY6iDrRuI8A/8tEEXAA7g4BF5Ydd0SCisGAQQBl1UBBQEBB0AG7cjWy2SFAU8KnltlubVW67rFiyfp01JrRe6Xqy22HQMBCAeIeAQYFggAIBYhBC5vossjtTLXKGNLWGSwj2Gp7ZRDBQJeWHXdAhsMAAoJEGSwj2Gp7ZRDLo8BAObE8GnsGVwKzNqCvHeWgJsqhjS3C6gvSlV3tEm9XmF6AQDXucIyVfoBwoyMh2h6cSn/ATn5QJb35pgo+ivp3jsMAg== \ No newline at end of file diff --git a/test-data/key/alice-secret.asc b/test-data/key/alice-secret.asc index b09800bc5..ccc7e0f64 100644 --- a/test-data/key/alice-secret.asc +++ b/test-data/key/alice-secret.asc @@ -1 +1 @@ -xcLYBF425W8BCADLIbltPzG1vk/V2ov2+eBeJJJnRu1kJHdo6e3oNB+HTIxVde5+7Uq8tTEDZB1O7m9NBUFrXr7UYQsA/86G2jmsyWKTzIu1O/t5kdcNDqsNcTVZAhBu2ixYsYVc3ws6kJONjpXLtD2u3P7vEXU3INiOb2JrBQDT8/ubEm1xas/UirYnP5DMaH068IHRdVEYs9ULFaD5scw1m/94buXYZ1CRt/2hT8iRrtBi6ki8kArnhsZC2Xr0+jRQNMUnG5k7Bwi6saCqVmd7IlqSM6MbfYank30Gi/UyDmyIrOk7daTg6WIqgiVOTHav65EK/aUvvjlr+awM+C+u35rQytzyTitZABEBAAEAB/oDQFnwdrd7+jza5nGhFWTS/PDe+FKqbK8AneXx9ouepcoFQCr+Gxw8IwZS0JJrhgOADxp59n1FdvwvGukaXXnY2yxZw0dlMj2XN49ipR51y58X+qF6tMFK9iR1VRif6lqCRIr/RLZMCzuFZhkjNcJhnUTNA7p8qgYX+FaKHzSOaVat/v0kIUHUcZDkREWPUESYDmc1Nv6FXhB0WBiTsBglF+fq5Rm7UWPSmA59Cr7BrW8DctbzTh0+6bkzum2xdOcZ59nuTZa+IKcReI1+kVne5JPNFNJ2tP2f9GSSlL7u+NBtx3zRxZgAotXcJK9cVNIWtegqf+2hoLvm7m2CkWKRBADglpC7TpjV+8wJH+KuyGQ7jepqzf5EHwMrK2i6lPnnmoi0nkKvkklvtdcC7FoFGtLCDJ7vwlUdeN+itDxPlP8bbbUabcy0lLuzyGOVt5NwYXgIuPicpdt2ZTJgvChd9oWi1DG8pVpm+EMJZPyYVEpvDGl6q95oktrytbqjASZbBQQA54roJnwBcptLMTrttDrglULX7ciSKY5HXN1c0rqZn1dTKB1nPYB26hNbu6lZ8ixSOyZm3KwpeDUNW7A3hyzXOfoGFPaddH6WMSFFsGGC/orRVxnuPZLr3UJ3uFX7J0JOav90n/6A4YmS7uImRAG4/vTrAbEfmlBl5msHVUaYh0UD/jSX22JLenO1o8pNU04JQl3lQ4mWY6MvgTyCvpchTzDDva+wdOBTUeVUmb/KqYkYBq98tXl1VnGnNpeEymUISSi60RjaXDhbg7a3ELV0yvvWcBN9zreyyINuCU5OmNefPRvPt4Co12KtIxPACByFNTevzPKbrXd1cyhHOxAuqfzLRbDNEzxhbGljZUBleGFtcGxlLmNvbT7CwIkEEAEIADMCGQEFAl425ZQCGwMECwkIBwYVCAkKCwIDFgIBFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HiZQf+PLDxzWchkHAdQFbxxtoXj66aiknofjlRWHDWvUG4nULZ15tjDjnv3z22Meldr8kSV4r1+ejhLFHou9gTzAYk7eAxiybDd8AJOdK+ZgK/Nn7xjdO+HTZLhNdi+R7EektDyf8WDNktEaS8pZc74VKu4984ESi4PoqVxqGHRiSisH4cw4b2pQYxp32BkIdil7sWnqRUEoCpMoKdw2h0N7/lm+rS7/JR9cdjXaVzy1dYTqAVsTL1FTGy4osOKGOyQbkP+Cm6uNq7kC/Bt+fefsb+c2JycmI1uwdvnG7PoFslKv3lRnfkNSmrcIYlJHUl5z0yAgliophr5fqMfzQpO4zMc8fC2AReNuVvAQgAuNjE1i+g4v25UNDPIMgXODU4WztE30074gQs5sZa0DQnDUMsdWc2g1o060YZDojMYJQAtBjlW1Dz8FEE7WsLNohGtRyUWmIgNxE5CpodjpwIZ0MdO4Aji0YM+g+WsOSS8kiHMs+dMFfQJuNKjujGFaMIciSaMMrUmPtzkQ/o8NEJs2Aftw90fpVR+M7Mue3++rcEX09ntbgqkgm8SV6OIrOY2kfILudtybocgYkCTeNVqz5VFXuxrnT4ceyFQ64JkwsZxb+X/pCm4V5Q2TbKRwtdonU8HfAz0nAd5tsNeGmf/dPLOKBCxlNEme399YmzWrT+kJBp7CIH5jlWQKyuLwARAQABAAf/YmpfWp5fLZvjJ8kVDqIZ4r5LNB+5Sp7nbC3G7lPblBDAXgpOyG9ckdDcbguTWa6yChWizkCXFOhkCKZKVlHw1Wb3JoSB5CFsf4U29pMZe41N2BTeoohV5Fg2nojgNWxtZHwDJ6VsTonidGH9l1sN5AU6gPNF+QZ07MKsRCbRYi0yMgX064gwZXRtkm8AECz8ay1wDzoBy14ALe9aDClafVwfxdYUcxDBqtvjLhGeTWX5lMMAQ1Ix8D0Gp4r0Zvtl+oxlTSZFAt9m6sbRBbJf4LJjRQh07aWF2gUOiyIyz7YymYdwsyFnCPn2Aj84uRdqYCekAUfzBeNTBukUQq1DYQQA3BeH38pnr34m0UyD/tCrTvrX60MOJVvFuaTQw+IgY4XmT9UiiiqYMaoLfzxeevdMCQ9EtMdXUTjI27/II3dR5Obg6J0QTybj78IKPbH8Vdlg0etllRjC3bV/M4a5UcXPKG6W5CvB0UJg7eqn/8wUqwiL9x+hZoLy+nU5rCAjzZEEANcBK8Vy9eBxkKmEfH/mChDSKE82ua0xdQuZiTvvGedUYG3ucH4rAlkZaZcZrtJTod1BKhAhDBrjxk/yLCjK3z5JDsacdDGGfaqga3zdPBJubWE7f4mg6uYVs04Uf90YVY7t0LEQAh7i9QYiIqUOJDy3L8y3+bNgNz2r1p8pFd+/A/0YoYE0YDgABbLKmBQFoWjF3Op7P+k6Z4ENK4Me1fkNSAU451QX7ZduI7i3pGTM06bXG2umhTI1lg48ZveMRk1vBezHU+ThnciEkuhYafnq7NRdkEtI20MyFmN7dZF8LQ/joYKsJbeSG5svj8f1ue2eHkiIIlTtDqVUTizDU3ddlzUZwsB2BBgBCAAgBQJeNuWUAhsMFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HrEgf/Xu8eRPPdskwtyd98y64teidBpkHuIjuZKJpNyy2HhdGXQwYbNIzwINg0EJ+u2nkreNF/h2Lu+/saqI8Dai02dpYXjvxJIlCgP2os7sNhVaZSaS4XmmJjkHCfZuIKblZypKDJVc5AceZxrtvUbgG+94+H3zeRWVAA30S5ep6YPvxigvhmQah/sdzY7708/jd9uXcCbkP47PBaXCpuPiYLb3t7z8mOteJb7LOZUmSI1efiLDLTGj7ofkdDfA7E6/nF/1+nq+UIDWqljwiUzeNIJsFlZRa/9/uDEjcQbaDe9/knBs7k9pEDZX5u8SSwSED75L+OvRpFWenp4SSKvd2BUw== \ No newline at end of file +lFgEXlh13RYJKwYBBAHaRw8BAQdAzfVIAleCXMJrq8VeLlEVof6ITCviMktKjmcBKAu4m5AAAQDMpCY4sD5/DUR0jRjGC5WstwShz1q+5Vofo5mY9+XRXRA3tBlBbGljZSA8YWxpY2VAZXhhbXBsZS5vcmc+iJAEExYIADgWIQQub6LLI7Uy1yhjS1hksI9hqe2UQwUCXlh13QIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBksI9hqe2UQxN6AP4uDAsgzE7I1qg2aEMF/wPxqEztv1dMCj4YyScv/JtqTAD5AYzpxjrcdkmaULyRk2vYfNOEWOog60biPAP/LRBFwAOcXQReWHXdEgorBgEEAZdVAQUBAQdABu3I1stkhQFPCp5bZbm1Vuu6xYsn6dNSa0Xul6stth0DAQgHAAD/X9y9I/JFBeArkgR3U363cWXXxMCWftS+BDwM9zE4PrgQb4h4BBgWCAAgFiEELm+iyyO1MtcoY0tYZLCPYantlEMFAl5Ydd0CGwwACgkQZLCPYantlEMujwEA5sTwaewZXArM2oK8d5aAmyqGNLcLqC9KVXe0Sb1eYXoBANe5wjJV+gHCjIyHaHpxKf8BOflAlvfmmCj6K+neOwwC \ No newline at end of file diff --git a/tests/stress.rs b/tests/stress.rs index 2e70ae799..ee00e713f 100644 --- a/tests/stress.rs +++ b/tests/stress.rs @@ -46,9 +46,9 @@ async fn stress_functions(context: &Context) { // assert!(dc_is_configured(context) != 0, "Missing configured context"); // let setupcode = dc_create_setup_code(context); - // let setupcode_c = CString::yolo(setupcode.clone()); + // let setupcode_c = CString::new(setupcode.clone()).unwrap(); // let setupfile = dc_render_setup_file(context, &setupcode).unwrap(); - // let setupfile_c = CString::yolo(setupfile); + // let setupfile_c = CString::new(setupfile).unwrap(); // let mut headerline_2: *const libc::c_char = ptr::null(); // let payload = dc_decrypt_setup_file(context, setupcode_c.as_ptr(), setupfile_c.as_ptr());