diff --git a/Cargo.toml b/Cargo.toml index d1e739b69..3c3fa5e03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ proptest = "0.10" async-std = { version = "1.6.4", features = ["unstable", "attributes"] } futures-lite = "1.7.0" criterion = "0.3" +ansi_term = "0.12.0" [workspace] members = [ diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index f6622376f..08df2eb39 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -579,12 +579,12 @@ class Account(object): raise ValueError("account not configured, cannot start io") lib.dc_start_io(self._dc_context) - def configure(self): + def configure(self, reconfigure=False): """ Start configuration process and return a Configtracker instance on which you can block with wait_finish() to get a True/False success value for the configuration process. """ - assert not self.is_configured() + assert self.is_configured() == reconfigure if not self.get_config("addr") or not self.get_config("mail_pw"): raise MissingCredentials("addr or mail_pwd not set in config") configtracker = ConfigureTracker(self) diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index dcc01097a..3462169f7 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -9,6 +9,7 @@ import ssl import pathlib from imapclient import IMAPClient from imapclient.exceptions import IMAPClientError +import imaplib import deltachat from deltachat import const @@ -25,13 +26,29 @@ def dc_account_extra_configure(account): """ Reset the account (we reuse accounts across tests) and make 'account.direct_imap' available for direct IMAP ops. """ - if not hasattr(account, "direct_imap"): - imap = DirectImap(account) - if imap.select_config_folder("mvbox"): - imap.delete(ALL, expunge=True) - assert imap.select_config_folder("inbox") - imap.delete(ALL, expunge=True) - setattr(account, "direct_imap", imap) + try: + + if not hasattr(account, "direct_imap"): + imap = DirectImap(account) + + for folder in imap.list_folders(): + if folder.lower() == "inbox" or folder.lower() == "deltachat": + assert imap.select_folder(folder) + imap.delete(ALL, expunge=True) + else: + imap.conn.delete_folder(folder) + # We just deleted the folder, so we have to make DC forget about it, too + if account.get_config("configured_sentbox_folder") == folder: + account.set_config("configured_sentbox_folder", None) + if account.get_config("configured_spam_folder") == folder: + account.set_config("configured_spam_folder", None) + + setattr(account, "direct_imap", imap) + + except Exception as e: + # Uncaught exceptions here would lead to a timeout without any note written to the log + account.log("=============================== CAN'T RESET ACCOUNT: ===============================") + account.log("===================", e, "===================") @deltachat.global_hookimpl @@ -90,6 +107,12 @@ class DirectImap: except (OSError, IMAPClientError): print("Could not logout direct_imap conn") + def create_folder(self, foldername): + try: + self.conn.create_folder(foldername) + except imaplib.IMAP4.error as e: + print("Can't create", foldername, "probably it already exists:", str(e)) + def select_folder(self, foldername): assert not self._idling return self.conn.select_folder(foldername) @@ -240,3 +263,9 @@ class DirectImap: res = self.conn.idle_done() self._idling = False return res + + def append(self, folder, msg): + if msg.startswith("\n"): + msg = msg[1:] + msg = '\n'.join([s.lstrip() for s in msg.splitlines()]) + self.conn.append(folder, msg) diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index bbd7b8007..6426a40ce 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -359,11 +359,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): def wait_configure_and_start_io(self): started_accounts = [] for acc in self._accounts: - if hasattr(acc, "_configtracker"): - acc._configtracker.wait_finish() - acc._evtracker.consume_events() - acc.get_device_chat().mark_noticed() - del acc._configtracker + self.wait_configure(acc) acc.set_config("bcc_self", "0") if acc.is_configured() and not acc.is_started(): acc.start_io() @@ -373,6 +369,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): for acc in started_accounts: acc._evtracker.wait_all_initial_fetches() + def wait_configure(self, acc): + if hasattr(acc, "_configtracker"): + acc._configtracker.wait_finish() + acc._evtracker.consume_events() + acc.get_device_chat().mark_noticed() + del acc._configtracker + def run_bot_process(self, module, ffi=True): fn = module.__file__ diff --git a/python/tests/test_account.py b/python/tests/test_account.py index cb54eddd1..5d58e43bc 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -1136,6 +1136,51 @@ class TestOnlineAccount: assert not device_chat.can_send() assert device_chat.get_draft() is None + def test_dont_show_emails_in_draft_folder(self, acfactory): + """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them. + So: If there is no Received header AND it's not in the sentbox, then ignore the email.""" + ac1 = acfactory.get_online_configuring_account() + ac1.set_config("show_emails", "2") + + acfactory.wait_configure(ac1) + ac1.direct_imap.create_folder("Drafts") + ac1.direct_imap.create_folder("Sent") + + acfactory.wait_configure_and_start_io() + # Wait until each folder was selected once and we are IDLEing again: + ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state") + ac1.stop_io() + + ac1.direct_imap.append("Drafts", """ + From: Bob + Subject: subj + To: alice@example.com + Message-ID: + Content-Type: text/plain; charset=utf-8 + + message in Drafts + """) + ac1.direct_imap.append("Sent", """ + From: Bob + Subject: subj + To: alice@example.com + Message-ID: + Content-Type: text/plain; charset=utf-8 + + message in Sent + """) + + ac1.set_config("scan_all_folders_debounce_secs", "0") + ac1.start_io() + + msg = ac1._evtracker.wait_next_messages_changed() + + # Wait until each folder was scanned, this is necessary for this test to test what it should test: + ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state") + + assert msg.text == "subj – message in Sent" + assert len(msg.chat.get_messages()) == 1 + def test_prefer_encrypt(self, acfactory, lp): """Test quorum rule for encryption preference in 1:1 and group chat.""" ac1, ac2, ac3 = acfactory.get_many_online_accounts(3) @@ -2088,26 +2133,82 @@ class TestOnlineAccount: assert received_reply.quoted_text == "hello" assert received_reply.quote.id == out_msg.id + @pytest.mark.parametrize("folder,move,expected_destination,", [ + ("xyz", False, "xyz"), # Test that emails are recognized in a random folder but not moved + ("xyz", True, "DeltaChat"), # ...emails are found in a random folder and moved to DeltaChat + ("Spam", False, "INBOX") # ...emails are moved from the spam folder to the Inbox + ]) + # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with + # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag. + def test_scan_folders(self, acfactory, lp, folder, move, expected_destination): + """Delta Chat periodically scans all folders for new messages to make sure we don't miss any.""" + variant = folder + "-" + str(move) + "-" + expected_destination + lp.sec("Testing variant " + variant) + ac1 = acfactory.get_online_configuring_account(move=move) + ac2 = acfactory.get_online_configuring_account() + + acfactory.wait_configure(ac1) + ac1.direct_imap.create_folder(folder) + + acfactory.wait_configure_and_start_io() + # Wait until each folder was selected once and we are IDLEing: + ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state") + ac1.stop_io() + + # Send a message to ac1 and move it to the mvbox: + ac1.direct_imap.select_config_folder("inbox") + ac1.direct_imap.idle_start() + acfactory.get_accepted_chat(ac2, ac1).send_text("hello") + ac1.direct_imap.idle_check(terminate=True) + ac1.direct_imap.conn.move(["*"], folder) # "*" means "biggest UID in mailbox" + + lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")") + ac1.set_config("scan_all_folders_debounce_secs", "0") + ac1.start_io() + msg = ac1._evtracker.wait_next_incoming_message() + assert msg.text == "hello" + + # Wait until the message was moved (if at all) and we are IDLEing again: + ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state") + ac1.direct_imap.select_folder(expected_destination) + assert len(ac1.direct_imap.get_all_messages()) == 1 + if folder != expected_destination: + ac1.direct_imap.select_folder(folder) + assert len(ac1.direct_imap.get_all_messages()) == 0 + @pytest.mark.parametrize("mvbox_move", [False, True]) def test_fetch_existing(self, acfactory, lp, mvbox_move): """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts. This way, we can already offer them some email addresses they can write to. - Also test that existing emails are fetched during onboarding. + Also, the newest existing emails from each folder are fetched during onboarding. - Lastly, tests that bcc_self messages moved to the mvbox are marked as read.""" + Additionally tests that bcc_self messages moved to the mvbox are marked as read.""" ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move) ac2 = acfactory.get_online_configuring_account() + acfactory.wait_configure(ac1) + + ac1.direct_imap.create_folder("Sent") + ac1.set_config("sentbox_watch", "1") + + # We need to reconfigure to find the new "Sent" folder. + # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked, + # would also find the "Sent" folder, but it would be too late: + # The sentbox thread, started by `start_io()`, would have seen that there is no + # ConfiguredSentboxFolder and do nothing. + ac1._configtracker = ac1.configure(reconfigure=True) acfactory.wait_configure_and_start_io() - chat = acfactory.get_accepted_chat(ac1, ac2) - - lp.sec("send out message with bcc to ourselves") if mvbox_move: ac1.direct_imap.select_config_folder("mvbox") + else: + ac1.direct_imap.select_config_folder("sentbox") ac1.direct_imap.idle_start() + + lp.sec("send out message with bcc to ourselves") ac1.set_config("bcc_self", "1") + chat = acfactory.get_accepted_chat(ac1, ac2) chat.send_text("message text") # now wait until the bcc_self message arrives diff --git a/src/chat.rs b/src/chat.rs index 253327305..b902fef2f 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -3580,7 +3580,7 @@ mod tests { .unwrap(); add_info_msg(&t, chat_id, "foo info").await; - let msg = t.get_last_msg(chat_id).await; + let msg = t.get_last_msg_in(chat_id).await; assert_eq!(msg.get_chat_id(), chat_id); assert_eq!(msg.get_viewtype(), Viewtype::Text); assert_eq!(msg.get_text().unwrap(), "foo info"); @@ -3610,7 +3610,7 @@ mod tests { assert!(msg.is_info()); assert_eq!(msg.get_info_type(), SystemMessage::EphemeralTimerChanged); - let msg2 = t.get_last_msg(chat_id).await; + let msg2 = t.get_last_msg_in(chat_id).await; assert_eq!(msg.get_id(), msg2.get_id()); } @@ -3637,7 +3637,7 @@ mod tests { let msgs = get_chat_msgs(&t, chat_id, 0, None).await; assert_eq!(msgs.len(), 1); - let msg = t.get_last_msg(chat_id).await; + let msg = t.get_last_msg_in(chat_id).await; assert!(msg.is_info()); assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); assert_eq!(msg.get_state(), MessageState::InNoticed); @@ -3652,7 +3652,7 @@ mod tests { assert!(!chat.is_protected()); assert!(chat.is_unpromoted()); - let msg = t.get_last_msg(chat_id).await; + let msg = t.get_last_msg_in(chat_id).await; assert!(msg.is_info()); assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionDisabled); assert_eq!(msg.get_state(), MessageState::InNoticed); @@ -3676,7 +3676,7 @@ mod tests { assert!(chat.is_protected()); assert!(!chat.is_unpromoted()); - let msg = t.get_last_msg(chat_id).await; + let msg = t.get_last_msg_in(chat_id).await; assert!(msg.is_info()); assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); assert_eq!(msg.get_state(), MessageState::OutDelivered); // as bcc-self is disabled and there is nobody else in the chat diff --git a/src/config.rs b/src/config.rs index eb62f5be8..929530376 100644 --- a/src/config.rs +++ b/src/config.rs @@ -116,6 +116,8 @@ pub enum Config { ConfiguredInboxFolder, ConfiguredMvboxFolder, ConfiguredSentboxFolder, + ConfiguredSpamFolder, + ConfiguredTimestamp, ConfiguredProvider, Configured, @@ -140,6 +142,10 @@ pub enum Config { /// Timestamp of the last time housekeeping was run LastHousekeeping, + + /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely. + #[strum(props(default = "60"))] + ScanAllFoldersDebounceSecs, } impl Context { @@ -186,6 +192,13 @@ impl Context { .unwrap_or_default() } + pub async fn get_config_u64(&self, key: Config) -> u64 { + self.get_config(key) + .await + .and_then(|s| s.parse().ok()) + .unwrap_or_default() + } + pub async fn get_config_bool(&self, key: Config) -> bool { self.get_config_int(key).await != 0 } @@ -263,16 +276,6 @@ impl Context { job::schedule_resync(self).await; ret } - Config::InboxWatch => { - if self.get_config(Config::InboxWatch).await.as_deref() != value { - // If Inbox-watch is disabled and enabled again, do not fetch emails from in between. - // this avoids unexpected mass-downloads and -deletions (if delete_server_after is set) - if let Some(inbox) = self.get_config(Config::ConfiguredInboxFolder).await { - crate::imap::set_config_last_seen_uid(self, inbox, 0, 0).await; - } - } - self.sql.set_raw_config(self, key, value).await - } _ => self.sql.set_raw_config(self, key, value).await, } } diff --git a/src/configure/mod.rs b/src/configure/mod.rs index 57a268a3d..04a95391c 100644 --- a/src/configure/mod.rs +++ b/src/configure/mod.rs @@ -12,7 +12,6 @@ use itertools::Itertools; use job::Action; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; -use crate::config::Config; use crate::dc_tools::EmailAddress; use crate::imap::Imap; use crate::login_param::{LoginParam, ServerLoginParam}; @@ -22,6 +21,7 @@ use crate::provider::{Protocol, Socket, UsernamePattern}; use crate::smtp::Smtp; use crate::stock::StockMessage; use crate::{chat, e2ee, provider}; +use crate::{config::Config, dc_tools::time}; use crate::{ constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2}, job, @@ -413,6 +413,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> { // the trailing underscore is correct param.save_to_database(ctx, "configured_").await?; ctx.sql.set_raw_config_bool(ctx, "configured", true).await?; + ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string())) + .await?; progress!(ctx, 920); diff --git a/src/context.rs b/src/context.rs index 5edd23393..bddcf2d25 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,8 +1,11 @@ //! Context module -use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; use std::ops::Deref; +use std::{ + collections::{BTreeMap, HashMap}, + time::Instant, +}; use async_std::path::{Path, PathBuf}; use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender}; @@ -59,6 +62,8 @@ pub struct InnerContext { pub(crate) scheduler: RwLock, pub(crate) ephemeral_task: RwLock>>, + pub(crate) last_full_folder_scan: Mutex>, + /// Id for this context on the current device. pub(crate) id: u32, @@ -131,6 +136,7 @@ impl Context { scheduler: RwLock::new(Scheduler::Stopped), ephemeral_task: RwLock::new(None), creation_time: std::time::SystemTime::now(), + last_full_folder_scan: Mutex::new(None), }; let ctx = Context { @@ -465,6 +471,11 @@ impl Context { == Some(folder_name.as_ref().to_string()) } + pub async fn is_spam_folder(&self, folder_name: impl AsRef) -> bool { + self.get_config(Config::ConfiguredSpamFolder).await + == Some(folder_name.as_ref().to_string()) + } + pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf { let mut blob_fname = OsString::new(); blob_fname.push(dbfile.file_name().unwrap_or_default()); diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 53557d857..585976b4b 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -251,6 +251,7 @@ pub(crate) async fn dc_receive_imf_inner( .needs_move(context, server_folder.as_ref()) .await .unwrap_or_default() + .is_some() { // Move message if we don't delete it immediately. job::add( @@ -406,6 +407,14 @@ async fn add_parts( } } + if !context.is_sentbox(&server_folder).await && mime_parser.get(HeaderDef::Received).is_none() { + // Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them + // So: If there is no Received header AND it's not in the sentbox, then ignore the email. + info!(context, "Email is probably just a draft (TRASH)"); + *chat_id = ChatId::new(DC_CHAT_ID_TRASH); + allow_creation = false; + } + // check if the message introduces a new chat: // - outgoing messages introduce a chat with the first to: address if they are sent by a messenger // - incoming messages introduce a chat only for known contacts if they are sent by a messenger @@ -1983,7 +1992,8 @@ mod tests { #[async_std::test] async fn test_grpid_simple() { let context = TestContext::new().await; - let raw = b"From: hello\n\ + let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: hello\n\ Subject: outer-subject\n\ In-Reply-To: \n\ References: \n\ @@ -2000,7 +2010,8 @@ mod tests { #[async_std::test] async fn test_grpid_from_multiple() { let context = TestContext::new().await; - let raw = b"From: hello\n\ + let raw = b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: hello\n\ Subject: outer-subject\n\ In-Reply-To: \n\ References: , \n\ @@ -2034,7 +2045,9 @@ mod tests { ); } - static MSGRMSG: &[u8] = b"From: Bob \n\ + static MSGRMSG: &[u8] = + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Bob \n\ To: alice@example.com\n\ Chat-Version: 1.0\n\ Subject: Chat: hello\n\ @@ -2043,7 +2056,9 @@ mod tests { \n\ hello\n"; - static ONETOONE_NOREPLY_MAIL: &[u8] = b"From: Bob \n\ + static ONETOONE_NOREPLY_MAIL: &[u8] = + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Bob \n\ To: alice@example.com\n\ Subject: Chat: hello\n\ Message-ID: <2222@example.com>\n\ @@ -2051,7 +2066,9 @@ mod tests { \n\ hello\n"; - static GRP_MAIL: &[u8] = b"From: bob@example.com\n\ + static GRP_MAIL: &[u8] = + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: bob@example.com\n\ To: alice@example.com, claire@example.com\n\ Subject: group with Alice, Bob and Claire\n\ Message-ID: <3333@example.com>\n\ @@ -2218,7 +2235,8 @@ mod tests { dc_receive_imf( &t, format!( - "From: alice@example.com\n\ + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: alice@example.com\n\ To: bob@example.com\n\ Subject: foo\n\ Message-ID: \n\ @@ -2256,7 +2274,8 @@ mod tests { dc_receive_imf( &t, format!( - "From: bob@example.com\n\ + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: bob@example.com\n\ To: alice@example.com\n\ Subject: message opened\n\ Date: Sun, 22 Mar 2020 23:37:57 +0000\n\ @@ -2320,7 +2339,8 @@ mod tests { dc_receive_imf( context, - b"To: bob@example.com\n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + To: bob@example.com\n\ Subject: foo\n\ Message-ID: <3924@example.com>\n\ Chat-Version: 1.0\n\ @@ -2348,7 +2368,8 @@ mod tests { let chat_id = chat::create_by_contact_id(&t, contact_id).await.unwrap(); dc_receive_imf( &t, - b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= \n\ To: alice@example.com\n\ Subject: foo\n\ Message-ID: \n\ @@ -2396,7 +2417,8 @@ mod tests { dc_receive_imf( &t, - b"From: Foobar \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Foobar \n\ To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.com\n\ Cc: =?utf-8?q?=3Ch2=3E?= \n\ Subject: foo\n\ @@ -2444,7 +2466,8 @@ mod tests { dc_receive_imf( &t, - b"From: Foobar \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Foobar \n\ To: alice@example.com\n\ Cc: Carl \n\ Subject: foo\n\ @@ -2555,7 +2578,8 @@ mod tests { dc_receive_imf( &t, format!( - "From: {}\n\ + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: {}\n\ To: {}\n\ Subject: foo\n\ Message-ID: <{}>\n\ @@ -2601,7 +2625,8 @@ mod tests { dc_receive_imf( &t, - b"From: alice@gmail.com\n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: alice@gmail.com\n\ To: bob@example.com, assidhfaaspocwaeofi@gmail.com\n\ Subject: foo\n\ Message-ID: \n\ diff --git a/src/ephemeral.rs b/src/ephemeral.rs index f2740065d..68d2073ed 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -601,7 +601,7 @@ mod tests { t.send_text(chat.id, "Saved message, which we delete manually") .await; - let msg = t.get_last_msg(chat.id).await; + let msg = t.get_last_msg_in(chat.id).await; msg.id.delete_from_db(&t).await.unwrap(); check_msg_was_deleted(&t, &chat, msg.id).await; diff --git a/src/headerdef.rs b/src/headerdef.rs index f349df5b9..e3b9dcd43 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -44,6 +44,7 @@ pub enum HeaderDef { SecureJoinInvitenumber, SecureJoinAuth, EphemeralTimer, + Received, _TestHeader, } diff --git a/src/imap/idle.rs b/src/imap/idle.rs index cd4f3a1f6..a7914bd1d 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -58,6 +58,12 @@ impl Imap { return Ok(info); } + if let Ok(info) = self.idle_interrupt.try_recv() { + info!(context, "skip idle, got interrupt {:?}", info); + self.session = Some(session); + return Ok(info); + } + let mut handle = session.idle(); if let Err(err) = handle.init().await { bail!("IMAP IDLE protocol failed to init/complete: {}", err); @@ -71,14 +77,18 @@ impl Imap { Interrupt(InterruptInfo), } - info!(context, "Idle entering wait-on-remote state"); + info!( + context, + "{}: Idle entering wait-on-remote state", + watch_folder.as_deref().unwrap_or("None") + ); let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async { - let probe_network = self.idle_interrupt.recv().await; + let info = self.idle_interrupt.recv().await; // cancel imap idle connection properly drop(interrupt); - Ok(Event::Interrupt(probe_network.unwrap_or_default())) + Ok(Event::Interrupt(info.unwrap_or_default())) }); match fut.await { @@ -178,7 +188,7 @@ impl Imap { } } Err(err) => { - error!(context, "could not fetch from folder: {}", err); + error!(context, "could not fetch from folder: {:#}", err); self.trigger_reconnect() } } diff --git a/src/imap/mod.rs b/src/imap/mod.rs index b906bf4f8..d5aeb8d7f 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -3,7 +3,7 @@ //! uses [async-email/async-imap](https://github.com/async-email/async-imap) //! to implement connect, fetch, delete functionality with standard IMAP servers. -use std::{cmp, collections::BTreeMap}; +use std::{cmp, cmp::max, collections::BTreeMap}; use anyhow::Context as _; use async_imap::{ @@ -37,6 +37,7 @@ use crate::{config::Config, dc_receive_imf::dc_receive_imf_inner}; mod client; mod idle; +pub mod scan_folders; pub mod select_folder; mod session; @@ -46,6 +47,8 @@ use mailparse::SingleInfo; use message::Message; use session::Session; +use self::select_folder::NewlySelected; + #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)] pub enum ImapActionResult { Failed, @@ -103,6 +106,7 @@ impl async_imap::Authenticator for OAuth2 { #[derive(Debug, PartialEq)] enum FolderMeaning { Unknown, + Spam, SentObjects, Other, } @@ -534,98 +538,100 @@ impl Imap { Ok(()) } - /// return Result with (uid_validity, last_seen_uid) tuple. + /// Select a folder and take care of uidvalidity changes. + /// Also, when selecting a folder for the first time, sets the uid_next to the current + /// mailbox.uid_next so that no old emails are fetched. + /// Returns Result (i.e. whether new emails arrived), + /// if in doubt, returns new_emails=true so emails are fetched. pub(crate) async fn select_with_uidvalidity( &mut self, context: &Context, folder: &str, - ) -> Result<(u32, u32)> { - self.select_folder(context, Some(folder)).await?; + ) -> Result { + let newly_selected = self.select_folder(context, Some(folder)).await?; - // compare last seen UIDVALIDITY against the current one - let (uid_validity, last_seen_uid) = get_config_last_seen_uid(context, &folder).await; + let mailbox = &mut self.config.selected_mailbox.as_ref(); + let mailbox = + mailbox.with_context(|| format!("No mailbox selected, folder: {}", folder))?; - let config = &mut self.config; - let mailbox = config - .selected_mailbox - .as_ref() - .ok_or_else(|| format_err!("No mailbox selected, folder: {}", folder))?; + let new_uid_validity = mailbox + .uid_validity + .with_context(|| format!("No UIDVALIDITY for folder {}", folder))?; - let new_uid_validity = match mailbox.uid_validity { - Some(v) => v, - None => { - bail!("No UIDVALIDITY for folder {:?}", folder); - } - }; + let old_uid_validity = get_uidvalidity(context, folder).await; + let old_uid_next = get_uid_next(context, folder).await; - if new_uid_validity == uid_validity { - return Ok((uid_validity, last_seen_uid)); + if new_uid_validity == old_uid_validity { + let new_emails = if newly_selected == NewlySelected::No { + // The folder was not newly selected i.e. no SELECT command was run. This means that mailbox.uid_next + // was not updated and may contain an incorrect value. So, just return true so that + // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch + // new messages is only one command, just as a SELECT command) + true + } else if let Some(uid_next) = mailbox.uid_next { + uid_next != old_uid_next // If uid_next changed, there are new emails + } else { + true // We have no uid_next and if in doubt, return true + }; + return Ok(new_emails); } if mailbox.exists == 0 { info!(context, "Folder \"{}\" is empty.", folder); - // set lastseenuid=0 for empty folders. - // id we do not do this here, we'll miss the first message - // as we will get in here again and fetch from lastseenuid+1 then - - set_config_last_seen_uid(context, &folder, new_uid_validity, 0).await; - return Ok((new_uid_validity, 0)); + // set uid_next=1 for empty folders. + // If we do not do this here, we'll miss the first message + // as we will get in here again and fetch from uid_next then. + // Also, the "fall back to fetching" below would need a non-zero mailbox.exists to work. + set_uid_next(context, folder, 1).await?; + set_uidvalidity(context, folder, new_uid_validity).await?; + return Ok(false); } - // uid_validity has changed or is being set the first time. - // find the last seen uid within the new uid_validity scope. - let new_last_seen_uid = match mailbox.uid_next { - Some(uid_next) => { - uid_next - 1 // XXX could uid_next be 0? - } + // ============== uid_validity has changed or is being set the first time. ============== + + let new_uid_next = match mailbox.uid_next { + Some(uid_next) => uid_next, None => { warn!( context, "IMAP folder has no uid_next, fall back to fetching" ); - if let Some(ref mut session) = &mut self.session { - // note that we use fetch by sequence number - // and thus we only need to get exactly the - // last-index message. - let set = format!("{}", mailbox.exists); - match session.fetch(set, JUST_UID).await { - Ok(mut list) => { - let mut new_last_seen_uid = None; - while let Some(fetch) = list.next().await.transpose()? { - if fetch.message == mailbox.exists && fetch.uid.is_some() { - new_last_seen_uid = fetch.uid; - } - } - if let Some(new_last_seen_uid) = new_last_seen_uid { - new_last_seen_uid - } else { - bail!("failed to fetch"); - } - } - Err(err) => { - bail!("IMAP Could not fetch: {}", err); - } + let session = self.session.as_mut().context("Get uid_next: Nosession")?; + // note that we use fetch by sequence number + // and thus we only need to get exactly the + // last-index message. + let set = format!("{}", mailbox.exists); + let mut list = session + .fetch(set, JUST_UID) + .await + .context("Error fetching UID")?; + + let mut new_last_seen_uid = None; + while let Some(fetch) = list.next().await.transpose()? { + if fetch.message == mailbox.exists && fetch.uid.is_some() { + new_last_seen_uid = fetch.uid; } - } else { - bail!("IMAP No Connection established"); } + new_last_seen_uid.context("select: failed to fetch")? + 1 } }; - set_config_last_seen_uid(context, &folder, new_uid_validity, new_last_seen_uid).await; - if uid_validity != 0 || last_seen_uid != 0 { + set_uid_next(context, folder, new_uid_next).await?; + set_uidvalidity(context, folder, new_uid_validity).await?; + if old_uid_validity != 0 || old_uid_next != 0 { job::schedule_resync(context).await; } info!( context, - "uid/validity change: new {}/{} current {}/{}", - new_last_seen_uid, + "uid/validity change folder {}: new {}/{} previous {}/{}", + folder, + new_uid_next, new_uid_validity, - uid_validity, - last_seen_uid + old_uid_next, + old_uid_validity, ); - Ok((new_uid_validity, new_last_seen_uid)) + Ok(false) } pub(crate) async fn fetch_new_messages>( @@ -637,21 +643,28 @@ impl Imap { let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await) .unwrap_or_default(); - let (uid_validity, last_seen_uid) = self + let new_emails = self .select_with_uidvalidity(context, folder.as_ref()) .await?; + if !new_emails && !fetch_existing_msgs { + info!(context, "No new emails in folder {}", folder.as_ref()); + return Ok(false); + } + + let old_uid_next = get_uid_next(context, folder.as_ref()).await; + let msgs = if fetch_existing_msgs { - self.fetch_existing_msgs_prefetch().await? + self.prefetch_existing_msgs().await? } else { - self.fetch_after(context, last_seen_uid).await? + self.prefetch(old_uid_next).await? }; let read_cnt = msgs.len(); let folder: &str = folder.as_ref(); let mut read_errors = 0; let mut uids = Vec::with_capacity(msgs.len()); - let mut new_last_seen_uid = None; + let mut largest_uid_skipped = None; for (current_uid, msg) in msgs.into_iter() { let (headers, msg_id) = match get_fetch_headers(&msg) { @@ -676,27 +689,33 @@ impl Imap { ) .await { - // Trigger download and processing for this message. uids.push(current_uid); } else if read_errors == 0 { - // No errors so far, but this was skipped, so mark as last_seen_uid - new_last_seen_uid = Some(current_uid); + // If there were errors (`read_errors != 0`), stop updating largest_uid_skipped so that uid_next will + // not be updated and we will retry prefetching next time + largest_uid_skipped = Some(current_uid); } } - // check passed, go fetch the emails - let (new_last_seen_uid_processed, error_cnt) = self + let (largest_uid_processed, error_cnt) = self .fetch_many_msgs(context, &folder, uids, fetch_existing_msgs) .await; read_errors += error_cnt; - // determine which last_seen_uid to use to update to - let new_last_seen_uid_processed = new_last_seen_uid_processed.unwrap_or_default(); - let new_last_seen_uid = new_last_seen_uid.unwrap_or_default(); - let last_one = new_last_seen_uid.max(new_last_seen_uid_processed); + // determine which uid_next to use to update to + // dc_receive_imf() returns an `Err` value only on recoverable errors, otherwise it just logs an error. + // `largest_uid_processed` is the largest uid where dc_receive_imf() did NOT return an error. - if last_one > last_seen_uid { - set_config_last_seen_uid(context, &folder, uid_validity, last_one).await; + // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was + // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times. + let largest_uid_without_errors = max( + largest_uid_processed.unwrap_or(0), + largest_uid_skipped.unwrap_or(0), + ); + let new_uid_next = largest_uid_without_errors + 1; + + if new_uid_next > old_uid_next { + set_uid_next(context, &folder, new_uid_next).await?; } if read_errors == 0 { @@ -761,18 +780,13 @@ impl Imap { Ok(result) } - /// Fetch all uids larger than the passed in. Returns a sorted list of fetch results. - async fn fetch_after( - &mut self, - context: &Context, - uid: u32, - ) -> Result> { + /// Prefetch all messages greater than or equal to `uid_next`. Return a list of fetch results. + async fn prefetch(&mut self, uid_next: u32) -> Result> { let session = self.session.as_mut(); let session = session.context("fetch_after(): IMAP No Connection established")?; // fetch messages with larger UID than the last one seen - // `(UID FETCH lastseenuid+1:*)`, see RFC 4549 - let set = format!("{}:*", uid + 1); + let set = format!("{}:*", uid_next); let mut list = session .uid_fetch(set, PREFETCH_FLAGS) .await @@ -790,26 +804,17 @@ impl Imap { // If the mailbox is not empty, results always include // at least one UID, even if last_seen_uid+1 is past // the last UID in the mailbox. It happens because - // uid+1:* is interpreted the same way as *:uid+1. + // uid:* is interpreted the same way as *:uid. // See https://tools.ietf.org/html/rfc3501#page-61 for // standard reference. Therefore, sometimes we receive // already seen messages and have to filter them out. - let new_msgs = msgs.split_off(&(uid + 1)); - - for current_uid in msgs.keys() { - info!( - context, - "fetch_new_messages: ignoring uid {}, last seen was {}", current_uid, uid - ); - } + let new_msgs = msgs.split_off(&uid_next); Ok(new_msgs) } /// Like fetch_after(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages) - async fn fetch_existing_msgs_prefetch( - &mut self, - ) -> Result> { + async fn prefetch_existing_msgs(&mut self) -> Result> { let exists: i64 = { let mailbox = self.config.selected_mailbox.as_ref(); let mailbox = mailbox.context("fetch_existing_msgs_prefetch(): no mailbox selected")?; @@ -841,7 +846,6 @@ impl Imap { } /// Fetches a list of messages by server UID. - /// The passed in list of uids must be sorted. /// /// Returns the last uid fetch successfully and an error count. async fn fetch_many_msgs>( @@ -1116,7 +1120,7 @@ impl Imap { } } match self.select_folder(context, Some(&folder)).await { - Ok(()) => None, + Ok(_) => None, Err(select_folder::Error::ConnectionLost) => { warn!(context, "Lost imap connection"); Some(ImapActionResult::RetryLater) @@ -1286,6 +1290,7 @@ impl Imap { let mut delimiter = ".".to_string(); let mut delimiter_is_default = true; let mut sentbox_folder = None; + let mut spam_folder = None; let mut mvbox_folder = None; let mut fallback_folder = get_fallback_folder(&delimiter); @@ -1302,6 +1307,8 @@ impl Imap { } } + let folder_meaning = get_folder_meaning(&folder); + let folder_name_meaning = get_folder_meaning_by_name(&folder.name()); if folder.name() == "DeltaChat" { // Always takes precendent mvbox_folder = Some(folder.name().to_string()); @@ -1310,16 +1317,18 @@ impl Imap { if mvbox_folder.is_none() { mvbox_folder = Some(folder.name().to_string()); } - } else if let FolderMeaning::SentObjects = get_folder_meaning(&folder) { + } else if folder_meaning == FolderMeaning::SentObjects { // Always takes precedent sentbox_folder = Some(folder.name().to_string()); - } else if let FolderMeaning::SentObjects = - get_folder_meaning_by_name(&folder.name()) - { + } else if folder_meaning == FolderMeaning::Spam { + spam_folder = Some(folder.name().to_string()); + } else if folder_name_meaning == FolderMeaning::SentObjects { // only set iff none has been already set if sentbox_folder.is_none() { sentbox_folder = Some(folder.name().to_string()); } + } else if folder_name_meaning == FolderMeaning::Spam && spam_folder.is_none() { + spam_folder = Some(folder.name().to_string()); } } drop(folders); @@ -1378,6 +1387,11 @@ impl Imap { .set_config(Config::ConfiguredSentboxFolder, Some(sentbox_folder)) .await?; } + if let Some(ref spam_folder) = spam_folder { + context + .set_config(Config::ConfiguredSpamFolder, Some(spam_folder)) + .await?; + } context .sql .set_raw_config_int(context, "folders_configured", DC_FOLDERS_CONFIGURED_VERSION) @@ -1396,7 +1410,7 @@ impl Imap { // but sth. different in others - a hard job. fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning { // source: https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders - let sent_names = vec![ + const SENT_NAMES: &[&str] = &[ "sent", "sentmail", "sent objects", @@ -1428,17 +1442,40 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning { "送信済み", "보낸편지함", ]; + const SPAM_NAMES: &[&str] = &[ + "spam", + "junk", + "Correio electrónico não solicitado", + "Correo basura", + "Lixo", + "Nettsøppel", + "Nevyžádaná pošta", + "No solicitado", + "Ongewenst", + "Posta indesiderata", + "Skräp", + "Wiadomości-śmieci", + "Önemsiz", + "Ανεπιθύμητα", + "Спам", + "垃圾邮件", + "垃圾郵件", + "迷惑メール", + "스팸", + ]; let lower = folder_name.to_lowercase(); - if sent_names.into_iter().any(|s| s.to_lowercase() == lower) { + if SENT_NAMES.iter().any(|s| s.to_lowercase() == lower) { FolderMeaning::SentObjects + } else if SPAM_NAMES.iter().any(|s| s.to_lowercase() == lower) { + FolderMeaning::Spam } else { FolderMeaning::Unknown } } fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { - let special_names = vec!["\\Spam", "\\Trash", "\\Drafts", "\\Junk"]; + let special_names = vec!["\\Trash", "\\Drafts"]; for attr in folder_name.attributes() { if let NameAttribute::Custom(ref label) = attr { @@ -1446,6 +1483,8 @@ fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { return FolderMeaning::Other; } else if label == "\\Sent" { return FolderMeaning::SentObjects; + } else if label == "\\Spam" || label == "\\Junk" { + return FolderMeaning::Spam; } } } @@ -1474,6 +1513,7 @@ async fn precheck_imf( .needs_move(context, server_folder) .await .unwrap_or_default() + .is_some() { // If the bcc-self message is not moved, directly // add MarkSeen job, otherwise MarkSeen job is @@ -1659,23 +1699,68 @@ fn get_fallback_folder(delimiter: &str) -> String { format!("INBOX{}DeltaChat", delimiter) } -pub async fn set_config_last_seen_uid>( - context: &Context, - folder: S, - uidvalidity: u32, - lastseenuid: u32, -) { - let key = format!("imap.mailbox.{}", folder.as_ref()); - let val = format!("{}:{}", uidvalidity, lastseenuid); - +/// uid_next is the next unique identifier value from the last time we fetched a folder +/// See https://tools.ietf.org/html/rfc3501#section-2.3.1.1 +/// This function is used to update our uid_next after fetching messages. +pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32) -> Result<()> { context .sql - .set_raw_config(context, &key, Some(&val)) - .await - .ok(); + .execute( + "INSERT INTO imap_sync (folder, uidvalidity, uid_next) VALUES (?,?,?) + ON CONFLICT(folder) DO UPDATE SET uid_next=? WHERE folder=?;", + paramsv![folder, 0u32, uid_next, uid_next, folder], + ) + .await?; + Ok(()) } -async fn get_config_last_seen_uid>(context: &Context, folder: S) -> (u32, u32) { +/// uid_next is the next unique identifier value from the last time we fetched a folder +/// See https://tools.ietf.org/html/rfc3501#section-2.3.1.1 +/// This method returns the uid_next from the last time we fetched messages. +/// We can compare this to the current uid_next to find out whether there are new messages +/// and fetch from this value on to get all new messages. +async fn get_uid_next(context: &Context, folder: &str) -> u32 { + context + .sql + .query_get_value( + context, + "SELECT uid_next FROM imap_sync WHERE folder=?;", + paramsv![folder], + ) + .await + .unwrap_or(0) +} + +pub(crate) async fn set_uidvalidity( + context: &Context, + folder: &str, + uidvalidity: u32, +) -> Result<()> { + context + .sql + .execute( + "INSERT INTO imap_sync (folder, uidvalidity, uid_next) VALUES (?,?,?) + ON CONFLICT(folder) DO UPDATE SET uidvalidity=? WHERE folder=?;", + paramsv![folder, uidvalidity, 0u32, uidvalidity, folder], + ) + .await?; + Ok(()) +} + +async fn get_uidvalidity(context: &Context, folder: &str) -> u32 { + context + .sql + .query_get_value( + context, + "SELECT uidvalidity FROM imap_sync WHERE folder=?;", + paramsv![folder], + ) + .await + .unwrap_or(0) +} + +/// Deprecated, use get_uid_next() and get_uidvalidity() +pub async fn get_config_last_seen_uid>(context: &Context, folder: S) -> (u32, u32) { let key = format!("imap.mailbox.{}", folder.as_ref()); if let Some(entry) = context.sql.get_raw_config(context, &key).await { // the entry has the format `imap.mailbox.=:` @@ -1750,6 +1835,7 @@ impl std::fmt::Display for UidRange { #[cfg(test)] mod tests { use super::*; + use crate::test_utils::TestContext; #[test] fn test_get_folder_meaning_by_name() { assert_eq!( @@ -1773,6 +1859,23 @@ mod tests { FolderMeaning::SentObjects ); assert_eq!(get_folder_meaning_by_name("xxx"), FolderMeaning::Unknown); + assert_eq!(get_folder_meaning_by_name("SPAM"), FolderMeaning::Spam); + } + + #[async_std::test] + async fn test_set_uid_next_validity() { + let t = TestContext::new_alice().await; + assert_eq!(get_uid_next(&t.ctx, "Inbox").await, 0); + assert_eq!(get_uidvalidity(&t.ctx, "Inbox").await, 0); + + set_uidvalidity(&t.ctx, "Inbox", 7).await.unwrap(); + assert_eq!(get_uidvalidity(&t.ctx, "Inbox").await, 7); + assert_eq!(get_uid_next(&t.ctx, "Inbox").await, 0); + + set_uid_next(&t.ctx, "Inbox", 5).await.unwrap(); + set_uidvalidity(&t.ctx, "Inbox", 6).await.unwrap(); + assert_eq!(get_uid_next(&t.ctx, "Inbox").await, 5); + assert_eq!(get_uidvalidity(&t.ctx, "Inbox").await, 6); } #[test] diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs new file mode 100644 index 000000000..ef46bb891 --- /dev/null +++ b/src/imap/scan_folders.rs @@ -0,0 +1,80 @@ +use std::time::Instant; + +use crate::{config::Config, context::Context}; +use anyhow::Context as _; + +use crate::error::Result; +use crate::imap::Imap; +use async_std::prelude::*; + +use super::{get_folder_meaning, get_folder_meaning_by_name, FolderMeaning}; + +impl Imap { + pub async fn scan_folders(&mut self, context: &Context) -> Result<()> { + // First of all, debounce to once per minute: + let mut last_scan = context.last_full_folder_scan.lock().await; + if let Some(last_scan) = *last_scan { + let elapsed_secs = last_scan.elapsed().as_secs(); + let debounce_secs = context + .get_config_u64(Config::ScanAllFoldersDebounceSecs) + .await; + + if elapsed_secs < debounce_secs { + info!(context, "Not scanning, we scanned {}s ago", elapsed_secs); + return Ok(()); + } + } + info!(context, "Starting full folder scan"); + + self.setup_handle(context).await?; + let session = self.session.as_mut(); + let session = session.context("scan_folders(): IMAP No Connection established")?; + let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await; + + let mut sentbox_folder = None; + let mut spam_folder = None; + + for folder in folders { + let folder = match folder { + Ok(f) => f, + Err(e) => { + warn!(context, "Can't get folder: {}", e); + continue; + } + }; + let foldername = folder.name(); + info!(context, "Scanning folder: {}", foldername); + + let folder_meaning = get_folder_meaning(&folder); + let folder_name_meaning = get_folder_meaning_by_name(&foldername); + + if folder_meaning == FolderMeaning::SentObjects { + // Always takes precedent + sentbox_folder = Some(folder.name().to_string()); + } else if folder_meaning == FolderMeaning::Spam { + spam_folder = Some(folder.name().to_string()); + } else if folder_name_meaning == FolderMeaning::SentObjects { + // only set iff none has been already set + if sentbox_folder.is_none() { + sentbox_folder = Some(folder.name().to_string()); + } + } else if folder_name_meaning == FolderMeaning::Spam && spam_folder.is_none() { + spam_folder = Some(folder.name().to_string()); + } + + if let Err(e) = self.fetch_new_messages(context, foldername, false).await { + warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e); + } + } + + context + .set_config(Config::ConfiguredSentboxFolder, sentbox_folder.as_deref()) + .await?; + context + .set_config(Config::ConfiguredSpamFolder, spam_folder.as_deref()) + .await?; + + last_scan.replace(Instant::now()); + Ok(()) + } +} diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index 4c335015e..0d792a124 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -61,11 +61,12 @@ impl Imap { /// select a folder, possibly update uid_validity and, if needed, /// expunge the folder to remove delete-marked messages. + /// Returns whether a new folder was selected. pub(super) async fn select_folder>( &mut self, context: &Context, folder: Option, - ) -> Result<()> { + ) -> Result { if self.session.is_none() { self.config.selected_folder = None; self.config.selected_folder_needs_expunge = false; @@ -78,7 +79,7 @@ impl Imap { if let Some(ref folder) = folder { if let Some(ref selected_folder) = self.config.selected_folder { if folder.as_ref() == selected_folder { - return Ok(()); + return Ok(NewlySelected::No); } } } @@ -99,7 +100,7 @@ impl Imap { Ok(mailbox) => { self.config.selected_folder = Some(folder.as_ref().to_string()); self.config.selected_mailbox = Some(mailbox); - Ok(()) + Ok(NewlySelected::Yes) } Err(async_imap::error::Error::ConnectionLost) => { self.trigger_reconnect(); @@ -119,7 +120,15 @@ impl Imap { Err(Error::NoSession) } } else { - Ok(()) + Ok(NewlySelected::No) } } } +#[derive(PartialEq, Debug, Copy, Clone, Eq)] +pub(super) enum NewlySelected { + /// The folder was newly selected during this call to select_folder(). + Yes, + /// No SELECT command was run because the folder already was selected + /// and self.config.selected_mailbox was not updated (so, e.g. it may contain an outdated uid_next) + No, +} diff --git a/src/job.rs b/src/job.rs index 636d52741..b37513bed 100644 --- a/src/job.rs +++ b/src/job.rs @@ -10,6 +10,7 @@ use deltachat_derive::{FromSql, ToSql}; use itertools::Itertools; use rand::{thread_rng, Rng}; +use anyhow::Context as _; use async_smtp::smtp::response::Category; use async_smtp::smtp::response::Code; use async_smtp::smtp::response::Detail; @@ -511,11 +512,27 @@ impl Job { } let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await); - let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await; + let server_folder = &job_try!(msg + .server_folder + .context("Can't move message out of folder if we don't know the current folder")); + + let move_res = msg.id.needs_move(context, server_folder).await; + let dest_folder = match move_res { + Err(e) => { + warn!(context, "could not load dest folder: {}", e); + return Status::RetryLater; + } + Ok(None) => { + warn!( + context, + "msg {} does not need to be moved from {}", msg.id, server_folder + ); + return Status::Finished(Ok(())); + } + Ok(Some(config)) => context.get_config(config).await, + }; if let Some(dest_folder) = dest_folder { - let server_folder = msg.server_folder.as_ref().unwrap(); - match imap .mv(context, server_folder, msg.server_uid, &dest_folder) .await @@ -753,6 +770,7 @@ impl Job { // retry. If the message was moved, we will create another // job to mark the message as seen later. If it was // deleted, there is nothing to do. + info!(context, "Can't mark message as seen: No UID"); ImapActionResult::Failed } else { imap.set_seen(context, folder, msg.server_uid).await @@ -844,7 +862,8 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold return; }; if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await { - warn!(context, "Could not select {}: {}", mailbox, e); + // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}: + warn!(context, "Could not select {}: {:#}", mailbox, e); return; } match imap.get_all_recipients(context).await { diff --git a/src/message.rs b/src/message.rs index 695be074f..aff80bd52 100644 --- a/src/message.rs +++ b/src/message.rs @@ -84,18 +84,54 @@ impl MsgId { Ok(result) } - /// Returns true if the message needs to be moved from `folder`. - pub async fn needs_move(self, context: &Context, folder: &str) -> Result { - if !context.get_config_bool(Config::MvboxMove).await { - return Ok(false); - } - + /// Returns Some if the message needs to be moved from `folder`. + /// If yes, returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder`, + /// depending on where the message should be moved + pub async fn needs_move( + self, + context: &Context, + folder: &str, + ) -> Result, Error> { + use Config::*; if context.is_mvbox(folder).await { - return Ok(false); + return Ok(None); } let msg = Message::load_from_db(context, self).await?; + if context.is_spam_folder(folder).await { + return if msg.chat_blocked == Blocked::Not { + if self.needs_move_to_mvbox(context, &msg).await? { + Ok(Some(ConfiguredMvboxFolder)) + } else { + Ok(Some(ConfiguredInboxFolder)) + } + } else { + // Blocked/deaddrop message in the spam folder, leave it there + Ok(None) + }; + } + + if self.needs_move_to_mvbox(context, &msg).await? { + Ok(Some(ConfiguredMvboxFolder)) + } else if msg.state.is_outgoing() + && msg.is_dc_message == MessengerMessage::Yes + && !msg.is_setupmessage() + && msg.to_id != DC_CONTACT_ID_SELF // Leave self-chat-messages in the inbox, not sure about this + && context.is_inbox(folder).await + && context.get_config(ConfiguredSentboxFolder).await.is_some() + { + Ok(Some(ConfiguredSentboxFolder)) + } else { + Ok(None) + } + } + + async fn needs_move_to_mvbox(self, context: &Context, msg: &Message) -> Result { + if !context.get_config_bool(Config::MvboxMove).await { + return Ok(false); + } + if msg.is_setupmessage() { // do not move setup messages; // there may be a non-delta device that wants to handle it @@ -1878,6 +1914,7 @@ mod tests { use crate::chat::ChatItem; use crate::constants::DC_CONTACT_ID_DEVICE; use crate::test_utils as test; + use crate::test_utils::TestContext; #[test] fn test_guess_msgtype_from_suffix() { @@ -1887,6 +1924,197 @@ mod tests { ); } + // chat_msg means that the message was sent by Delta Chat + // The tuples are (folder, mvbox_move, chat_msg, expected_destination) + const COMBINATIONS_ACCEPTED_CHAT: &[(&str, bool, bool, &str)] = &[ + ("INBOX", false, false, "INBOX"), + ("INBOX", false, true, "INBOX"), + ("INBOX", true, false, "INBOX"), + ("INBOX", true, true, "DeltaChat"), + ("Sent", false, false, "Sent"), + ("Sent", false, true, "Sent"), + ("Sent", true, false, "Sent"), + ("Sent", true, true, "DeltaChat"), + ("Spam", false, false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs + ("Spam", false, true, "INBOX"), + ("Spam", true, false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs + ("Spam", true, true, "DeltaChat"), + ]; + + // These are the same as above, but all messages in Spam stay in Spam + const COMBINATIONS_DEADDROP: &[(&str, bool, bool, &str)] = &[ + ("INBOX", false, false, "INBOX"), + ("INBOX", false, true, "INBOX"), + ("INBOX", true, false, "INBOX"), + ("INBOX", true, true, "DeltaChat"), + ("Sent", false, false, "Sent"), + ("Sent", false, true, "Sent"), + ("Sent", true, false, "Sent"), + ("Sent", true, true, "DeltaChat"), + ("Spam", false, false, "Spam"), + ("Spam", false, true, "Spam"), + ("Spam", true, false, "Spam"), + ("Spam", true, true, "Spam"), + ]; + + #[async_std::test] + async fn test_needs_move_incoming_accepted() { + for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT { + check_needs_move_combination( + folder, + *mvbox_move, + *chat_msg, + expected_destination, + true, + false, + false, + ) + .await; + } + } + + #[async_std::test] + async fn test_needs_move_incoming_deaddrop() { + for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_DEADDROP { + check_needs_move_combination( + folder, + *mvbox_move, + *chat_msg, + expected_destination, + false, + false, + false, + ) + .await; + } + } + + #[async_std::test] + async fn test_needs_move_outgoing() { + // Test outgoing emails + for (folder, mvbox_move, chat_msg, mut expected_destination) in COMBINATIONS_ACCEPTED_CHAT { + if *folder == "INBOX" && !mvbox_move && *chat_msg { + expected_destination = "Sent" + } + check_needs_move_combination( + folder, + *mvbox_move, + *chat_msg, + expected_destination, + true, + true, + false, + ) + .await; + } + } + + #[async_std::test] + async fn test_needs_move_setupmsg() { + // Test setupmessages + for (folder, mvbox_move, chat_msg, _expected_destination) in COMBINATIONS_ACCEPTED_CHAT { + check_needs_move_combination( + folder, + *mvbox_move, + *chat_msg, + if folder == &"Spam" { "INBOX" } else { folder }, // Never move setup messages, except if they are in "Spam" + true, + true, + true, + ) + .await; + } + } + + async fn check_needs_move_combination( + folder: &str, + mvbox_move: bool, + chat_msg: bool, + expected_destination: &str, + accepted_chat: bool, + outgoing: bool, + setupmessage: bool, + ) { + use crate::dc_receive_imf::dc_receive_imf; + println!("Testing: For folder {}, mvbox_move {}, chat_msg {}, accepted {}, outgoing {}, setupmessage {}", + folder, mvbox_move, chat_msg, accepted_chat, outgoing, setupmessage); + + let t = TestContext::new_alice().await; + t.ctx + .set_config(Config::ConfiguredSpamFolder, Some("Spam")) + .await + .unwrap(); + t.ctx + .set_config(Config::ConfiguredMvboxFolder, Some("DeltaChat")) + .await + .unwrap(); + t.ctx + .set_config(Config::ConfiguredSentboxFolder, Some("Sent")) + .await + .unwrap(); + t.ctx + .set_config(Config::MvboxMove, Some(if mvbox_move { "1" } else { "0" })) + .await + .unwrap(); + t.ctx + .set_config(Config::ShowEmails, Some("2")) + .await + .unwrap(); + + if accepted_chat { + let contact_id = Contact::create(&t.ctx, "", "bob@example.net") + .await + .unwrap(); + chat::create_by_contact_id(&t.ctx, contact_id) + .await + .unwrap(); + } + let temp; + dc_receive_imf( + &t.ctx, + if setupmessage { + include_bytes!("../test-data/message/AutocryptSetupMessage.eml") + } else { + temp = format!( + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + {}\ + Subject: foo\n\ + Message-ID: \n\ + {}\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ + \n\ + hello\n", + if outgoing { + "From: alice@example.com\nTo: bob@example.net\n" + } else { + "From: bob@example.net\nTo: alice@example.com\n" + }, + if chat_msg { "Chat-Version: 1.0\n" } else { "" }, + ); + temp.as_bytes() + }, + folder, + 1, + false, + ) + .await + .unwrap(); + + let msg = t.get_last_msg().await; + let actual = if let Some(config) = msg.id.needs_move(&t.ctx, folder).await.unwrap() { + Some(t.ctx.get_config(config).await.unwrap()) + } else { + None + }; + let expected = if expected_destination == folder { + None + } else { + Some(expected_destination) + }; + assert_eq!(expected, actual.as_deref(), "For folder {}, mvbox_move {}, chat_msg {}, accepted {}, outgoing {}, setupmessage {}: expected {:?} , got {:?}", + folder, mvbox_move, chat_msg, accepted_chat, outgoing, setupmessage, expected, actual); + } + #[async_std::test] async fn test_prepare_message_and_send() { use crate::config::Config; diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 8a8b155bd..91947a3b2 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1372,7 +1372,8 @@ mod tests { // 1.: Receive a mail from an MUA or Delta Chat assert_eq!( msg_to_subject_str( - b"From: Bob \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Bob \n\ To: alice@example.com\n\ Subject: Antw: Chat: hello\n\ Message-ID: <2222@example.com>\n\ @@ -1386,7 +1387,8 @@ mod tests { assert_eq!( msg_to_subject_str( - b"From: Bob \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Bob \n\ To: alice@example.com\n\ Subject: Infos: 42\n\ Message-ID: <2222@example.com>\n\ @@ -1401,7 +1403,8 @@ mod tests { // 2. Receive a message from Delta Chat when we did not send any messages before assert_eq!( msg_to_subject_str( - b"From: Charlie \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Charlie \n\ To: alice@example.com\n\ Subject: Chat: hello\n\ Chat-Version: 1.0\n\ @@ -1427,7 +1430,8 @@ mod tests { // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result) msg_to_subject_str( - "From: Charlie \n\ + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Charlie \n\ To: alice@example.com\n\ Subject: äääää\n\ Chat-Version: 1.0\n\ @@ -1440,7 +1444,8 @@ mod tests { .await; msg_to_subject_str( - "From: Charlie \n\ + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Charlie \n\ To: alice@example.com\n\ Subject: aäääää\n\ Chat-Version: 1.0\n\ @@ -1456,7 +1461,8 @@ mod tests { let t = TestContext::new_alice().await; dc_receive_imf( &t, - b"From: alice@example.com\n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: alice@example.com\n\ To: Charlie \n\ Subject: Hello, Charlie\n\ Chat-Version: 1.0\n\ @@ -1470,7 +1476,9 @@ mod tests { ) .await .unwrap(); - let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\ + let new_msg = incoming_msg_to_reply_msg( + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: charlie@example.com\n\ To: alice@example.com\n\ Subject: message opened\n\ Date: Sun, 22 Mar 2020 23:37:57 +0000\n\ @@ -1559,7 +1567,8 @@ mod tests { let context = &t; let msg = incoming_msg_to_reply_msg( - b"From: Charlie \n\ + b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ + From: Charlie \n\ To: alice@example.com\n\ Subject: Chat: hello\n\ Chat-Version: 1.0\n\ diff --git a/src/scheduler.rs b/src/scheduler.rs index b7a189713..a956c0373 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -118,7 +118,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) { // fetch if let Err(err) = connection.fetch(&ctx, &watch_folder).await { connection.trigger_reconnect(); - warn!(ctx, "{}", err); + warn!(ctx, "{:#}", err); } } None => { @@ -140,6 +140,12 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int // fetch if let Err(err) = connection.fetch(&ctx, &watch_folder).await { connection.trigger_reconnect(); + warn!(ctx, "{:#}", err); + } + + if let Err(err) = connection.scan_folders(&ctx).await { + // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing + // but maybe just one folder can't be selected or something warn!(ctx, "{}", err); } info!(ctx, "verbose (issue 2065): step 1 done fetching"); diff --git a/src/sql.rs b/src/sql.rs index 7a7997375..3192b5227 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -9,18 +9,23 @@ use std::time::Duration; use rusqlite::{Connection, Error as SqlError, OpenFlags}; +use crate::chat::add_device_msg; +use crate::config::Config::DeleteServerAfter; use crate::constants::{ShowEmails, DC_CHAT_ID_TRASH}; use crate::context::Context; use crate::dc_tools::{dc_delete_file, time, EmailAddress}; use crate::ephemeral::start_ephemeral_timers; use crate::error::format_err; +use crate::imap; use crate::param::{Param, Params}; use crate::peerstate::Peerstate; use crate::provider::get_provider_by_domain; +use crate::stock::StockMessage; use crate::{ chat::{update_device_icon, update_saved_messages_icon}, config::Config, }; +use crate::{constants::Viewtype, message::Message}; #[macro_export] macro_rules! paramsv { @@ -965,6 +970,7 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label); let mut dbversion = dbversion_before_update; let mut recalc_fingerprints = false; let mut update_icons = !exists_before_update; + let mut disable_server_delete = false; if dbversion < 1 { info!(context, "[migration] v1"); @@ -1449,6 +1455,33 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label); } sql.set_raw_config_int(context, "dbversion", 72).await?; } + if dbversion < 73 { + use Config::*; + info!(context, "[migration] v73"); + sql.execute( + "CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0, uid_next INTEGER DEFAULT 0);", + paramsv![], + ) + .await?; + for c in &[ + ConfiguredInboxFolder, + ConfiguredSentboxFolder, + ConfiguredMvboxFolder, + ] { + if let Some(folder) = context.get_config(*c).await { + let (uid_validity, last_seen_uid) = + imap::get_config_last_seen_uid(context, &folder).await; + if last_seen_uid > 0 { + imap::set_uid_next(context, &folder, last_seen_uid + 1).await?; + imap::set_uidvalidity(context, &folder, uid_validity).await?; + } + } + } + if exists_before_update { + disable_server_delete = true; + } + sql.set_raw_config_int(context, "dbversion", 73).await?; + } // (2) updates that require high-level objects // (the structure is complete now and all objects are usable) @@ -1479,6 +1512,21 @@ CREATE INDEX devmsglabels_index1 ON devmsglabels (label); update_saved_messages_icon(context).await?; update_device_icon(context).await?; } + if disable_server_delete { + // We now always watch all folders and delete messages there if delete_server is enabled. + // So, for people who have delete_server enabled, disable it and add a hint to the devicechat: + if context.get_config_delete_server_after().await.is_some() { + let mut msg = Message::new(Viewtype::Text); + msg.text = Some( + context + .stock_str(StockMessage::DeleteServerTurnedOff) + .await + .into(), + ); + add_device_msg(context, None, Some(&mut msg)).await?; + context.set_config(DeleteServerAfter, Some("0")).await?; + } + } } info!(context, "Opened {:?}.", dbfile.as_ref(),); diff --git a/src/stock.rs b/src/stock.rs index afb28e440..5485b3363 100644 --- a/src/stock.rs +++ b/src/stock.rs @@ -248,6 +248,13 @@ pub enum StockMessage { #[strum(props(fallback = "You deleted the \"Saved messages\" chat.\n\n\ To use the \"Saved messages\" feature again, create a new chat with yourself."))] SelfDeletedMsgBody = 91, + + #[strum(props( + fallback = "⚠️ The \"Delete messages from server\" feature now also deletes messages in folders other than Inbox, DeltaChat and Sent.\n\n\ + ℹ️ To avoid accidentally deleting messages, we turned it off for you. Please turn it on again at \ + Settings → \"Chats and Media\" → \"Delete messages from server\" to continue using it." + ))] + DeleteServerTurnedOff = 92, } /* diff --git a/src/test_utils.rs b/src/test_utils.rs index 0d6a6fd72..adcf9a74e 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -5,24 +5,32 @@ use std::time::{Duration, Instant}; use std::{ops::Deref, str::FromStr}; +use ansi_term::Color; use async_std::path::PathBuf; use async_std::sync::RwLock; use tempfile::{tempdir, TempDir}; +use crate::config::Config; +use crate::constants::DC_CONTACT_ID_SELF; +use crate::contact::Contact; +use crate::context::Context; +use crate::dc_receive_imf::dc_receive_imf; +use crate::dc_tools::EmailAddress; use crate::job::Action; use crate::key::{self, DcKey}; use crate::message::{update_msg_state, Message, MessageState, MsgId}; use crate::mimeparser::MimeMessage; use crate::param::{Param, Params}; -use crate::{chat, contact::Contact}; +use crate::{chat, chatlist::Chatlist}; +use crate::{chat::ChatItem, EventType}; use crate::{ - chat::{Chat, ChatId, ChatItem}, + chat::{Chat, ChatId}, contact::Origin, }; -use crate::{config::Config, constants::DC_CONTACT_ID_SELF}; -use crate::{constants::Viewtype, context::Context}; -use crate::{constants::DC_MSG_ID_DAYMARKER, dc_tools::EmailAddress}; -use crate::{constants::DC_MSG_ID_MARKER1, dc_receive_imf::dc_receive_imf}; + +use crate::constants::Viewtype; +use crate::constants::DC_MSG_ID_DAYMARKER; +use crate::constants::DC_MSG_ID_MARKER1; /// A Context and temporary directory. /// @@ -53,6 +61,13 @@ impl TestContext { let ctx = Context::new("FakeOS".into(), dbfile.into(), id) .await .unwrap(); + + let events = ctx.get_event_emitter(); + async_std::task::spawn(async move { + while let Some(event) = events.recv().await { + receive_event(event.typ); + } + }); Self { ctx, dir, @@ -189,7 +204,11 @@ impl TestContext { pub async fn recv_msg(&self, msg: &SentMessage) { let mut idx = self.recv_idx.write().await; *idx += 1; - dc_receive_imf(&self.ctx, msg.payload().as_bytes(), "INBOX", *idx, false) + let received_msg = + "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n" + .to_owned() + + &msg.payload(); + dc_receive_imf(&self.ctx, received_msg.as_bytes(), "INBOX", *idx, false) .await .unwrap(); } @@ -197,7 +216,7 @@ impl TestContext { /// Get the most recent message of a chat. /// /// Panics on errors or if the most recent message is a marker. - pub async fn get_last_msg(&self, chat_id: ChatId) -> Message { + pub async fn get_last_msg_in(&self, chat_id: ChatId) -> Message { let msgs = chat::get_chat_msgs(&self.ctx, chat_id, 0, None).await; let msg_id = if let ChatItem::Message { msg_id } = msgs.last().unwrap() { msg_id @@ -207,6 +226,13 @@ impl TestContext { Message::load_from_db(&self.ctx, *msg_id).await.unwrap() } + /// Get the most recent message over all chats. + pub async fn get_last_msg(&self) -> Message { + let chats = Chatlist::try_load(&self.ctx, 0, None, None).await.unwrap(); + let msg_id = chats.get_msg_id(chats.len() - 1).unwrap(); + Message::load_from_db(&self.ctx, msg_id).await.unwrap() + } + pub async fn create_chat(&self, other: &TestContext) -> Chat { let (contact_id, _modified) = Contact::add_or_lookup( self, @@ -364,6 +390,95 @@ pub(crate) fn bob_keypair() -> key::KeyPair { } } +fn receive_event(event: EventType) { + let green = Color::Green.normal(); + let yellow = Color::Yellow.normal(); + let red = Color::Red.normal(); + + match event { + EventType::Info(msg) => { + /* do not show the event as this would fill the screen */ + println!("{}", msg); + } + EventType::SmtpConnected(msg) => { + println!("[SMTP_CONNECTED] {}", msg); + } + EventType::ImapConnected(msg) => { + println!("[IMAP_CONNECTED] {}", msg); + } + EventType::SmtpMessageSent(msg) => { + println!("[SMTP_MESSAGE_SENT] {}", msg); + } + EventType::Warning(msg) => { + println!("{}", yellow.paint(msg)); + } + EventType::Error(msg) => { + println!("{}", red.paint(msg)); + } + EventType::ErrorNetwork(msg) => { + println!("{}", red.paint(format!("[NETWORK] msg={}", msg))); + } + EventType::ErrorSelfNotInGroup(msg) => { + println!("{}", red.paint(format!("[SELF_NOT_IN_GROUP] {}", msg))); + } + EventType::MsgsChanged { chat_id, msg_id } => { + println!( + "{}", + green.paint(format!( + "Received MSGS_CHANGED(chat_id={}, msg_id={})", + chat_id, msg_id, + )) + ); + } + EventType::ContactsChanged(_) => { + println!("{}", green.paint("Received CONTACTS_CHANGED()")); + } + EventType::LocationChanged(contact) => { + println!( + "{}", + green.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact)) + ); + } + EventType::ConfigureProgress { progress, comment } => { + if let Some(comment) = comment { + println!( + "{}", + green.paint(format!( + "Received CONFIGURE_PROGRESS({} ‰, {})", + progress, comment + )) + ); + } else { + println!( + "{}", + green.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress)) + ); + } + } + EventType::ImexProgress(progress) => { + println!( + "{}", + green.paint(format!("Received IMEX_PROGRESS({} ‰)", progress)) + ); + } + EventType::ImexFileWritten(file) => { + println!( + "{}", + green.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display())) + ); + } + EventType::ChatModified(chat) => { + println!( + "{}", + green.paint(format!("Received CHAT_MODIFIED({})", chat)) + ); + } + _ => { + println!("Received {:?}", event); + } + } +} + async fn log_msg(context: &Context, prefix: impl AsRef, msg: &Message) { let contact = Contact::get_by_id(context, msg.get_from_id()) .await diff --git a/test-data/message/AutocryptSetupMessage.eml b/test-data/message/AutocryptSetupMessage.eml new file mode 100644 index 000000000..5cef7b9b5 --- /dev/null +++ b/test-data/message/AutocryptSetupMessage.eml @@ -0,0 +1,77 @@ +Return-Path: +Delivered-To: alice@example.com +Received: from hq5.merlinux.eu + by hq5.merlinux.eu with LMTP + id gNKpOrrTvF+tVAAAPzvFDg + (envelope-from ) + for ; Tue, 24 Nov 2020 10:34:50 +0100 +Subject: Autocrypt Setup Message +DKIM-Signature: v=1; a=rsa-sha256; c=simple/simple; d=testrun.org; + s=testrun; t=1606210490; + bh=MXqLqHFK1xC48pxx2TS1GUdxKSi4tdejRRSV4EAN5Tc=; + h=Subject:Date:To:From:From; + b=DRajftyu+Ycfhaxy0jXAIKCihQRMI0rxbo9+EBu6y5jhtZx13emW3odgZnvyhU6uD + IKfMXaqlmc/2HNV1/mloJVIRsIp5ORncSPX9tLykNApJVyPHg3NKdMo3Ib4NGIJ1Qo + binmLtL5qqL3bYCL68WUgieH1rcgCaf9cwck9GvwZ79pexGuWz4ItgtNWqYfapG8Zc + 9eD5maiTMNkV7UwgtOzhbBd39uKgKCoGdLAq63hoJF6dhdBBRVRyRMusAooGUZMgwm + QVuTZ76z9G8w3rDgZuHmoiICWsLsar4CDl4zAgicE6bHwtw3a7YuMiHoCtceq0RjQP + BHVaXT7B75BoA== +MIME-Version: 1.0 +Date: Tue, 24 Nov 2020 09:34:48 +0000 +Chat-Version: 1.0 +Autocrypt-Setup-Message: v1 +Message-ID: +To: +From: +Content-Type: multipart/mixed; boundary="dKhu3bbmBniQsT8W8w58YRCCiBK2YY" + + +--dKhu3bbmBniQsT8W8w58YRCCiBK2YY +Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no + +This is the Autocrypt Setup Message used to transfer your end-to-end setup +between clients. + +To decrypt and use your setup, open the message in an Autocrypt-compliant +client and enter the setup code presented on the generating device. + +-- +Sent with my Delta Chat Messenger: https://delta.chat + +--dKhu3bbmBniQsT8W8w58YRCCiBK2YY +Content-Type: application/autocrypt-setup +Content-Disposition: attachment; filename="autocrypt-setup-message.html" +Content-Transfer-Encoding: base64 + +PCFET0NUWVBFIGh0bWw+DQo8aHRtbD4NCiAgPGhlYWQ+DQogICAgPHRpdGxlPkF1dG9jcnlwdCBTZX +R1cCBNZXNzYWdlPC90aXRsZT4NCiAgPC9oZWFkPg0KICA8Ym9keT4NCiAgICA8aDE+QXV0b2NyeXB0 +IFNldHVwIE1lc3NhZ2U8L2gxPg0KICAgIDxwPlRoaXMgaXMgdGhlIEF1dG9jcnlwdCBTZXR1cCBNZX +NzYWdlIHVzZWQgdG8gdHJhbnNmZXIgeW91ciBlbmQtdG8tZW5kIHNldHVwIGJldHdlZW4gY2xpZW50 +cy48YnI+PGJyPlRvIGRlY3J5cHQgYW5kIHVzZSB5b3VyIHNldHVwLCBvcGVuIHRoZSBtZXNzYWdlIG +luIGFuIEF1dG9jcnlwdC1jb21wbGlhbnQgY2xpZW50IGFuZCBlbnRlciB0aGUgc2V0dXAgY29kZSBw +cmVzZW50ZWQgb24gdGhlIGdlbmVyYXRpbmcgZGV2aWNlLjwvcD4NCiAgICA8cHJlPg0KLS0tLS1CRU +dJTiBQR1AgTUVTU0FHRS0tLS0tDQpQYXNzcGhyYXNlLUZvcm1hdDogbnVtZXJpYzl4NA0KUGFzc3Bo +cmFzZS1CZWdpbjogNjIKCnd4NEVCd01JWEUzNCs4RGhtSC9nRDNNY21JTjhCSUorbmhpbDMrOFE3bF +hTd21JQnhDSnhBU2VhQUJlTGdFOTIKTi9WaER5MHlrUHFBQkp0S0xvSG9pQmxTQWZJajFRemdPeVlV +Wjl3czRtSng5OVREUE1lSnNmNHJaemJhUHZFSApQcEIrTTgyTjVhUitvV0dTcWRtUUZNQUplNWNtWX +hwM3p4eE5aTEc2cXVnRzUzOFNxNUV1TzBDSGduaXlFeEwyCkJya2hFOWVFVE1oSkNRQ3dCZDc5alhN +U2Mwcm5xYjFHbS9Kd21jbXFqVFNHMlBLTWNNcFlaV1QwQkNMaDE2TmwKTkNNbmRQWGt2cTlHd1crNX +pEMHc4cElyOERNRHk1SWVBcG83amNZR1U5UWNUR1lMWmltR2QxK1RYYlgvdGxqRQplMnNZd0hZeU5D +R1N5bHVsYi9XYnNkS2FrYXVodHJ6cUVHOXNYSkJkMnF5ZjNJajRULzdCd1pJVk42OXF1T21sCnlmMm +9PTmtYY1pCcFBJUE9ZQzdhMnJ5aFh0Q0NhbWhIVEw0czdzclg2NzJXMTVXS3VqNGVBK25URlNocFBC +cXoKb05EY3QzbG95V0hNSUluSzRha1VJeTFZak42TDFSbGwwRVhudlVQS0lkT0FpY0swbFBPaDVUZU +t6ZFMvTklyMQpQc2x6c2RyWTRZd0diMWNTdk95OXJQRFpaS3Y4d0dzbFczcFpFOCs3NnJWckllbkNY +dTdvOUZ6OFhQcVlxTGRrCkpCZGRHUGZnY0l6Um5nZjZqb0lmT0RsU2NiajR0VlgyK3htVVN5RlVhSD +RQcDFzZDgwVjhDN2xhREJ2WTc0TlAKQW9ydEVhL2xGbzQzcHNOdlhrc0JUUEVRNHFoTVZneVdQWW9V +ZGV2aUFZOGVDMmJjT0dMSFVURk5zaHZCaDFGRgozVGpIZEVRVk5zZVlqaWtZRWtkUU9Mb3B5VWdqbj +lSTUJnV2xIZTNKL1VRcmtFUkNYWi9BSVRXeGdYdmE0NHBPCkkzUHllcnF2T1lpVlJLam9JSTVIZGU4 +UFdkTnZwb2J5ZCsrTHlqN3Jxd0kyNFRwbVRwYWtIZ1RJNEJvYWtLSUcKWm1JWDhsQm4xMnQ5dlcvcD +lrbDluYWluS3Z1VFBoTk4xZmkrTE1YYTRDK1hqRXVPUnQwMFMzc01MdVo3RnBPaQprcXdGWk12RUtw +bHA3dmRLSnJNbmVzZ2dKLzBLeWc1RTJ4dVd2VFdkZUFBOE1saEJqSGlsK3JVK0dSZzdaTmxsCkxUej +RKeGpWUVl5TGpFbkhqdGU4bUVnZlNIZEE3ZDErVnV1RTZSZjlYMzRPeXhkL3NocllJSU8xY3FVdnQw +V3MKNGIwQURIN0lkbjkveTdDRjVrbWFONkMyQURBRkhFRzNIRWFZaDVNNmIwVzVJSW55WkhUQ0QxdC +tmUFdQYndxUQo0TzFRMEROZ01QT1FCRVJ0ODNXR3g5YW5GQU9YCj05dTUrCi0tLS0tRU5EIFBHUCBN +RVNTQUdFLS0tLS0KDQo8L3ByZT4NCiAgPC9ib2R5Pg0KPC9odG1sPg0K + +--dKhu3bbmBniQsT8W8w58YRCCiBK2YY-- + diff --git a/test-data/message/pdf_filename_continuation.eml b/test-data/message/pdf_filename_continuation.eml index 5a715e320..74cfaa1d3 100644 --- a/test-data/message/pdf_filename_continuation.eml +++ b/test-data/message/pdf_filename_continuation.eml @@ -1,3 +1,4 @@ +Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ To: dcuser@b44t.com From: "Mail User" Subject: Re: test pdf with umlauts diff --git a/test-data/message/pdf_filename_simple.eml b/test-data/message/pdf_filename_simple.eml index 485e714fe..2df934c1e 100644 --- a/test-data/message/pdf_filename_simple.eml +++ b/test-data/message/pdf_filename_simple.eml @@ -1,3 +1,4 @@ +Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ To: dcuser@b44t.com From: "Mail User" Subject: Re: simple named pdf diff --git a/test-data/message/subj_with_multimedia_msg.eml b/test-data/message/subj_with_multimedia_msg.eml index 7e1be823f..20b19cf80 100644 --- a/test-data/message/subj_with_multimedia_msg.eml +++ b/test-data/message/subj_with_multimedia_msg.eml @@ -1,3 +1,4 @@ +Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ From: Hocuri Subject: subj with important info To: alice@example.com