From 2eeacb0f8a474167319bc047b57cc95942cde450 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 20 Feb 2023 14:02:19 +0000 Subject: [PATCH 01/30] sql: organize connection pool as a stack rather than a queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When connection pool is organized as a stack, it always returns most recently used connection. Because each connection has its own page cache, using the connection with fresh cache improves performance. I commented out `oauth2::tests::test_oauth_from_mx` because it requires network connection, turned off the Wi-Fi and ran the tests. Before the change, with a queue: ``` $ hyperfine "cargo test" Benchmark 1: cargo test Time (mean ± σ): 56.424 s ± 4.515 s [User: 183.181 s, System: 128.156 s] Range (min … max): 52.123 s … 68.193 s 10 runs ``` With a stack: ``` $ hyperfine "cargo test" Benchmark 1: cargo test Time (mean ± σ): 29.887 s ± 1.377 s [User: 101.226 s, System: 45.573 s] Range (min … max): 26.591 s … 31.010 s 10 runs ``` On version 1.107.1: ``` $ hyperfine "cargo test" Benchmark 1: cargo test Time (mean ± σ): 43.658 s ± 1.079 s [User: 202.582 s, System: 50.723 s] Range (min … max): 41.531 s … 45.170 s 10 runs ``` --- CHANGELOG.md | 2 ++ Cargo.lock | 2 +- Cargo.toml | 2 +- src/sql/pool.rs | 31 +++++++++++++++++++------------ 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 098244902..b057be945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Changes - use transaction in `Contact::add_or_lookup()` #4059 +- Organize the connection pool as a stack rather than a queue to ensure that + connection page cache is reused more often. #4065 ### Fixes diff --git a/Cargo.lock b/Cargo.lock index 39890839c..836a44375 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -848,7 +848,6 @@ dependencies = [ "bitflags", "chrono", "criterion", - "crossbeam-queue", "deltachat_derive", "email", "encoded-words", @@ -870,6 +869,7 @@ dependencies = [ "num-traits", "num_cpus", "once_cell", + "parking_lot", "percent-encoding", "pgp", "pretty_env_logger", diff --git a/Cargo.toml b/Cargo.toml index 9f85cf1f1..ecbab5a88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,6 @@ backtrace = "0.3" base64 = "0.21" bitflags = "1.3" chrono = { version = "0.4", default-features=false, features = ["clock", "std"] } -crossbeam-queue = "0.3" email = { git = "https://github.com/deltachat/rust-email", branch = "master" } encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" } escaper = "0.1" @@ -58,6 +57,7 @@ num-derive = "0.3" num-traits = "0.2" once_cell = "1.17.0" percent-encoding = "2.2" +parking_lot = "0.12" pgp = { version = "0.9", default-features = false } pretty_env_logger = { version = "0.4", optional = true } qrcodegen = "1.7.0" diff --git a/src/sql/pool.rs b/src/sql/pool.rs index fc7bf05bf..b7459976a 100644 --- a/src/sql/pool.rs +++ b/src/sql/pool.rs @@ -1,10 +1,18 @@ -//! Connection pool. +//! # SQLite connection pool. +//! +//! The connection pool holds a number of SQLite connections and allows to allocate them. +//! When allocated connection is dropped, underlying connection is returned back to the pool. +//! +//! The pool is organized as a stack. It always allocates the most recently used connection. +//! Each SQLite connection has its own page cache, so allocating recently used connections +//! improves the performance compared to, for example, organizing the pool as a queue +//! and returning the least recently used connection each time. use std::ops::{Deref, DerefMut}; use std::sync::{Arc, Weak}; use anyhow::{Context, Result}; -use crossbeam_queue::ArrayQueue; +use parking_lot::Mutex; use rusqlite::Connection; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; @@ -12,7 +20,7 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore}; #[derive(Debug)] struct InnerPool { /// Available connections. - connections: ArrayQueue, + connections: Mutex>, /// Counts the number of available connections. semaphore: Arc, @@ -23,7 +31,9 @@ impl InnerPool { /// /// The connection could be new or returned back. fn put(&self, connection: Connection) { - self.connections.force_push(connection); + let mut connections = self.connections.lock(); + connections.push(connection); + drop(connections); } } @@ -74,22 +84,19 @@ pub struct Pool { impl Pool { /// Creates a new connection pool. pub fn new(connections: Vec) -> Self { + let semaphore = Arc::new(Semaphore::new(connections.len())); let inner = Arc::new(InnerPool { - connections: ArrayQueue::new(connections.len()), - semaphore: Arc::new(Semaphore::new(connections.len())), + connections: Mutex::new(connections), + semaphore, }); - for connection in connections { - inner.connections.force_push(connection); - } Pool { inner } } /// Retrieves a connection from the pool. pub async fn get(&self) -> Result { let permit = self.inner.semaphore.clone().acquire_owned().await?; - let conn = self - .inner - .connections + let mut connections = self.inner.connections.lock(); + let conn = connections .pop() .context("got a permit when there are no connections in the pool")?; let conn = PooledConnection { From 604c4fcb714adf390d2ed9a59f545b6476dd663b Mon Sep 17 00:00:00 2001 From: iequidoo Date: Wed, 18 Jan 2023 13:35:19 -0300 Subject: [PATCH 02/30] Delete messages to the Trash folder for Gmail by default (#3957) Gmail archives messages marked as `\Deleted` by default if those messages aren't in the Trash. But if move them to the Trash instead, they will be auto-deleted in 30 days. --- CHANGELOG.md | 2 + python/tests/test_1_online.py | 29 +++++ src/config.rs | 37 ++++-- src/constants.rs | 2 +- src/context.rs | 40 ++++++- src/download.rs | 2 +- src/ephemeral.rs | 10 +- src/imap.rs | 215 ++++++++++++++++++++------------- src/imap/idle.rs | 5 +- src/imap/scan_folders.rs | 39 +++--- src/job.rs | 8 +- src/message.rs | 5 +- src/provider.rs | 10 ++ src/provider/data.rs | 67 ++++++++++- src/receive_imf.rs | 5 +- src/scheduler.rs | 217 ++++++++++++++++------------------ src/scheduler/connectivity.rs | 100 ++++++---------- 17 files changed, 483 insertions(+), 310 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e522e0a1..7199274ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 +- Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted. + Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972 ### API-Changes diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index b38afe88c..c25fcb85b 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1954,6 +1954,7 @@ def test_immediate_autodelete(acfactory, lp): assert msg.text == "hello" lp.sec("ac2: wait for close/expunge on autodelete") + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") ac2._evtracker.get_info_contains("close/expunge succeeded") lp.sec("ac2: check that message was autodeleted on server") @@ -1995,6 +1996,34 @@ def test_delete_multiple_messages(acfactory, lp): assert len(ac2.direct_imap.get_all_messages()) == 1 +def test_trash_multiple_messages(acfactory, lp): + ac1, ac2 = acfactory.get_online_accounts(2) + ac2.set_config("delete_to_trash", "1") + chat12 = acfactory.get_accepted_chat(ac1, ac2) + + lp.sec("ac1: sending 3 messages") + texts = ["first", "second", "third"] + for text in texts: + chat12.send_text(text) + + lp.sec("ac2: waiting for all messages on the other side") + to_delete = [] + for text in texts: + msg = ac2._evtracker.wait_next_incoming_message() + assert msg.text in texts + if text != "second": + to_delete.append(msg) + + lp.sec("ac2: deleting all messages except second") + assert len(to_delete) == len(texts) - 1 + ac2.delete_messages(to_delete) + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + + lp.sec("ac2: test that only one message is left") + ac2.direct_imap.select_config_folder("inbox") + assert len(ac2.direct_imap.get_all_messages()) == 1 + + def test_configure_error_msgs_wrong_pw(acfactory): configdict = acfactory.get_next_liveconfig() ac1 = acfactory.get_unconfigured_account() diff --git a/src/config.rs b/src/config.rs index 67d77811f..8c1357a8e 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,7 @@ //! # Key-value configuration management. +use std::str::FromStr; + use anyhow::{ensure, Context as _, Result}; use strum::{EnumProperty, IntoEnumIterator}; use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString}; @@ -173,6 +175,10 @@ pub enum Config { #[strum(props(default = "0"))] DeleteDeviceAfter, + /// Move messages to the Trash folder instead of marking them "\Deleted". Overrides + /// `ProviderOptions::delete_to_trash`. + DeleteToTrash, + /// Save raw MIME messages with headers in the database if true. SaveMimeHeaders, @@ -227,6 +233,9 @@ pub enum Config { /// Configured "Sent" folder. ConfiguredSentboxFolder, + /// Configured "Trash" folder. + ConfiguredTrashFolder, + /// Unix timestamp of the last successful configuration. ConfiguredTimestamp, @@ -327,30 +336,37 @@ impl Context { } } - /// Returns 32-bit signed integer configuration value for the given key. - pub async fn get_config_int(&self, key: Config) -> Result { + /// Returns Some(T) if a value for the given key exists and was successfully parsed. + /// Returns None if could not parse. + pub async fn get_config_parsed(&self, key: Config) -> Result> { self.get_config(key) .await - .map(|s: Option| s.and_then(|s| s.parse().ok()).unwrap_or_default()) + .map(|s: Option| s.and_then(|s| s.parse().ok())) + } + + /// Returns 32-bit signed integer configuration value for the given key. + pub async fn get_config_int(&self, key: Config) -> Result { + Ok(self.get_config_parsed(key).await?.unwrap_or_default()) } /// Returns 64-bit signed integer configuration value for the given key. pub async fn get_config_i64(&self, key: Config) -> Result { - self.get_config(key) - .await - .map(|s: Option| s.and_then(|s| s.parse().ok()).unwrap_or_default()) + Ok(self.get_config_parsed(key).await?.unwrap_or_default()) } /// Returns 64-bit unsigned integer configuration value for the given key. pub async fn get_config_u64(&self, key: Config) -> Result { - self.get_config(key) - .await - .map(|s: Option| s.and_then(|s| s.parse().ok()).unwrap_or_default()) + Ok(self.get_config_parsed(key).await?.unwrap_or_default()) + } + + /// Returns boolean configuration value (if any) for the given key. + pub async fn get_config_bool_opt(&self, key: Config) -> Result> { + Ok(self.get_config_parsed::(key).await?.map(|x| x != 0)) } /// Returns boolean configuration value for the given key. pub async fn get_config_bool(&self, key: Config) -> Result { - Ok(self.get_config_int(key).await? != 0) + Ok(self.get_config_bool_opt(key).await?.unwrap_or_default()) } /// Returns true if movebox ("DeltaChat" folder) should be watched. @@ -550,7 +566,6 @@ fn get_config_keys_string() -> String { #[cfg(test)] mod tests { - use std::str::FromStr; use std::string::ToString; use num_traits::FromPrimitive; diff --git a/src/constants.rs b/src/constants.rs index 2775f97ef..4c19abba6 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -201,7 +201,7 @@ pub const BALANCED_IMAGE_SIZE: u32 = 1280; pub const WORSE_IMAGE_SIZE: u32 = 640; // this value can be increased if the folder configuration is changed and must be redone on next program start -pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3; +pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4; #[cfg(test)] mod tests { diff --git a/src/context.rs b/src/context.rs index c672a1171..04687ad0c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -7,7 +7,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; -use anyhow::{bail, ensure, Result}; +use anyhow::{bail, ensure, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; use ratelimit::Ratelimit; use tokio::sync::{Mutex, RwLock}; @@ -623,6 +623,10 @@ impl Context { .get_config(Config::ConfiguredMvboxFolder) .await? .unwrap_or_else(|| "".to_string()); + let configured_trash_folder = self + .get_config(Config::ConfiguredTrashFolder) + .await? + .unwrap_or_else(|| "".to_string()); let mut res = get_info(); @@ -689,6 +693,7 @@ impl Context { res.insert("configured_inbox_folder", configured_inbox_folder); res.insert("configured_sentbox_folder", configured_sentbox_folder); res.insert("configured_mvbox_folder", configured_mvbox_folder); + res.insert("configured_trash_folder", configured_trash_folder); res.insert("mdns_enabled", mdns_enabled.to_string()); res.insert("e2ee_enabled", e2ee_enabled.to_string()); res.insert( @@ -722,6 +727,12 @@ impl Context { .await? .to_string(), ); + res.insert( + "delete_to_trash", + self.get_config(Config::DeleteToTrash) + .await? + .unwrap_or_else(|| "".to_string()), + ); res.insert( "last_housekeeping", self.get_config_int(Config::LastHousekeeping) @@ -887,6 +898,33 @@ impl Context { Ok(mvbox.as_deref() == Some(folder_name)) } + /// Returns true if given folder name is the name of the trash folder. + pub async fn is_trash(&self, folder_name: &str) -> Result { + let trash = self.get_config(Config::ConfiguredTrashFolder).await?; + Ok(trash.as_deref() == Some(folder_name)) + } + + pub(crate) async fn should_delete_to_trash(&self) -> Result { + if let Some(v) = self.get_config_bool_opt(Config::DeleteToTrash).await? { + return Ok(v); + } + if let Some(provider) = self.get_configured_provider().await? { + return Ok(provider.opt.delete_to_trash); + } + Ok(false) + } + + /// Returns `target` for deleted messages as per `imap` table. Empty string means "delete w/o + /// moving to trash". + pub(crate) async fn get_delete_msgs_target(&self) -> Result { + if !self.should_delete_to_trash().await? { + return Ok("".into()); + } + self.get_config(Config::ConfiguredTrashFolder) + .await? + .context("No configured trash folder") + } + pub(crate) fn derive_blobdir(dbfile: &Path) -> PathBuf { let mut blob_fname = OsString::new(); blob_fname.push(dbfile.file_name().unwrap_or_default()); diff --git a/src/download.rs b/src/download.rs index fb56f729c..5b544fe31 100644 --- a/src/download.rs +++ b/src/download.rs @@ -138,7 +138,7 @@ impl Job { context .sql .query_row_optional( - "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''", + "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target=folder", paramsv![msg.rfc724_mid], |row| { let server_uid: u32 = row.get(0)?; diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 2e38386ea..606c7270f 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -588,19 +588,25 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<() now - max(delete_server_after, MIN_DELETE_SERVER_AFTER), ), }; + let target = context.get_delete_msgs_target().await?; context .sql .execute( "UPDATE imap - SET target='' + SET target=? WHERE rfc724_mid IN ( SELECT rfc724_mid FROM msgs WHERE ((download_state = 0 AND timestamp < ?) OR (download_state != 0 AND timestamp < ?) OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?)) )", - paramsv![threshold_timestamp, threshold_timestamp_extended, now], + paramsv![ + target, + threshold_timestamp, + threshold_timestamp_extended, + now, + ], ) .await?; diff --git a/src/imap.rs b/src/imap.rs index 56a2cc6e6..5cdecae9b 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -113,13 +113,15 @@ impl async_imap::Authenticator for OAuth2 { } } -#[derive(Debug, PartialEq, Clone, Copy)] -enum FolderMeaning { +#[derive(Debug, Display, PartialEq, Eq, Clone, Copy)] +pub enum FolderMeaning { Unknown, Spam, + Inbox, + Mvbox, Sent, + Trash, Drafts, - Other, /// Virtual folders. /// @@ -131,13 +133,15 @@ enum FolderMeaning { } impl FolderMeaning { - fn to_config(self) -> Option { + pub fn to_config(self) -> Option { match self { FolderMeaning::Unknown => None, FolderMeaning::Spam => None, + FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder), + FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder), FolderMeaning::Sent => Some(Config::ConfiguredSentboxFolder), + FolderMeaning::Trash => Some(Config::ConfiguredTrashFolder), FolderMeaning::Drafts => None, - FolderMeaning::Other => None, FolderMeaning::Virtual => None, } } @@ -449,7 +453,7 @@ impl Imap { &mut self, context: &Context, watch_folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, ) -> Result<()> { if !context.sql.is_open().await { // probably shutdown @@ -458,7 +462,7 @@ impl Imap { self.prepare(context).await?; let msgs_fetched = self - .fetch_new_messages(context, watch_folder, is_spam_folder, false) + .fetch_new_messages(context, watch_folder, folder_meaning, false) .await .context("fetch_new_messages")?; if msgs_fetched && context.get_config_delete_device_after().await?.is_some() { @@ -490,49 +494,60 @@ impl Imap { pub(crate) async fn resync_folder_uids( &mut self, context: &Context, - folder: String, + folder: &str, + folder_meaning: FolderMeaning, ) -> Result<()> { // Collect pairs of UID and Message-ID. - let mut msg_ids = BTreeMap::new(); + let mut msgs = BTreeMap::new(); let session = self .session .as_mut() .context("IMAP No connection established")?; - session.select_folder(context, Some(&folder)).await?; + session.select_folder(context, Some(folder)).await?; let mut list = session .uid_fetch("1:*", RFC724MID_UID) .await .with_context(|| format!("can't resync folder {folder}"))?; while let Some(fetch) = list.next().await { - let msg = fetch?; + let fetch = fetch?; + let headers = match get_fetch_headers(&fetch) { + Ok(headers) => headers, + Err(err) => { + warn!(context, "Failed to parse FETCH headers: {}", err); + continue; + } + }; + let message_id = prefetch_get_message_id(&headers); - // Get Message-ID - let message_id = - get_fetch_headers(&msg).map_or(None, |headers| prefetch_get_message_id(&headers)); - - if let (Some(uid), Some(rfc724_mid)) = (msg.uid, message_id) { - msg_ids.insert(uid, rfc724_mid); + if let (Some(uid), Some(rfc724_mid)) = (fetch.uid, message_id) { + msgs.insert( + uid, + ( + rfc724_mid, + target_folder(context, folder, folder_meaning, &headers).await?, + ), + ); } } info!( context, "Resync: collected {} message IDs in folder {}", - msg_ids.len(), - &folder + msgs.len(), + folder, ); - let uid_validity = get_uidvalidity(context, &folder).await?; + let uid_validity = get_uidvalidity(context, folder).await?; // Write collected UIDs to SQLite database. context .sql .transaction(move |transaction| { transaction.execute("DELETE FROM imap WHERE folder=?", params![folder])?; - for (uid, rfc724_mid) in &msg_ids { + for (uid, (rfc724_mid, target)) in &msgs { // This may detect previously undetected moved // messages, so we update server_folder too. transaction.execute( @@ -541,7 +556,7 @@ impl Imap { ON CONFLICT(folder, uid, uidvalidity) DO UPDATE SET rfc724_mid=excluded.rfc724_mid, target=excluded.target", - params![rfc724_mid, folder, uid, uid_validity, folder], + params![rfc724_mid, folder, uid, uid_validity, target], )?; } Ok(()) @@ -683,10 +698,10 @@ impl Imap { &mut self, context: &Context, folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, fetch_existing_msgs: bool, ) -> Result { - if should_ignore_folder(context, folder, is_spam_folder).await? { + if should_ignore_folder(context, folder, folder_meaning).await? { info!(context, "Not fetching from {}", folder); return Ok(false); } @@ -732,14 +747,7 @@ impl Imap { // Get the Message-ID or generate a fake one to identify the message in the database. let message_id = prefetch_get_or_create_message_id(&headers); - - let target = match target_folder(context, folder, is_spam_folder, &headers).await? { - Some(config) => match context.get_config(config).await? { - Some(target) => target, - None => folder.to_string(), - }, - None => folder.to_string(), - }; + let target = target_folder(context, folder, folder_meaning, &headers).await?; context .sql @@ -763,8 +771,8 @@ impl Imap { // Never download messages directly from the spam folder. // If the sender is known, the message will be moved to the Inbox or Mvbox // and then we download the message from there. - // Also see `spam_target_folder()`. - && !is_spam_folder + // Also see `spam_target_folder_cfg()`. + && folder_meaning != FolderMeaning::Spam && prefetch_should_download( context, &headers, @@ -870,17 +878,21 @@ impl Imap { .context("failed to get recipients from the inbox")?; if context.get_config_bool(Config::FetchExistingMsgs).await? { - for config in &[ - Config::ConfiguredMvboxFolder, - Config::ConfiguredInboxFolder, - Config::ConfiguredSentboxFolder, + for meaning in [ + FolderMeaning::Mvbox, + FolderMeaning::Inbox, + FolderMeaning::Sent, ] { - if let Some(folder) = context.get_config(*config).await? { + let config = match meaning.to_config() { + Some(c) => c, + None => continue, + }; + if let Some(folder) = context.get_config(config).await? { info!( context, "Fetching existing messages from folder \"{}\"", folder ); - self.fetch_new_messages(context, &folder, false, true) + self.fetch_new_messages(context, &folder, meaning, true) .await .context("could not fetch existing messages")?; } @@ -952,44 +964,60 @@ impl Session { return Ok(()); } Err(err) => { + if context.should_delete_to_trash().await? { + error!( + context, + "Cannot move messages {} to {}, no fallback to COPY/DELETE because \ + delete_to_trash is set. Error: {:#}", + set, + target, + err, + ); + return Err(err.into()); + } warn!( context, - "Cannot move message, fallback to COPY/DELETE {} to {}: {}", + "Cannot move messages, fallback to COPY/DELETE {} to {}: {}", set, target, err ); } } - } else { + } + + // Server does not support MOVE or MOVE failed. + // Copy messages to the destination folder if needed and mark records for deletion. + let copy = !context.is_trash(target).await?; + if copy { info!( context, "Server does not support MOVE, fallback to COPY/DELETE {} to {}", set, target ); + self.uid_copy(&set, &target).await?; + } else { + error!( + context, + "Server does not support MOVE, fallback to DELETE {} to {}", set, target, + ); } - - // Server does not support MOVE or MOVE failed. - // Copy the message to the destination folder and mark the record for deletion. - match self.uid_copy(&set, &target).await { - Ok(()) => { - context - .sql - .execute( - &format!( - "UPDATE imap SET target='' WHERE id IN ({})", - sql::repeat_vars(row_ids.len()) - ), - rusqlite::params_from_iter(row_ids), - ) - .await - .context("cannot plan deletion of copied messages")?; - context.emit_event(EventType::ImapMessageMoved(format!( - "IMAP messages {set} copied to {target}" - ))); - Ok(()) - } - Err(err) => Err(err.into()), + context + .sql + .execute( + &format!( + "UPDATE imap SET target='' WHERE id IN ({})", + sql::repeat_vars(row_ids.len()) + ), + rusqlite::params_from_iter(row_ids), + ) + .await + .context("cannot plan deletion of messages")?; + if copy { + context.emit_event(EventType::ImapMessageMoved(format!( + "IMAP messages {set} copied to {target}" + ))); } + Ok(()) } /// Moves and deletes messages as planned in the `imap` table. @@ -1644,7 +1672,7 @@ impl Imap { } } - let folder_meaning = get_folder_meaning(&folder); + let folder_meaning = get_folder_meaning_by_attrs(folder.attributes()); let folder_name_meaning = get_folder_meaning_by_name(folder.name()); if let Some(config) = folder_meaning.to_config() { // Always takes precedence @@ -1776,7 +1804,7 @@ async fn should_move_out_of_spam( /// If this returns None, the message will not be moved out of the /// Spam folder, and as `fetch_new_messages()` doesn't download /// messages from the Spam folder, the message will be ignored. -async fn spam_target_folder( +async fn spam_target_folder_cfg( context: &Context, headers: &[mailparse::MailHeader<'_>], ) -> Result> { @@ -1797,18 +1825,18 @@ async fn spam_target_folder( /// Returns `ConfiguredInboxFolder`, `ConfiguredMvboxFolder` or `ConfiguredSentboxFolder` if /// the message needs to be moved from `folder`. Otherwise returns `None`. -pub async fn target_folder( +pub async fn target_folder_cfg( context: &Context, folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, headers: &[mailparse::MailHeader<'_>], ) -> Result> { if context.is_mvbox(folder).await? { return Ok(None); } - if is_spam_folder { - spam_target_folder(context, headers).await + if folder_meaning == FolderMeaning::Spam { + spam_target_folder_cfg(context, headers).await } else if needs_move_to_mvbox(context, headers).await? { Ok(Some(Config::ConfiguredMvboxFolder)) } else { @@ -1816,6 +1844,21 @@ pub async fn target_folder( } } +pub async fn target_folder( + context: &Context, + folder: &str, + folder_meaning: FolderMeaning, + headers: &[mailparse::MailHeader<'_>], +) -> Result { + match target_folder_cfg(context, folder, folder_meaning, headers).await? { + Some(config) => match context.get_config(config).await? { + Some(target) => Ok(target), + None => Ok(folder.to_string()), + }, + None => Ok(folder.to_string()), + } +} + async fn needs_move_to_mvbox( context: &Context, headers: &[mailparse::MailHeader<'_>], @@ -1940,10 +1983,10 @@ fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning { } } -fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { - for attr in folder_name.attributes() { +fn get_folder_meaning_by_attrs(folder_attrs: &[NameAttribute]) -> FolderMeaning { + for attr in folder_attrs { match attr { - NameAttribute::Trash => return FolderMeaning::Other, + NameAttribute::Trash => return FolderMeaning::Trash, NameAttribute::Sent => return FolderMeaning::Sent, NameAttribute::Junk => return FolderMeaning::Spam, NameAttribute::Drafts => return FolderMeaning::Drafts, @@ -1961,6 +2004,13 @@ fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { FolderMeaning::Unknown } +pub(crate) fn get_folder_meaning(folder: &Name) -> FolderMeaning { + match get_folder_meaning_by_attrs(folder.attributes()) { + FolderMeaning::Unknown => get_folder_meaning_by_name(folder.name()), + meaning => meaning, + } +} + /// Parses the headers from the FETCH result. fn get_fetch_headers(prefetch_msg: &Fetch) -> Result> { match prefetch_msg.header() { @@ -2272,7 +2322,7 @@ pub async fn get_config_last_seen_uid(context: &Context, folder: &str) -> Result async fn should_ignore_folder( context: &Context, folder: &str, - is_spam_folder: bool, + folder_meaning: FolderMeaning, ) -> Result { if !context.get_config_bool(Config::OnlyFetchMvbox).await? { return Ok(false); @@ -2281,7 +2331,7 @@ async fn should_ignore_folder( // Still respect the SentboxWatch setting. return Ok(!context.get_config_bool(Config::SentboxWatch).await?); } - Ok(!(context.is_mvbox(folder).await? || is_spam_folder)) + Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam)) } /// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000 @@ -2564,14 +2614,13 @@ mod tests { }; let (headers, _) = mailparse::parse_headers(bytes)?; - - let is_spam_folder = folder == "Spam"; - let actual = - if let Some(config) = target_folder(&t, folder, is_spam_folder, &headers).await? { - t.get_config(config).await? - } else { - None - }; + let actual = if let Some(config) = + target_folder_cfg(&t, folder, get_folder_meaning_by_name(folder), &headers).await? + { + t.get_config(config).await? + } else { + None + }; let expected = if expected_destination == folder { None diff --git a/src/imap/idle.rs b/src/imap/idle.rs index 4c0910fbe..fbce499b7 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -7,7 +7,7 @@ use futures_lite::FutureExt; use super::session::Session; use super::Imap; -use crate::imap::client::IMAP_TIMEOUT; +use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning}; use crate::{context::Context, scheduler::InterruptInfo}; const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60); @@ -113,6 +113,7 @@ impl Imap { &mut self, context: &Context, watch_folder: Option, + folder_meaning: FolderMeaning, ) -> InterruptInfo { // Idle using polling. This is also needed if we're not yet configured - // in this case, we're waiting for a configure job (and an interrupt). @@ -173,7 +174,7 @@ impl Imap { // will have already fetched the messages so perform_*_fetch // will not find any new. match self - .fetch_new_messages(context, &watch_folder, false, false) + .fetch_new_messages(context, &watch_folder, folder_meaning, false) .await { Ok(res) => { diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs index 991b6b38f..29051e7b9 100644 --- a/src/imap/scan_folders.rs +++ b/src/imap/scan_folders.rs @@ -3,7 +3,7 @@ use std::{collections::BTreeMap, time::Instant}; use anyhow::{Context as _, Result}; use futures::stream::StreamExt; -use super::{get_folder_meaning, get_folder_meaning_by_name}; +use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name}; use crate::config::Config; use crate::imap::Imap; use crate::log::LogExt; @@ -33,7 +33,7 @@ impl Imap { let mut folder_configs = BTreeMap::new(); for folder in folders { - let folder_meaning = get_folder_meaning(&folder); + let folder_meaning = get_folder_meaning_by_attrs(folder.attributes()); if folder_meaning == FolderMeaning::Virtual { // Gmail has virtual folders that should be skipped. For example, // emails appear in the inbox and under "All Mail" as soon as it is @@ -53,21 +53,22 @@ impl Imap { .or_insert_with(|| folder.name().to_string()); } - let is_drafts = folder_meaning == FolderMeaning::Drafts - || (folder_meaning == FolderMeaning::Unknown - && folder_name_meaning == FolderMeaning::Drafts); - let is_spam_folder = folder_meaning == FolderMeaning::Spam - || (folder_meaning == FolderMeaning::Unknown - && folder_name_meaning == FolderMeaning::Spam); + let folder_meaning = match folder_meaning { + FolderMeaning::Unknown => folder_name_meaning, + _ => folder_meaning, + }; // Don't scan folders that are watched anyway - if !watched_folders.contains(&folder.name().to_string()) && !is_drafts { + if !watched_folders.contains(&folder.name().to_string()) + && folder_meaning != FolderMeaning::Drafts + && folder_meaning != FolderMeaning::Trash + { let session = self.session.as_mut().context("no session")?; // Drain leftover unsolicited EXISTS messages session.server_sent_unsolicited_exists(context)?; loop { - self.fetch_move_delete(context, folder.name(), is_spam_folder) + self.fetch_move_delete(context, folder.name(), folder_meaning) .await .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder"); @@ -80,15 +81,15 @@ impl Imap { } } - // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted. - context - .set_config( - Config::ConfiguredSentboxFolder, - folder_configs - .get(&Config::ConfiguredSentboxFolder) - .map(|s| s.as_str()), - ) - .await?; + // Set configs for necessary folders. Or reset if the folder was deleted. + for conf in [ + Config::ConfiguredSentboxFolder, + Config::ConfiguredTrashFolder, + ] { + context + .set_config(conf, folder_configs.get(&conf).map(|s| s.as_str())) + .await?; + } last_scan.replace(Instant::now()); Ok(true) diff --git a/src/job.rs b/src/job.rs index 51c06d372..3279843a9 100644 --- a/src/job.rs +++ b/src/job.rs @@ -12,7 +12,7 @@ use deltachat_derive::{FromSql, ToSql}; use rand::{thread_rng, Rng}; use crate::context::Context; -use crate::imap::Imap; +use crate::imap::{get_folder_meaning, FolderMeaning, Imap}; use crate::param::Params; use crate::scheduler::InterruptInfo; use crate::tools::time; @@ -172,8 +172,12 @@ impl Job { let mut any_failed = false; for folder in all_folders { + let folder_meaning = get_folder_meaning(&folder); + if folder_meaning == FolderMeaning::Virtual { + continue; + } if let Err(e) = imap - .resync_folder_uids(context, folder.name().to_string()) + .resync_folder_uids(context, folder.name(), folder_meaning) .await { warn!(context, "{:#}", e); diff --git a/src/message.rs b/src/message.rs index e4bae9a67..a7b2e27e3 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1386,11 +1386,12 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> { context.emit_event(EventType::WebxdcInstanceDeleted { msg_id: *msg_id }); } + let target = context.get_delete_msgs_target().await?; context .sql .execute( - "UPDATE imap SET target='' WHERE rfc724_mid=?", - paramsv![msg.rfc724_mid], + "UPDATE imap SET target=? WHERE rfc724_mid=?", + paramsv![target, msg.rfc724_mid], ) .await?; diff --git a/src/provider.rs b/src/provider.rs index 565d54d10..b461e600c 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -138,6 +138,16 @@ pub struct Provider { /// Type of OAuth 2 authorization if provider supports it. pub oauth2_authorizer: Option, + + /// Options with good defaults. + pub opt: ProviderOptions, +} + +/// Provider options with good defaults. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct ProviderOptions { + /// Move messages to the Trash folder instead of marking them "\Deleted". + pub delete_to_trash: bool, } /// Get resolver to query MX records. diff --git a/src/provider/data.rs b/src/provider/data.rs index 21084f821..49a0d30a9 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -7,7 +7,9 @@ use once_cell::sync::Lazy; use crate::provider::Protocol::*; use crate::provider::Socket::*; use crate::provider::UsernamePattern::*; -use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status}; +use crate::provider::{ + Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status, +}; // 163.md: 163.com static P_163: Lazy = Lazy::new(|| Provider { @@ -36,6 +38,7 @@ static P_163: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // aktivix.org.md: aktivix.org @@ -65,6 +68,7 @@ static P_AKTIVIX_ORG: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // aol.md: aol.com @@ -83,6 +87,7 @@ static P_AOL: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -113,6 +118,7 @@ static P_ARCOR_DE: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // autistici.org.md: autistici.org @@ -142,6 +148,7 @@ static P_AUTISTICI_ORG: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // blindzeln.org.md: delta.blinzeln.de, delta.blindzeln.org @@ -171,6 +178,7 @@ static P_BLINDZELN_ORG: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // bluewin.ch.md: bluewin.ch @@ -200,6 +208,7 @@ static P_BLUEWIN_CH: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // buzon.uy.md: buzon.uy @@ -229,6 +238,7 @@ static P_BUZON_UY: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // chello.at.md: chello.at @@ -258,6 +268,7 @@ static P_CHELLO_AT: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // comcast.md: xfinity.com, comcast.net @@ -272,6 +283,7 @@ static P_COMCAST: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // dismail.de.md: dismail.de @@ -286,6 +298,7 @@ static P_DISMAIL_DE: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // disroot.md: disroot.org @@ -315,6 +328,7 @@ static P_DISROOT: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // e.email.md: e.email @@ -344,6 +358,7 @@ static P_E_EMAIL: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // espiv.net.md: espiv.net @@ -358,6 +373,7 @@ static P_ESPIV_NET: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // example.com.md: example.com, example.org, example.net @@ -376,6 +392,7 @@ static P_EXAMPLE_COM: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -407,6 +424,7 @@ static P_FASTMAIL: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // firemail.de.md: firemail.at, firemail.de @@ -423,6 +441,7 @@ static P_FIREMAIL_DE: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -451,6 +470,7 @@ static P_FIVE_CHAT: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // freenet.de.md: freenet.de @@ -469,6 +489,7 @@ static P_FREENET_DE: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -488,6 +509,9 @@ static P_GMAIL: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: Some(Oauth2Authorizer::Gmail), + opt: ProviderOptions { + delete_to_trash: true, + }, } }); @@ -525,6 +549,7 @@ static P_GMX_NET: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, ec.hermes.radio, ec1.hermes.radio, ec2.hermes.radio, ec3.hermes.radio, ec4.hermes.radio, ec5.hermes.radio, ec6.hermes.radio, ec7.hermes.radio, ec8.hermes.radio, ec9.hermes.radio, ec10.hermes.radio, ec11.hermes.radio, ec12.hermes.radio, ec13.hermes.radio, ec14.hermes.radio, ec15.hermes.radio, hermes.radio @@ -552,6 +577,7 @@ static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { strict_tls: false, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // hey.com.md: hey.com @@ -568,6 +594,7 @@ static P_HEY_COM: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -583,6 +610,7 @@ static P_I_UA: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // i3.net.md: i3.net @@ -597,6 +625,7 @@ static P_I3_NET: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // icloud.md: icloud.com, me.com, mac.com @@ -626,6 +655,7 @@ static P_ICLOUD: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // infomaniak.com.md: ik.me @@ -655,6 +685,7 @@ static P_INFOMANIAK_COM: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: Some(10), oauth2_authorizer: None, + opt: Default::default(), }); // kolst.com.md: kolst.com @@ -669,6 +700,7 @@ static P_KOLST_COM: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // kontent.com.md: kontent.com @@ -683,6 +715,7 @@ static P_KONTENT_COM: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // mail.de.md: mail.de @@ -712,6 +745,7 @@ static P_MAIL_DE: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // mail.ru.md: mail.ru, inbox.ru, internet.ru, bk.ru, list.ru @@ -730,6 +764,7 @@ static P_MAIL_RU: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -760,6 +795,7 @@ static P_MAIL2TOR: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // mailbox.org.md: mailbox.org, secure.mailbox.org @@ -789,6 +825,7 @@ static P_MAILBOX_ORG: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // mailo.com.md: mailo.com @@ -818,6 +855,7 @@ static P_MAILO_COM: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // nauta.cu.md: nauta.cu @@ -872,6 +910,7 @@ static P_NAUTA_CU: Lazy = Lazy::new(|| Provider { strict_tls: false, max_smtp_rcpt_to: Some(20), oauth2_authorizer: None, + opt: Default::default(), }); // naver.md: naver.com @@ -901,6 +940,7 @@ static P_NAVER: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // nubo.coop.md: nubo.coop @@ -930,6 +970,7 @@ static P_NUBO_COOP: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com, outlook.de @@ -959,6 +1000,7 @@ static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // ouvaton.coop.md: ouvaton.org @@ -988,6 +1030,7 @@ static P_OUVATON_COOP: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us @@ -1017,6 +1060,7 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // protonmail.md: protonmail.com, protonmail.ch, pm.me @@ -1033,6 +1077,7 @@ static P_PROTONMAIL: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1052,6 +1097,7 @@ static P_QQ: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1082,6 +1128,7 @@ static P_RISEUP_NET: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // rogers.com.md: rogers.com @@ -1096,6 +1143,7 @@ static P_ROGERS_COM: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // systemausfall.org.md: systemausfall.org, solidaris.me @@ -1125,6 +1173,7 @@ static P_SYSTEMAUSFALL_ORG: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // systemli.org.md: systemli.org @@ -1154,6 +1203,7 @@ static P_SYSTEMLI_ORG: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // t-online.md: t-online.de, magenta.de @@ -1172,6 +1222,7 @@ static P_T_ONLINE: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1222,6 +1273,7 @@ static P_TESTRUN: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // tiscali.it.md: tiscali.it @@ -1251,6 +1303,7 @@ static P_TISCALI_IT: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // tutanota.md: tutanota.com, tutanota.de, tutamail.com, tuta.io, keemail.me @@ -1267,6 +1320,7 @@ static P_TUTANOTA: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1282,6 +1336,7 @@ static P_UKR_NET: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // undernet.uy.md: undernet.uy @@ -1311,6 +1366,7 @@ static P_UNDERNET_UY: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // vfemail.md: vfemail.net @@ -1325,6 +1381,7 @@ static P_VFEMAIL: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // vivaldi.md: vivaldi.net @@ -1354,6 +1411,7 @@ static P_VIVALDI: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // vodafone.de.md: vodafone.de, vodafonemail.de @@ -1383,6 +1441,7 @@ static P_VODAFONE_DE: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms @@ -1402,6 +1461,7 @@ static P_WEB_DE: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1421,6 +1481,7 @@ static P_YAHOO: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1451,6 +1512,7 @@ static P_YANDEX_RU: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: Some(Oauth2Authorizer::Yandex), + opt: Default::default(), }); // yggmail.md: yggmail @@ -1471,6 +1533,7 @@ static P_YGGMAIL: Lazy = Lazy::new(|| { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), } }); @@ -1501,6 +1564,7 @@ static P_ZIGGO_NL: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); // zoho.md: zohomail.eu, zohomail.com, zoho.com @@ -1530,6 +1594,7 @@ static P_ZOHO: Lazy = Lazy::new(|| Provider { strict_tls: true, max_smtp_rcpt_to: None, oauth2_authorizer: None, + opt: Default::default(), }); pub(crate) static PROVIDER_DATA: Lazy> = Lazy::new(|| { diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 95db04d9c..1896e844b 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -342,11 +342,12 @@ pub(crate) async fn receive_imf_inner( if received_msg.needs_delete_job || (delete_server_after == Some(0) && is_partial_download.is_none()) { + let target = context.get_delete_msgs_target().await?; context .sql .execute( - "UPDATE imap SET target='' WHERE rfc724_mid=?", - paramsv![rfc724_mid], + "UPDATE imap SET target=? WHERE rfc724_mid=?", + paramsv![target, rfc724_mid], ) .await?; } else if !mime_parser.mdn_reports.is_empty() && mime_parser.has_chat_version() { diff --git a/src/scheduler.rs b/src/scheduler.rs index 356c40e07..df9312c4e 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,6 +1,8 @@ +use std::iter::{self, once}; + use anyhow::{bail, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; -use futures::try_join; +use futures::future::try_join_all; use futures_lite::FutureExt; use tokio::task; @@ -9,7 +11,7 @@ use crate::config::Config; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; use crate::ephemeral::{self, delete_expired_imap_messages}; -use crate::imap::Imap; +use crate::imap::{FolderMeaning, Imap}; use crate::job; use crate::location; use crate::log::LogExt; @@ -20,15 +22,19 @@ use crate::tools::{duration_to_str, maybe_add_time_based_warnings}; pub(crate) mod connectivity; +#[derive(Debug)] +struct SchedBox { + meaning: FolderMeaning, + conn_state: ImapConnectionState, + handle: task::JoinHandle<()>, +} + /// Job and connection scheduler. #[derive(Debug)] pub(crate) struct Scheduler { - inbox: ImapConnectionState, - inbox_handle: task::JoinHandle<()>, - mvbox: ImapConnectionState, - mvbox_handle: Option>, - sentbox: ImapConnectionState, - sentbox_handle: Option>, + inbox: SchedBox, + /// Optional boxes -- mvbox, sentbox. + oboxes: Vec, smtp: SmtpConnectionState, smtp_handle: task::JoinHandle<()>, ephemeral_handle: task::JoinHandle<()>, @@ -161,7 +167,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne } } - info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await; + info = fetch_idle(&ctx, &mut connection, FolderMeaning::Inbox).await; } } } @@ -182,7 +188,20 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne /// handling all the errors. In case of an error, it is logged, but not propagated upwards. If /// critical operation fails such as fetching new messages fails, connection is reset via /// `trigger_reconnect`, so a fresh one can be opened. -async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) -> InterruptInfo { +async fn fetch_idle( + ctx: &Context, + connection: &mut Imap, + folder_meaning: FolderMeaning, +) -> InterruptInfo { + let folder_config = match folder_meaning.to_config() { + Some(c) => c, + None => { + error!(ctx, "Bad folder meaning: {}", folder_meaning); + return connection + .fake_idle(ctx, None, FolderMeaning::Unknown) + .await; + } + }; let folder = match ctx.get_config(folder_config).await { Ok(folder) => folder, Err(err) => { @@ -190,7 +209,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) ctx, "Can not watch {} folder, failed to retrieve config: {:#}", folder_config, err ); - return connection.fake_idle(ctx, None).await; + return connection + .fake_idle(ctx, None, FolderMeaning::Unknown) + .await; } }; @@ -199,7 +220,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) } else { connection.connectivity.set_not_configured(ctx).await; info!(ctx, "Can not watch {} folder, not set", folder_config); - return connection.fake_idle(ctx, None).await; + return connection + .fake_idle(ctx, None, FolderMeaning::Unknown) + .await; }; // connect and fake idle if unable to connect @@ -210,7 +233,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) { warn!(ctx, "{:#}", err); connection.trigger_reconnect(ctx); - return connection.fake_idle(ctx, Some(watch_folder)).await; + return connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await; } if folder_config == Config::ConfiguredInboxFolder { @@ -227,7 +252,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) // Fetch the watched folder. if let Err(err) = connection - .fetch_move_delete(ctx, &watch_folder, false) + .fetch_move_delete(ctx, &watch_folder, folder_meaning) .await .context("fetch_move_delete") { @@ -265,7 +290,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) // no new messages. We want to select the watched folder anyway before going IDLE // there, so this does not take additional protocol round-trip. if let Err(err) = connection - .fetch_move_delete(ctx, &watch_folder, false) + .fetch_move_delete(ctx, &watch_folder, folder_meaning) .await .context("fetch_move_delete after scan_folders") { @@ -293,7 +318,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) ctx, "IMAP session does not support IDLE, going to fake idle." ); - return connection.fake_idle(ctx, Some(watch_folder)).await; + return connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await; } info!(ctx, "IMAP session supports IDLE, using it."); @@ -318,7 +345,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_config: Config) } } else { warn!(ctx, "No IMAP session, going to fake idle."); - connection.fake_idle(ctx, Some(watch_folder)).await + connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await } } @@ -326,11 +355,11 @@ async fn simple_imap_loop( ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers, - folder_config: Config, + folder_meaning: FolderMeaning, ) { use futures::future::FutureExt; - info!(ctx, "starting simple loop for {}", folder_config); + info!(ctx, "starting simple loop for {}", folder_meaning); let ImapConnectionHandlers { mut connection, stop_receiver, @@ -346,7 +375,7 @@ async fn simple_imap_loop( } loop { - fetch_idle(&ctx, &mut connection, folder_config).await; + fetch_idle(&ctx, &mut connection, folder_meaning).await; } }; @@ -443,75 +472,56 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect impl Scheduler { /// Start the scheduler. pub async fn start(ctx: Context) -> Result { - let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?; - let (sentbox, sentbox_handlers) = ImapConnectionState::new(&ctx).await?; let (smtp, smtp_handlers) = SmtpConnectionState::new(); - let (inbox, inbox_handlers) = ImapConnectionState::new(&ctx).await?; - let (inbox_start_send, inbox_start_recv) = channel::bounded(1); - let (mvbox_start_send, mvbox_start_recv) = channel::bounded(1); - let mut mvbox_handle = None; - let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1); - let mut sentbox_handle = None; let (smtp_start_send, smtp_start_recv) = channel::bounded(1); let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1); let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1); - let inbox_handle = { + let mut oboxes = Vec::new(); + let mut start_recvs = Vec::new(); + + let (conn_state, inbox_handlers) = ImapConnectionState::new(&ctx).await?; + let (inbox_start_send, inbox_start_recv) = channel::bounded(1); + let handle = { let ctx = ctx.clone(); task::spawn(async move { inbox_loop(ctx, inbox_start_send, inbox_handlers).await }) }; + let inbox = SchedBox { + meaning: FolderMeaning::Inbox, + conn_state, + handle, + }; + start_recvs.push(inbox_start_recv); - if ctx.should_watch_mvbox().await? { - let ctx = ctx.clone(); - mvbox_handle = Some(task::spawn(async move { - simple_imap_loop( - ctx, - mvbox_start_send, - mvbox_handlers, - Config::ConfiguredMvboxFolder, - ) - .await - })); - } else { - mvbox_start_send - .send(()) - .await - .context("mvbox start send, missing receiver")?; - mvbox_handlers - .connection - .connectivity - .set_not_configured(&ctx) - .await - } - - if ctx.get_config_bool(Config::SentboxWatch).await? { - let ctx = ctx.clone(); - sentbox_handle = Some(task::spawn(async move { - simple_imap_loop( - ctx, - sentbox_start_send, - sentbox_handlers, - Config::ConfiguredSentboxFolder, - ) - .await - })); - } else { - sentbox_start_send - .send(()) - .await - .context("sentbox start send, missing receiver")?; - sentbox_handlers - .connection - .connectivity - .set_not_configured(&ctx) - .await + for (meaning, should_watch) in [ + (FolderMeaning::Mvbox, ctx.should_watch_mvbox().await), + ( + FolderMeaning::Sent, + ctx.get_config_bool(Config::SentboxWatch).await, + ), + ] { + if should_watch? { + let (conn_state, handlers) = ImapConnectionState::new(&ctx).await?; + let (start_send, start_recv) = channel::bounded(1); + let ctx = ctx.clone(); + let handle = task::spawn(async move { + simple_imap_loop(ctx, start_send, handlers, meaning).await + }); + oboxes.push(SchedBox { + meaning, + conn_state, + handle, + }); + start_recvs.push(start_recv); + } } let smtp_handle = { let ctx = ctx.clone(); task::spawn(async move { smtp_loop(ctx, smtp_start_send, smtp_handlers).await }) }; + start_recvs.push(smtp_start_recv); let ephemeral_handle = { let ctx = ctx.clone(); @@ -531,12 +541,8 @@ impl Scheduler { let res = Self { inbox, - mvbox, - sentbox, + oboxes, smtp, - inbox_handle, - mvbox_handle, - sentbox_handle, smtp_handle, ephemeral_handle, ephemeral_interrupt_send, @@ -546,12 +552,7 @@ impl Scheduler { }; // wait for all loops to be started - if let Err(err) = try_join!( - inbox_start_recv.recv(), - mvbox_start_recv.recv(), - sentbox_start_recv.recv(), - smtp_start_recv.recv() - ) { + if let Err(err) = try_join_all(start_recvs.iter().map(|r| r.recv())).await { bail!("failed to start scheduler: {}", err); } @@ -559,30 +560,26 @@ impl Scheduler { Ok(res) } + fn boxes(&self) -> iter::Chain, std::slice::Iter<'_, SchedBox>> { + once(&self.inbox).chain(self.oboxes.iter()) + } + fn maybe_network(&self) { - self.interrupt_inbox(InterruptInfo::new(true)); - self.interrupt_mvbox(InterruptInfo::new(true)); - self.interrupt_sentbox(InterruptInfo::new(true)); + for b in self.boxes() { + b.conn_state.interrupt(InterruptInfo::new(true)); + } self.interrupt_smtp(InterruptInfo::new(true)); } fn maybe_network_lost(&self) { - self.interrupt_inbox(InterruptInfo::new(false)); - self.interrupt_mvbox(InterruptInfo::new(false)); - self.interrupt_sentbox(InterruptInfo::new(false)); + for b in self.boxes() { + b.conn_state.interrupt(InterruptInfo::new(false)); + } self.interrupt_smtp(InterruptInfo::new(false)); } fn interrupt_inbox(&self, info: InterruptInfo) { - self.inbox.interrupt(info); - } - - fn interrupt_mvbox(&self, info: InterruptInfo) { - self.mvbox.interrupt(info); - } - - fn interrupt_sentbox(&self, info: InterruptInfo) { - self.sentbox.interrupt(info); + self.inbox.conn_state.interrupt(info); } fn interrupt_smtp(&self, info: InterruptInfo) { @@ -605,29 +602,17 @@ impl Scheduler { /// /// It consumes the scheduler and never fails to stop it. In the worst case, long-running tasks /// are forcefully terminated if they cannot shutdown within the timeout. - pub(crate) async fn stop(mut self, context: &Context) { + pub(crate) async fn stop(self, context: &Context) { // Send stop signals to tasks so they can shutdown cleanly. - self.inbox.stop().await.ok_or_log(context); - if self.mvbox_handle.is_some() { - self.mvbox.stop().await.ok_or_log(context); - } - if self.sentbox_handle.is_some() { - self.sentbox.stop().await.ok_or_log(context); + for b in self.boxes() { + b.conn_state.stop().await.ok_or_log(context); } self.smtp.stop().await.ok_or_log(context); // Actually shutdown tasks. let timeout_duration = std::time::Duration::from_secs(30); - tokio::time::timeout(timeout_duration, self.inbox_handle) - .await - .ok_or_log(context); - if let Some(mvbox_handle) = self.mvbox_handle.take() { - tokio::time::timeout(timeout_duration, mvbox_handle) - .await - .ok_or_log(context); - } - if let Some(sentbox_handle) = self.sentbox_handle.take() { - tokio::time::timeout(timeout_duration, sentbox_handle) + for b in once(self.inbox).chain(self.oboxes.into_iter()) { + tokio::time::timeout(timeout_duration, b.handle) .await .ok_or_log(context); } diff --git a/src/scheduler/connectivity.rs b/src/scheduler/connectivity.rs index 6ddc9eede..6ede2074f 100644 --- a/src/scheduler/connectivity.rs +++ b/src/scheduler/connectivity.rs @@ -1,18 +1,18 @@ use core::fmt; -use std::{ops::Deref, sync::Arc}; +use std::{iter::once, ops::Deref, sync::Arc}; use anyhow::{anyhow, Result}; use humansize::{format_size, BINARY}; use tokio::sync::{Mutex, RwLockReadGuard}; use crate::events::EventType; -use crate::imap::scan_folders::get_watched_folder_configs; +use crate::imap::{scan_folders::get_watched_folder_configs, FolderMeaning}; use crate::quota::{ QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE, }; use crate::tools::time; -use crate::{config::Config, scheduler::Scheduler, stock_str, tools}; use crate::{context::Context, log::LogExt}; +use crate::{scheduler::Scheduler, stock_str, tools}; #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)] pub enum Connectivity { @@ -157,17 +157,14 @@ impl ConnectivityStore { /// Called during `dc_maybe_network()` to make sure that `dc_accounts_all_work_done()` /// returns false immediately after `dc_maybe_network()`. pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Option>) { - let [inbox, mvbox, sentbox] = match &*scheduler { - Some(Scheduler { - inbox, - mvbox, - sentbox, - .. - }) => [ - inbox.state.connectivity.clone(), - mvbox.state.connectivity.clone(), - sentbox.state.connectivity.clone(), - ], + let (inbox, oboxes) = match &*scheduler { + Some(Scheduler { inbox, oboxes, .. }) => ( + inbox.conn_state.state.connectivity.clone(), + oboxes + .iter() + .map(|b| b.conn_state.state.connectivity.clone()) + .collect::>(), + ), None => return, }; drop(scheduler); @@ -185,7 +182,7 @@ pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Option>, ) { - let stores = match &*scheduler { - Some(Scheduler { - inbox, - mvbox, - sentbox, - .. - }) => [ - inbox.state.connectivity.clone(), - mvbox.state.connectivity.clone(), - sentbox.state.connectivity.clone(), - ], + let stores: Vec<_> = match &*scheduler { + Some(sched) => sched + .boxes() + .map(|b| b.conn_state.state.connectivity.clone()) + .collect(), None => return, }; drop(scheduler); @@ -260,14 +251,9 @@ impl Context { pub async fn get_connectivity(&self) -> Connectivity { let lock = self.scheduler.read().await; let stores: Vec<_> = match &*lock { - Some(Scheduler { - inbox, - mvbox, - sentbox, - .. - }) => [&inbox.state, &mvbox.state, &sentbox.state] - .iter() - .map(|state| state.connectivity.clone()) + Some(sched) => sched + .boxes() + .map(|b| b.conn_state.state.connectivity.clone()) .collect(), None => return Connectivity::NotConnected, }; @@ -348,28 +334,12 @@ impl Context { let lock = self.scheduler.read().await; let (folders_states, smtp) = match &*lock { - Some(Scheduler { - inbox, - mvbox, - sentbox, - smtp, - .. - }) => ( - [ - ( - Config::ConfiguredInboxFolder, - inbox.state.connectivity.clone(), - ), - ( - Config::ConfiguredMvboxFolder, - mvbox.state.connectivity.clone(), - ), - ( - Config::ConfiguredSentboxFolder, - sentbox.state.connectivity.clone(), - ), - ], - smtp.state.connectivity.clone(), + Some(sched) => ( + sched + .boxes() + .map(|b| (b.meaning, b.conn_state.state.connectivity.clone())) + .collect::>(), + sched.smtp.state.connectivity.clone(), ), None => { return Err(anyhow!("Not started")); @@ -390,8 +360,8 @@ impl Context { for (folder, state) in &folders_states { let mut folder_added = false; - if watched_folders.contains(folder) { - let f = self.get_config(*folder).await.ok_or_log(self).flatten(); + if let Some(config) = folder.to_config().filter(|c| watched_folders.contains(c)) { + let f = self.get_config(config).await.ok_or_log(self).flatten(); if let Some(foldername) = f { let detailed = &state.get_detailed().await; @@ -407,7 +377,7 @@ impl Context { } } - if !folder_added && folder == &Config::ConfiguredInboxFolder { + if !folder_added && folder == &FolderMeaning::Inbox { let detailed = &state.get_detailed().await; if let DetailedConnectivity::Error(_) = detailed { // On the inbox thread, we also do some other things like scan_folders and run jobs @@ -535,14 +505,10 @@ impl Context { pub async fn all_work_done(&self) -> bool { let lock = self.scheduler.read().await; let stores: Vec<_> = match &*lock { - Some(Scheduler { - inbox, - mvbox, - sentbox, - smtp, - .. - }) => [&inbox.state, &mvbox.state, &sentbox.state, &smtp.state] - .iter() + Some(sched) => sched + .boxes() + .map(|b| &b.conn_state.state) + .chain(once(&sched.smtp.state)) .map(|state| state.connectivity.clone()) .collect(), None => return false, From 3b5227c42adee4822ec2e7c8648f0fbb7c672a29 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Thu, 16 Feb 2023 17:46:28 -0300 Subject: [PATCH 03/30] Move strict_tls, max_smtp_rcpt_to from Provider to ProviderOptions --- src/configure.rs | 4 +- src/imap.rs | 2 +- src/provider.rs | 26 +++++--- src/provider/data.rs | 139 +++++-------------------------------------- src/smtp.rs | 5 +- src/smtp/send.rs | 2 +- 6 files changed, 39 insertions(+), 139 deletions(-) diff --git a/src/configure.rs b/src/configure.rs index 4350f08f6..565169470 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -249,7 +249,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> { } } }, - strict_tls: Some(provider.strict_tls), + strict_tls: Some(provider.opt.strict_tls), }) .collect(); @@ -338,7 +338,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> { .collect(); let provider_strict_tls = param .provider - .map_or(socks5_config.is_some(), |provider| provider.strict_tls); + .map_or(socks5_config.is_some(), |provider| provider.opt.strict_tls); let smtp_config_task = task::spawn(async move { let mut smtp_configured = false; diff --git a/src/imap.rs b/src/imap.rs index 5cdecae9b..df4ed807e 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -274,7 +274,7 @@ impl Imap { param .provider .map_or(param.socks5_config.is_some(), |provider| { - provider.strict_tls + provider.opt.strict_tls }), idle_interrupt_receiver, )?; diff --git a/src/provider.rs b/src/provider.rs index b461e600c..2d37b1347 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -129,13 +129,6 @@ pub struct Provider { /// Default configuration values to set when provider is configured. pub config_defaults: Option>, - /// True if provider is known to use use proper, - /// not self-signed certificates. - pub strict_tls: bool, - - /// Maximum number of recipients the provider allows to send a single email to. - pub max_smtp_rcpt_to: Option, - /// Type of OAuth 2 authorization if provider supports it. pub oauth2_authorizer: Option, @@ -144,12 +137,29 @@ pub struct Provider { } /// Provider options with good defaults. -#[derive(Debug, Default, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] pub struct ProviderOptions { + /// True if provider is known to use use proper, + /// not self-signed certificates. + pub strict_tls: bool, + + /// Maximum number of recipients the provider allows to send a single email to. + pub max_smtp_rcpt_to: Option, + /// Move messages to the Trash folder instead of marking them "\Deleted". pub delete_to_trash: bool, } +impl Default for ProviderOptions { + fn default() -> Self { + Self { + strict_tls: true, + max_smtp_rcpt_to: None, + delete_to_trash: false, + } + } +} + /// Get resolver to query MX records. /// /// We first try to read the system's resolver from `/etc/resolv.conf`. diff --git a/src/provider/data.rs b/src/provider/data.rs index 49a0d30a9..673c2b815 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -35,8 +35,6 @@ static P_163: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -65,8 +63,6 @@ static P_AKTIVIX_ORG: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -84,8 +80,6 @@ static P_AOL: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Ssl, hostname: "smtp.aol.com", port: 465, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -115,8 +109,6 @@ static P_ARCOR_DE: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -145,8 +137,6 @@ static P_AUTISTICI_ORG: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -175,8 +165,6 @@ static P_BLINDZELN_ORG: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -205,8 +193,6 @@ static P_BLUEWIN_CH: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -235,8 +221,6 @@ static P_BUZON_UY: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -265,8 +249,6 @@ static P_CHELLO_AT: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -280,8 +262,6 @@ static P_COMCAST: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/comcast", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -295,8 +275,6 @@ static P_DISMAIL_DE: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/dismail-de", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -325,8 +303,6 @@ static P_DISROOT: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -355,8 +331,6 @@ static P_E_EMAIL: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -370,8 +344,6 @@ static P_ESPIV_NET: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/espiv-net", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -389,8 +361,6 @@ static P_EXAMPLE_COM: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Starttls, hostname: "smtp.example.com", port: 1337, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -421,8 +391,6 @@ static P_FASTMAIL: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -438,8 +406,6 @@ static P_FIREMAIL_DE: Lazy = Lazy::new(|| { server: vec![ ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -467,8 +433,6 @@ static P_FIVE_CHAT: Lazy = Lazy::new(|| Provider { value: "0", }, ]), - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -486,8 +450,6 @@ static P_FREENET_DE: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Starttls, hostname: "mx.freenet.de", port: 587, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -506,11 +468,10 @@ static P_GMAIL: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Ssl, hostname: "smtp.gmail.com", port: 465, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: Some(Oauth2Authorizer::Gmail), opt: ProviderOptions { delete_to_trash: true, + ..Default::default() }, } }); @@ -546,8 +507,6 @@ static P_GMX_NET: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -574,10 +533,11 @@ static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { value: "2", }, ]), - strict_tls: false, - max_smtp_rcpt_to: None, oauth2_authorizer: None, - opt: Default::default(), + opt: ProviderOptions { + strict_tls: false, + ..Default::default() + }, }); // hey.com.md: hey.com @@ -591,8 +551,6 @@ static P_HEY_COM: Lazy = Lazy::new(|| { server: vec![ ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -607,8 +565,6 @@ static P_I_UA: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/i-ua", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -622,8 +578,6 @@ static P_I3_NET: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/i3-net", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -652,8 +606,6 @@ static P_ICLOUD: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -682,10 +634,11 @@ static P_INFOMANIAK_COM: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: Some(10), oauth2_authorizer: None, - opt: Default::default(), + opt: ProviderOptions { + max_smtp_rcpt_to: Some(10), + ..Default::default() + }, }); // kolst.com.md: kolst.com @@ -697,8 +650,6 @@ static P_KOLST_COM: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/kolst-com", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -712,8 +663,6 @@ static P_KONTENT_COM: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/kontent-com", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -742,8 +691,6 @@ static P_MAIL_DE: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -761,8 +708,6 @@ static P_MAIL_RU: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.ru", port: 465, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -792,8 +737,6 @@ static P_MAIL2TOR: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -822,8 +765,6 @@ static P_MAILBOX_ORG: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -852,8 +793,6 @@ static P_MAILO_COM: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -907,10 +846,12 @@ static P_NAUTA_CU: Lazy = Lazy::new(|| Provider { value: "0", }, ]), - strict_tls: false, - max_smtp_rcpt_to: Some(20), oauth2_authorizer: None, - opt: Default::default(), + opt: ProviderOptions { + strict_tls: false, + max_smtp_rcpt_to: Some(20), + ..Default::default() + }, }); // naver.md: naver.com @@ -937,8 +878,6 @@ static P_NAVER: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -967,8 +906,6 @@ static P_NUBO_COOP: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -997,8 +934,6 @@ static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1027,8 +962,6 @@ static P_OUVATON_COOP: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1057,8 +990,6 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1074,8 +1005,6 @@ static P_PROTONMAIL: Lazy = Lazy::new(|| { server: vec![ ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1094,8 +1023,6 @@ static P_QQ: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Ssl, hostname: "smtp.qq.com", port: 465, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1125,8 +1052,6 @@ static P_RISEUP_NET: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1140,8 +1065,6 @@ static P_ROGERS_COM: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/rogers-com", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1170,8 +1093,6 @@ static P_SYSTEMAUSFALL_ORG: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1200,8 +1121,6 @@ static P_SYSTEMLI_ORG: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1219,8 +1138,6 @@ static P_T_ONLINE: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Ssl, hostname: "securesmtp.t-online.de", port: 465, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1270,8 +1187,6 @@ static P_TESTRUN: Lazy = Lazy::new(|| Provider { value: "0", }, ]), - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1300,8 +1215,6 @@ static P_TISCALI_IT: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1317,8 +1230,6 @@ static P_TUTANOTA: Lazy = Lazy::new(|| { server: vec![ ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1333,8 +1244,6 @@ static P_UKR_NET: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/ukr-net", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1363,8 +1272,6 @@ static P_UNDERNET_UY: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1378,8 +1285,6 @@ static P_VFEMAIL: Lazy = Lazy::new(|| Provider { overview_page: "https://providers.delta.chat/vfemail", server: vec![], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1408,8 +1313,6 @@ static P_VIVALDI: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1438,8 +1341,6 @@ static P_VODAFONE_DE: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1458,8 +1359,6 @@ static P_WEB_DE: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Starttls, hostname: "smtp.web.de", port: 587, username_pattern: Emaillocalpart }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1478,8 +1377,6 @@ static P_YAHOO: Lazy = Lazy::new(|| { Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: Email }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1509,8 +1406,6 @@ static P_YANDEX_RU: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: Some(Oauth2Authorizer::Yandex), opt: Default::default(), }); @@ -1530,8 +1425,6 @@ static P_YGGMAIL: Lazy = Lazy::new(|| { config_defaults: Some(vec![ ConfigDefault { key: Config::MvboxMove, value: "0" }, ]), - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), } @@ -1561,8 +1454,6 @@ static P_ZIGGO_NL: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); @@ -1591,8 +1482,6 @@ static P_ZOHO: Lazy = Lazy::new(|| Provider { }, ], config_defaults: None, - strict_tls: true, - max_smtp_rcpt_to: None, oauth2_authorizer: None, opt: Default::default(), }); diff --git a/src/smtp.rs b/src/smtp.rs index 5cf0dd73f..cda3aff05 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -101,8 +101,9 @@ impl Smtp { &lp.smtp, &lp.socks5_config, &lp.addr, - lp.provider - .map_or(lp.socks5_config.is_some(), |provider| provider.strict_tls), + lp.provider.map_or(lp.socks5_config.is_some(), |provider| { + provider.opt.strict_tls + }), ) .await } diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 3cd5c0567..938deb5fc 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -47,7 +47,7 @@ impl Smtp { let chunk_size = context .get_configured_provider() .await? - .and_then(|provider| provider.max_smtp_rcpt_to) + .and_then(|provider| provider.opt.max_smtp_rcpt_to) .map_or(DEFAULT_MAX_SMTP_RCPT_TO, usize::from); for recipients_chunk in recipients.chunks(chunk_size) { From ef03a33b29ba7c434793ed7299b822aafeef278b Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Mon, 20 Feb 2023 18:10:32 +0100 Subject: [PATCH 04/30] JSON-RPC: Add CommonJS build (#4062) add CommonJS build --- deltachat-jsonrpc/typescript/package.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 039057017..aded96fcc 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -14,7 +14,7 @@ "c8": "^7.10.0", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", - "esbuild": "^0.14.11", + "esbuild": "^0.17.9", "http-server": "^14.1.1", "mocha": "^9.1.1", "node-fetch": "^2.6.1", @@ -24,13 +24,20 @@ "typescript": "^4.5.5", "ws": "^8.5.0" }, + "exports": { + ".": { + "require": "./dist/deltachat.cjs", + "import": "./dist/deltachat.js" + } + }, "license": "MPL-2.0", "main": "dist/deltachat.js", "name": "@deltachat/jsonrpc-client", "scripts": { - "build": "run-s generate-bindings extract-constants build:tsc build:bundle", + "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs", "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js", "build:tsc": "tsc", + "build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs", "docs": "typedoc --out docs deltachat.ts", "example": "run-s build example:build example:start", "example:build": "esbuild --bundle dist/example/example.js --outfile=dist/example.bundle.js", From 56d10f7c420f10218ccd8b5aa37bb2280c8b21d0 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 19 Feb 2023 14:38:01 +0000 Subject: [PATCH 05/30] Use transaction in `update_blocked_mailinglist_contacts` --- CHANGELOG.md | 1 + src/contact.rs | 74 ++++++++++++++++++++++++-------------------------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7199274ec..34cebb853 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - use transaction in `Contact::add_or_lookup()` #4059 - Organize the connection pool as a stack rather than a queue to ensure that connection page cache is reused more often. #4065 +- Use transaction in `update_blocked_mailinglist_contacts`. #4058 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/src/contact.rs b/src/contact.rs index d27ccc361..2ae958603 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -869,47 +869,45 @@ impl Contact { Ok(ret) } - // add blocked mailinglists as contacts - // to allow unblocking them as if they are contacts - // (this way, only one unblock-ffi is needed and only one set of ui-functions, - // from the users perspective, - // there is not much difference in an email- and a mailinglist-address) + /// Adds blocked mailinglists as contacts + /// to allow unblocking them as if they are contacts + /// (this way, only one unblock-ffi is needed and only one set of ui-functions, + /// from the users perspective, + /// there is not much difference in an email- and a mailinglist-address) async fn update_blocked_mailinglist_contacts(context: &Context) -> Result<()> { - let blocked_mailinglists = context + context .sql - .query_map( - "SELECT name, grpid FROM chats WHERE type=? AND blocked=?;", - paramsv![Chattype::Mailinglist, Blocked::Yes], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - |rows| { - rows.collect::, _>>() - .map_err(Into::into) - }, - ) + .transaction(move |transaction| { + let mut stmt = transaction + .prepare("SELECT name, grpid FROM chats WHERE type=? AND blocked=?")?; + let rows = stmt.query_map(params![Chattype::Mailinglist, Blocked::Yes], |row| { + let name: String = row.get(0)?; + let grpid: String = row.get(1)?; + Ok((name, grpid)) + })?; + let blocked_mailinglists = rows.collect::, _>>()?; + for (name, grpid) in blocked_mailinglists { + let count = transaction.query_row( + "SELECT COUNT(id) FROM contacts WHERE addr=?", + [&grpid], + |row| { + let count: isize = row.get(0)?; + Ok(count) + }, + )?; + if count == 0 { + transaction.execute("INSERT INTO contacts (addr) VALUES (?)", [&grpid])?; + } + + // Always do an update in case the blocking is reset or name is changed. + transaction.execute( + "UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?", + params![&name, Origin::MailinglistAddress, &grpid], + )?; + } + Ok(()) + }) .await?; - for (name, grpid) in blocked_mailinglists { - if !context - .sql - .exists( - "SELECT COUNT(id) FROM contacts WHERE addr=?;", - paramsv![grpid], - ) - .await? - { - context - .sql - .execute("INSERT INTO contacts (addr) VALUES (?);", paramsv![grpid]) - .await?; - } - // always do an update in case the blocking is reset or name is changed - context - .sql - .execute( - "UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?;", - paramsv![name, Origin::MailinglistAddress, grpid], - ) - .await?; - } Ok(()) } From 710cec1beb5fb6e144a21215bae355c785952424 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 20 Feb 2023 19:06:43 +0100 Subject: [PATCH 06/30] Remove show_emails argument from prefetch_should_download() (#4064) IIRC, this was written this way back when we didn't have config caching, in order to save database accesses by getting the config outside the for loop. --- src/imap.rs | 7 +++---- src/receive_imf/tests.rs | 16 ++++++---------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index df4ed807e..d7fcf1397 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -728,8 +728,6 @@ impl Imap { }; let read_cnt = msgs.len(); - let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?) - .unwrap_or_default(); let download_limit = context.download_limit().await?; let mut uids_fetch = Vec::<(_, bool /* partially? */)>::with_capacity(msgs.len() + 1); let mut uid_message_ids = BTreeMap::new(); @@ -778,7 +776,6 @@ impl Imap { &headers, &message_id, fetch_response.flags(), - show_emails, ) .await.context("prefetch_should_download")? { @@ -2055,7 +2052,6 @@ pub(crate) async fn prefetch_should_download( headers: &[mailparse::MailHeader<'_>], message_id: &str, mut flags: impl Iterator>, - show_emails: ShowEmails, ) -> Result { if message::rfc724_mid_exists(context, message_id) .await? @@ -2115,6 +2111,9 @@ pub(crate) async fn prefetch_should_download( }) .unwrap_or_default(); + let show_emails = + ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?).unwrap_or_default(); + let show = is_autocrypt_setup_message || match show_emails { ShowEmails::Off => is_chat_message || is_reply_to_chat_message, diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index b2564fc75..50aa285ce 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -5,7 +5,7 @@ use crate::aheader::EncryptPreference; use crate::chat::get_chat_contacts; use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility}; use crate::chatlist::Chatlist; -use crate::constants::{ShowEmails, DC_GCL_NO_SPECIALS}; +use crate::constants::DC_GCL_NO_SPECIALS; use crate::imap::prefetch_should_download; use crate::message::Message; use crate::test_utils::{get_chat_msg, TestContext, TestContextManager}; @@ -647,15 +647,11 @@ async fn test_parse_ndn( // Check that the ndn would be downloaded: let headers = mailparse::parse_mail(raw_ndn).unwrap().headers; - assert!(prefetch_should_download( - &t, - &headers, - "some-other-message-id", - std::iter::empty(), - ShowEmails::Off, - ) - .await - .unwrap()); + assert!( + prefetch_should_download(&t, &headers, "some-other-message-id", std::iter::empty(),) + .await + .unwrap() + ); receive_imf(&t, raw_ndn, false).await.unwrap(); let msg = Message::load_from_db(&t, msg_id).await.unwrap(); From 92c7cc40d449415a7e9821b7fa5fcd6e454529f6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 18 Feb 2023 13:50:44 +0000 Subject: [PATCH 07/30] sql: replace .get_conn() interface with .call() .call() interface is safer because it ensures that blocking operations on SQL connection are called within tokio::task::block_in_place(). Previously some code called blocking operations in async context, e.g. add_parts() in receive_imf module. The underlying implementation of .call() can later be replaced with an implementation that does not require block_in_place(), e.g. a worker pool, without changing the code using the .call() interface. --- CHANGELOG.md | 1 + src/chat.rs | 3 +- src/imex.rs | 38 ++++++------- src/key.rs | 60 +++++++++++---------- src/location.rs | 52 ++++++++++-------- src/peerstate.rs | 2 +- src/receive_imf.rs | 73 ++++++++++++------------- src/sql.rs | 131 +++++++++++++++++++++++++-------------------- 8 files changed, 189 insertions(+), 171 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34cebb853..3abb9ca41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Organize the connection pool as a stack rather than a queue to ensure that connection page cache is reused more often. #4065 - Use transaction in `update_blocked_mailinglist_contacts`. #4058 +- Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/src/chat.rs b/src/chat.rs index dd88cdd30..522f55aa7 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -907,7 +907,8 @@ impl ChatId { async fn parent_query(self, context: &Context, fields: &str, f: F) -> Result> where - F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result, + T: Send + 'static, { let sql = &context.sql; let query = format!( diff --git a/src/imex.rs b/src/imex.rs index f23496753..a3de4c201 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -540,25 +540,27 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res .to_str() .with_context(|| format!("path {temp_db_path:?} is not valid unicode"))?; - let conn = context.sql.get_conn().await?; - tokio::task::block_in_place(move || { - if let Err(err) = conn.execute("VACUUM", params![]) { - info!(context, "Vacuum failed, exporting anyway: {:#}.", err); - } - conn.execute( - "ATTACH DATABASE ? AS backup KEY ?", - paramsv![path_str, passphrase], - ) - .context("failed to attach backup database")?; - let res = conn - .query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(())) - .context("failed to export to attached backup database"); - conn.execute("DETACH DATABASE backup", []) - .context("failed to detach backup database")?; - res?; + context + .sql + .call(|conn| { + if let Err(err) = conn.execute("VACUUM", params![]) { + info!(context, "Vacuum failed, exporting anyway: {:#}.", err); + } + conn.execute( + "ATTACH DATABASE ? AS backup KEY ?", + paramsv![path_str, passphrase], + ) + .context("failed to attach backup database")?; + let res = conn + .query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(())) + .context("failed to export to attached backup database"); + conn.execute("DETACH DATABASE backup", []) + .context("failed to detach backup database")?; + res?; - Ok::<_, Error>(()) - })?; + Ok::<_, Error>(()) + }) + .await?; let res = export_backup_inner(context, &temp_db_path, &temp_path).await; diff --git a/src/key.rs b/src/key.rs index bee59c60b..3b48fe19e 100644 --- a/src/key.rs +++ b/src/key.rs @@ -289,39 +289,41 @@ pub async fn store_self_keypair( keypair: &KeyPair, default: KeyPairUse, ) -> Result<()> { - let mut conn = context.sql.get_conn().await?; - let transaction = conn.transaction()?; + context + .sql + .transaction(|transaction| { + let public_key = DcKey::to_bytes(&keypair.public); + let secret_key = DcKey::to_bytes(&keypair.secret); + transaction + .execute( + "DELETE FROM keypairs WHERE public_key=? OR private_key=?;", + paramsv![public_key, secret_key], + ) + .context("failed to remove old use of key")?; + if default == KeyPairUse::Default { + transaction + .execute("UPDATE keypairs SET is_default=0;", paramsv![]) + .context("failed to clear default")?; + } + let is_default = match default { + KeyPairUse::Default => i32::from(true), + KeyPairUse::ReadOnly => i32::from(false), + }; - let public_key = DcKey::to_bytes(&keypair.public); - let secret_key = DcKey::to_bytes(&keypair.secret); - transaction - .execute( - "DELETE FROM keypairs WHERE public_key=? OR private_key=?;", - paramsv![public_key, secret_key], - ) - .context("failed to remove old use of key")?; - if default == KeyPairUse::Default { - transaction - .execute("UPDATE keypairs SET is_default=0;", paramsv![]) - .context("failed to clear default")?; - } - let is_default = match default { - KeyPairUse::Default => i32::from(true), - KeyPairUse::ReadOnly => i32::from(false), - }; + let addr = keypair.addr.to_string(); + let t = time(); - let addr = keypair.addr.to_string(); - let t = time(); - - transaction - .execute( - "INSERT INTO keypairs (addr, is_default, public_key, private_key, created) + transaction + .execute( + "INSERT INTO keypairs (addr, is_default, public_key, private_key, created) VALUES (?,?,?,?,?);", - paramsv![addr, is_default, public_key, secret_key, t], - ) - .context("failed to insert keypair")?; + paramsv![addr, is_default, public_key, secret_key, t], + ) + .context("failed to insert keypair")?; - transaction.commit()?; + Ok(()) + }) + .await?; Ok(()) } diff --git a/src/location.rs b/src/location.rs index 24a202c17..6698dea3b 100644 --- a/src/location.rs +++ b/src/location.rs @@ -601,32 +601,38 @@ pub(crate) async fn save( .. } = location; - let conn = context.sql.get_conn().await?; - let mut stmt_test = - conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?; - let mut stmt_insert = conn.prepare_cached(stmt_insert)?; + context + .sql + .call(|conn| { + let mut stmt_test = conn + .prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?; + let mut stmt_insert = conn.prepare_cached(stmt_insert)?; - let exists = stmt_test.exists(paramsv![timestamp, contact_id])?; + let exists = stmt_test.exists(paramsv![timestamp, contact_id])?; - if independent || !exists { - stmt_insert.execute(paramsv![ - timestamp, - contact_id, - chat_id, - latitude, - longitude, - accuracy, - independent, - ])?; + if independent || !exists { + stmt_insert.execute(paramsv![ + timestamp, + contact_id, + chat_id, + latitude, + longitude, + accuracy, + independent, + ])?; - if timestamp > newest_timestamp { - // okay to drop, as we use cached prepared statements - drop(stmt_test); - drop(stmt_insert); - newest_timestamp = timestamp; - newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?); - } - } + if timestamp > newest_timestamp { + // okay to drop, as we use cached prepared statements + drop(stmt_test); + drop(stmt_insert); + newest_timestamp = timestamp; + newest_location_id = Some(u32::try_from(conn.last_insert_rowid())?); + } + } + + Ok(()) + }) + .await?; } Ok(newest_location_id) diff --git a/src/peerstate.rs b/src/peerstate.rs index acac59a65..e7433f0f3 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -186,7 +186,7 @@ impl Peerstate { async fn from_stmt( context: &Context, query: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, ) -> Result> { let peerstate = context .sql diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 1896e844b..2540d7442 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1085,8 +1085,6 @@ async fn add_parts( let mut created_db_entries = Vec::with_capacity(mime_parser.parts.len()); - let conn = context.sql.get_conn().await?; - for part in &mime_parser.parts { if part.is_reaction { set_msg_reaction( @@ -1118,39 +1116,6 @@ async fn add_parts( } let mut txt_raw = "".to_string(); - let mut stmt = conn.prepare_cached( - r#" -INSERT INTO msgs - ( - id, - rfc724_mid, chat_id, - from_id, to_id, timestamp, timestamp_sent, - timestamp_rcvd, type, state, msgrmsg, - txt, subject, txt_raw, param, - bytes, mime_headers, mime_in_reply_to, - mime_references, mime_modified, error, ephemeral_timer, - ephemeral_timestamp, download_state, hop_info - ) - VALUES ( - ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ?, - ?, ?, ?, ? - ) -ON CONFLICT (id) DO UPDATE -SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, - from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent, - timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg, - txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param, - bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to, - mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer, - ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info -"#, - )?; - let (msg, typ): (&str, Viewtype) = if let Some(better_msg) = &better_msg { (better_msg, Viewtype::Text) } else { @@ -1184,7 +1149,38 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && msg.is_empty()); - stmt.execute(paramsv![ + let row_id = context.sql.insert( + r#" +INSERT INTO msgs + ( + id, + rfc724_mid, chat_id, + from_id, to_id, timestamp, timestamp_sent, + timestamp_rcvd, type, state, msgrmsg, + txt, subject, txt_raw, param, + bytes, mime_headers, mime_in_reply_to, + mime_references, mime_modified, error, ephemeral_timer, + ephemeral_timestamp, download_state, hop_info + ) + VALUES ( + ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, ? + ) +ON CONFLICT (id) DO UPDATE +SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, + from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent, + timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg, + txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param, + bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to, + mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer, + ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info +"#, + paramsv![ replace_msg_id, rfc724_mid, if trash { DC_CHAT_ID_TRASH } else { chat_id }, @@ -1223,17 +1219,14 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, DownloadState::Done }, mime_parser.hop_info - ])?; + ]).await?; // We only replace placeholder with a first part, // afterwards insert additional parts. replace_msg_id = None; - let row_id = conn.last_insert_rowid(); - drop(stmt); created_db_entries.push(MsgId::new(u32::try_from(row_id)?)); } - drop(conn); // check all parts whether they contain a new logging webxdc for (part, msg_id) in mime_parser.parts.iter().zip(&created_db_entries) { diff --git a/src/sql.rs b/src/sql.rs index 4b4b190eb..d96c85b66 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -2,8 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{bail, Context as _, Result}; use rusqlite::{self, config::DbConfig, Connection, OpenFlags, TransactionBehavior}; @@ -49,7 +48,7 @@ pub(crate) fn params_iter(iter: &[impl crate::ToSql]) -> impl Iterator Result<()> { let path_str = path .to_str() - .with_context(|| format!("path {path:?} is not valid unicode"))?; - let conn = self.get_conn().await?; - - tokio::task::block_in_place(move || { + .with_context(|| format!("path {path:?} is not valid unicode"))? + .to_string(); + self.call(move |conn| { // Check that backup passphrase is correct before resetting our database. conn.execute( "ATTACH DATABASE ? AS backup KEY ?", @@ -167,6 +165,7 @@ impl Sql { res?; Ok(()) }) + .await } /// Creates a new connection pool. @@ -294,22 +293,41 @@ impl Sql { } } + /// Allocates a connection and calls given function with the connection. + /// + /// Returns the result of the function. + pub async fn call<'a, F, R>(&'a self, function: F) -> Result + where + F: 'a + FnOnce(&mut Connection) -> Result + Send, + R: Send + 'static, + { + let lock = self.pool.read().await; + let pool = lock.as_ref().context("no SQL connection")?; + let mut conn = pool.get().await?; + let res = tokio::task::block_in_place(move || function(&mut conn))?; + Ok(res) + } + /// Execute the given query, returning the number of affected rows. - pub async fn execute(&self, query: &str, params: impl rusqlite::Params) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + pub async fn execute( + &self, + query: &str, + params: impl rusqlite::Params + Send, + ) -> Result { + self.call(move |conn| { let res = conn.execute(query, params)?; Ok(res) }) + .await } /// Executes the given query, returning the last inserted row ID. - pub async fn insert(&self, query: &str, params: impl rusqlite::Params) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + pub async fn insert(&self, query: &str, params: impl rusqlite::Params + Send) -> Result { + self.call(move |conn| { conn.execute(query, params)?; Ok(conn.last_insert_rowid()) }) + .await } /// Prepares and executes the statement and maps a function over the resulting rows. @@ -318,40 +336,32 @@ impl Sql { pub async fn query_map( &self, sql: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, f: F, mut g: G, ) -> Result where - F: FnMut(&rusqlite::Row) -> rusqlite::Result, - G: FnMut(rusqlite::MappedRows) -> Result, + F: Send + FnMut(&rusqlite::Row) -> rusqlite::Result, + G: Send + FnMut(rusqlite::MappedRows) -> Result, + H: Send + 'static, { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut stmt = conn.prepare(sql)?; let res = stmt.query_map(params, f)?; g(res) }) - } - - /// Allocates a connection from the connection pool and returns it. - pub(crate) async fn get_conn(&self) -> Result { - let lock = self.pool.read().await; - let pool = lock.as_ref().context("no SQL connection")?; - let conn = pool.get().await?; - - Ok(conn) + .await } /// Used for executing `SELECT COUNT` statements only. Returns the resulting count. - pub async fn count(&self, query: &str, params: impl rusqlite::Params) -> Result { + pub async fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result { let count: isize = self.query_row(query, params, |row| row.get(0)).await?; Ok(usize::try_from(count)?) } /// Used for executing `SELECT COUNT` statements only. Returns `true`, if the count is at least /// one, `false` otherwise. - pub async fn exists(&self, sql: &str, params: impl rusqlite::Params) -> Result { + pub async fn exists(&self, sql: &str, params: impl rusqlite::Params + Send) -> Result { let count = self.count(sql, params).await?; Ok(count > 0) } @@ -360,17 +370,18 @@ impl Sql { pub async fn query_row( &self, query: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, f: F, ) -> Result where - F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + F: FnOnce(&rusqlite::Row) -> rusqlite::Result + Send, + T: Send + 'static, { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let res = conn.query_row(query, params, f)?; Ok(res) }) + .await } /// Execute the function inside a transaction. @@ -388,8 +399,7 @@ impl Sql { H: Send + 'static, G: Send + FnOnce(&mut rusqlite::Transaction<'_>) -> Result, { - let mut conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut transaction = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let ret = callback(&mut transaction); @@ -404,12 +414,12 @@ impl Sql { } } }) + .await } /// Query the database if the requested table already exists. pub async fn table_exists(&self, name: &str) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut exists = false; conn.pragma(None, "table_info", name.to_string(), |_row| { // will only be executed if the info was found @@ -419,12 +429,12 @@ impl Sql { Ok(exists) }) + .await } /// Check if a column exists in a given table. pub async fn col_exists(&self, table_name: &str, col_name: &str) -> Result { - let conn = self.get_conn().await?; - tokio::task::block_in_place(move || { + self.call(move |conn| { let mut exists = false; // `PRAGMA table_info` returns one row per column, // each row containing 0=cid, 1=name, 2=type, 3=notnull, 4=dflt_value @@ -438,29 +448,27 @@ impl Sql { Ok(exists) }) + .await } /// Execute a query which is expected to return zero or one row. pub async fn query_row_optional( &self, sql: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, f: F, ) -> Result> where - F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + F: Send + FnOnce(&rusqlite::Row) -> rusqlite::Result, + T: Send + 'static, { - let conn = self.get_conn().await?; - let res = - tokio::task::block_in_place(move || match conn.query_row(sql.as_ref(), params, f) { - Ok(res) => Ok(Some(res)), - Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), - Err(rusqlite::Error::InvalidColumnType(_, _, rusqlite::types::Type::Null)) => { - Ok(None) - } - Err(err) => Err(err), - })?; - Ok(res) + self.call(move |conn| match conn.query_row(sql.as_ref(), params, f) { + Ok(res) => Ok(Some(res)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(rusqlite::Error::InvalidColumnType(_, _, rusqlite::types::Type::Null)) => Ok(None), + Err(err) => Err(err.into()), + }) + .await } /// Executes a query which is expected to return one row and one @@ -469,10 +477,10 @@ impl Sql { pub async fn query_get_value( &self, query: &str, - params: impl rusqlite::Params, + params: impl rusqlite::Params + Send, ) -> Result> where - T: rusqlite::types::FromSql, + T: rusqlite::types::FromSql + Send + 'static, { self.query_row_optional(query, params, |row| row.get::<_, T>(0)) .await @@ -935,11 +943,16 @@ mod tests { async fn test_auto_vacuum() -> Result<()> { let t = TestContext::new().await; - let conn = t.sql.get_conn().await?; - let auto_vacuum = conn.pragma_query_value(None, "auto_vacuum", |row| { - let auto_vacuum: i32 = row.get(0)?; - Ok(auto_vacuum) - })?; + let auto_vacuum = t + .sql + .call(|conn| { + let auto_vacuum = conn.pragma_query_value(None, "auto_vacuum", |row| { + let auto_vacuum: i32 = row.get(0)?; + Ok(auto_vacuum) + })?; + Ok(auto_vacuum) + }) + .await?; // auto_vacuum=2 is the same as auto_vacuum=INCREMENTAL assert_eq!(auto_vacuum, 2); From f07206bd6cb6e4e783a197eeb88b6367f0de9afd Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 20 Feb 2023 19:57:54 +0000 Subject: [PATCH 08/30] Pin ruff version in deltachat-rpc-client Latest versions 0.0.248 and 0.0.249 report false positive: src/deltachat_rpc_client/client.py:105:9: RET503 [*] Missing explicit `return` at the end of function able to return --- deltachat-rpc-client/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tox.ini b/deltachat-rpc-client/tox.ini index 7f74daa37..5437e5fce 100644 --- a/deltachat-rpc-client/tox.ini +++ b/deltachat-rpc-client/tox.ini @@ -22,7 +22,7 @@ deps = skipsdist = True skip_install = True deps = - ruff + ruff==0.0.247 black commands = black --check --diff src/ examples/ tests/ From 05a274a5f33d95a0a1611cc226b824a39e6843bf Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 11:17:10 +0000 Subject: [PATCH 09/30] Enable more ruff checks in deltachat-rpc-client --- deltachat-rpc-client/pyproject.toml | 30 ++++++++++++++++++- .../src/deltachat_rpc_client/account.py | 4 +-- .../src/deltachat_rpc_client/chat.py | 7 +++-- .../src/deltachat_rpc_client/client.py | 6 ++-- .../src/deltachat_rpc_client/contact.py | 4 +-- .../src/deltachat_rpc_client/deltachat.py | 6 ++-- .../src/deltachat_rpc_client/events.py | 6 ++-- .../src/deltachat_rpc_client/message.py | 4 +-- .../src/deltachat_rpc_client/pytestplugin.py | 2 +- deltachat-rpc-client/tests/test_something.py | 3 +- deltachat-rpc-client/tests/test_webxdc.py | 1 - 11 files changed, 53 insertions(+), 20 deletions(-) diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index ef76fa82b..f7d7fc905 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -25,7 +25,35 @@ deltachat_rpc_client = [ line-length = 120 [tool.ruff] -select = ["E", "F", "W", "N", "YTT", "B", "C4", "ISC", "ICN", "PT", "RET", "SIM", "TID", "ARG", "DTZ", "ERA", "PLC", "PLE", "PLW", "PIE", "COM"] +select = [ + "E", "W", # pycodestyle + "F", # Pyflakes + "N", # pep8-naming + "I", # isort + + "ARG", # flake8-unused-arguments + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "COM", # flake8-commas + "DTZ", # flake8-datetimez + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PIE", # flake8-pie + "PT", # flake8-pytest-style + "RET", # flake8-return + "SIM", # flake8-simplify + "TCH", # flake8-type-checking + "TID", # flake8-tidy-imports + "YTT", # flake8-2020 + + "ERA", # eradicate + + "PLC", # Pylint Convention + "PLE", # Pylint Error + "PLW", # Pylint Warning + + "RUF006" # asyncio-dangling-task +] line-length = 120 [tool.isort] diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/account.py b/deltachat-rpc-client/src/deltachat_rpc_client/account.py index 63a9b169c..86ee9addb 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/account.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/account.py @@ -1,15 +1,15 @@ -from typing import TYPE_CHECKING, List, Optional, Tuple, Union from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Tuple, Union from ._utils import AttrDict from .chat import Chat from .const import ChatlistFlag, ContactFlag, SpecialContactId from .contact import Contact from .message import Message -from .rpc import Rpc if TYPE_CHECKING: from .deltachat import DeltaChat + from .rpc import Rpc @dataclass diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index c2e5ca364..443927b38 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -1,16 +1,17 @@ import calendar -from datetime import datetime -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from dataclasses import dataclass +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from ._utils import AttrDict from .const import ChatVisibility from .contact import Contact from .message import Message -from .rpc import Rpc if TYPE_CHECKING: + from datetime import datetime + from .account import Account + from .rpc import Rpc @dataclass diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/client.py b/deltachat-rpc-client/src/deltachat_rpc_client/client.py index 393ac3cc7..1f1a14506 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/client.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/client.py @@ -2,6 +2,7 @@ import inspect import logging from typing import ( + TYPE_CHECKING, Callable, Coroutine, Dict, @@ -13,8 +14,6 @@ from typing import ( Union, ) -from deltachat_rpc_client.account import Account - from ._utils import ( AttrDict, parse_system_add_remove, @@ -31,6 +30,9 @@ from .events import ( RawEvent, ) +if TYPE_CHECKING: + from deltachat_rpc_client.account import Account + class Client: """Simple Delta Chat client that listen to events of a single account.""" diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/contact.py b/deltachat-rpc-client/src/deltachat_rpc_client/contact.py index 7999d59ed..9e9417790 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/contact.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/contact.py @@ -1,12 +1,12 @@ -from typing import TYPE_CHECKING from dataclasses import dataclass +from typing import TYPE_CHECKING from ._utils import AttrDict -from .rpc import Rpc if TYPE_CHECKING: from .account import Account from .chat import Chat + from .rpc import Rpc @dataclass diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py b/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py index 16afe458b..aea4ccd1f 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py @@ -1,8 +1,10 @@ -from typing import Dict, List +from typing import TYPE_CHECKING, Dict, List from ._utils import AttrDict from .account import Account -from .rpc import Rpc + +if TYPE_CHECKING: + from .rpc import Rpc class DeltaChat: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/events.py b/deltachat-rpc-client/src/deltachat_rpc_client/events.py index 0e160445c..0330c87d9 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/events.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/events.py @@ -2,11 +2,13 @@ import inspect import re from abc import ABC, abstractmethod -from typing import Callable, Iterable, Iterator, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union -from ._utils import AttrDict from .const import EventType +if TYPE_CHECKING: + from ._utils import AttrDict + def _tuple_of(obj, type_: type) -> tuple: if not obj: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/message.py b/deltachat-rpc-client/src/deltachat_rpc_client/message.py index 7784d8dac..224931ba1 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/message.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/message.py @@ -1,13 +1,13 @@ import json -from typing import TYPE_CHECKING, Union from dataclasses import dataclass +from typing import TYPE_CHECKING, Union from ._utils import AttrDict from .contact import Contact -from .rpc import Rpc if TYPE_CHECKING: from .account import Account + from .rpc import Rpc @dataclass diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py index 87e97f0e0..065f7744e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py @@ -1,8 +1,8 @@ +import asyncio import json import os from typing import AsyncGenerator, List, Optional -import asyncio import aiohttp import pytest_asyncio diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index 47b4cf8e3..fc05854d6 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -1,8 +1,7 @@ +import asyncio from unittest.mock import MagicMock import pytest -import asyncio - from deltachat_rpc_client import EventType, events from deltachat_rpc_client.rpc import JsonRpcError diff --git a/deltachat-rpc-client/tests/test_webxdc.py b/deltachat-rpc-client/tests/test_webxdc.py index 22d9db0b4..8a0584d03 100644 --- a/deltachat-rpc-client/tests/test_webxdc.py +++ b/deltachat-rpc-client/tests/test_webxdc.py @@ -1,5 +1,4 @@ import pytest - from deltachat_rpc_client import EventType From 17acbca57698c3a6cafb22962bcd7f195b9268c1 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 21 Feb 2023 12:23:34 +0100 Subject: [PATCH 10/30] Correctly clear database cache after import (#4067) --- CHANGELOG.md | 1 + src/imex.rs | 45 ++++++++++++++++++++++++++++++-- src/sql.rs | 74 ++++++++++++++++++++++++++++------------------------ 3 files changed, 84 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3abb9ca41..621d24ac1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 - Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted. Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972 +- Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 ### API-Changes diff --git a/src/imex.rs b/src/imex.rs index a3de4c201..ffc2589dd 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -427,8 +427,6 @@ async fn import_backup( context.get_dbfile().display() ); - context.sql.config_cache.write().await.clear(); - let mut archive = Archive::new(backup_file); let mut entries = archive.entries()?; @@ -785,7 +783,10 @@ where #[cfg(test)] mod tests { + use std::time::Duration; + use ::pgp::armor::BlockType; + use tokio::task; use super::*; use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE}; @@ -933,6 +934,46 @@ mod tests { Ok(()) } + /// This is a regression test for + /// https://github.com/deltachat/deltachat-android/issues/2263 + /// where the config cache wasn't reset properly after a backup. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_import_backup_reset_config_cache() -> Result<()> { + let backup_dir = tempfile::tempdir()?; + let context1 = TestContext::new_alice().await; + let context2 = TestContext::new().await; + assert!(!context2.is_configured().await?); + + // export from context1 + imex(&context1, ImexMode::ExportBackup, backup_dir.path(), None).await?; + + // import to context2 + let backup = has_backup(&context2, backup_dir.path()).await?; + let context2_cloned = context2.clone(); + let handle = task::spawn(async move { + imex( + &context2_cloned, + ImexMode::ImportBackup, + backup.as_ref(), + None, + ) + .await + .unwrap(); + }); + + while !handle.is_finished() { + // The database is still unconfigured; + // fill the config cache with the old value. + context2.is_configured().await.ok(); + tokio::time::sleep(Duration::from_micros(1)).await; + } + + // Assert that the config cache has the new value now. + assert!(context2.is_configured().await?); + + Ok(()) + } + #[test] fn test_normalize_setup_code() { let norm = normalize_setup_code("123422343234423452346234723482349234"); diff --git a/src/sql.rs b/src/sql.rs index d96c85b66..163d4e989 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -129,43 +129,49 @@ impl Sql { .to_str() .with_context(|| format!("path {path:?} is not valid unicode"))? .to_string(); - self.call(move |conn| { - // Check that backup passphrase is correct before resetting our database. - conn.execute( - "ATTACH DATABASE ? AS backup KEY ?", - paramsv![path_str, passphrase], - ) - .context("failed to attach backup database")?; - if let Err(err) = conn - .query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(())) - .context("backup passphrase is not correct") - { + let res = self + .call(move |conn| { + // Check that backup passphrase is correct before resetting our database. + conn.execute( + "ATTACH DATABASE ? AS backup KEY ?", + paramsv![path_str, passphrase], + ) + .context("failed to attach backup database")?; + if let Err(err) = conn + .query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(())) + .context("backup passphrase is not correct") + { + conn.execute("DETACH DATABASE backup", []) + .context("failed to detach backup database")?; + return Err(err); + } + + // Reset the database without reopening it. We don't want to reopen the database because we + // don't have main database passphrase at this point. + // See for documentation. + // Without resetting import may fail due to existing tables. + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true) + .context("failed to set SQLITE_DBCONFIG_RESET_DATABASE")?; + conn.execute("VACUUM", []) + .context("failed to vacuum the database")?; + conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false) + .context("failed to unset SQLITE_DBCONFIG_RESET_DATABASE")?; + let res = conn + .query_row("SELECT sqlcipher_export('main', 'backup')", [], |_row| { + Ok(()) + }) + .context("failed to import from attached backup database"); conn.execute("DETACH DATABASE backup", []) .context("failed to detach backup database")?; - return Err(err); - } + res?; + Ok(()) + }) + .await; - // Reset the database without reopening it. We don't want to reopen the database because we - // don't have main database passphrase at this point. - // See for documentation. - // Without resetting import may fail due to existing tables. - conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, true) - .context("failed to set SQLITE_DBCONFIG_RESET_DATABASE")?; - conn.execute("VACUUM", []) - .context("failed to vacuum the database")?; - conn.set_db_config(DbConfig::SQLITE_DBCONFIG_RESET_DATABASE, false) - .context("failed to unset SQLITE_DBCONFIG_RESET_DATABASE")?; - let res = conn - .query_row("SELECT sqlcipher_export('main', 'backup')", [], |_row| { - Ok(()) - }) - .context("failed to import from attached backup database"); - conn.execute("DETACH DATABASE backup", []) - .context("failed to detach backup database")?; - res?; - Ok(()) - }) - .await + // The config cache is wrong now that we have a different database + self.config_cache.write().await.clear(); + + res } /// Creates a new connection pool. From c01a2f2c24da4b6c817908eb5a33f3bed02e5102 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 12:21:17 +0000 Subject: [PATCH 11/30] Fix missing imports in deltachat_rpc_client --- .../src/deltachat_rpc_client/account.py | 2 +- .../src/deltachat_rpc_client/chat.py | 6 +++--- .../src/deltachat_rpc_client/client.py | 2 +- .../src/deltachat_rpc_client/contact.py | 2 +- .../src/deltachat_rpc_client/deltachat.py | 2 +- .../src/deltachat_rpc_client/events.py | 12 ++++++------ .../src/deltachat_rpc_client/message.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/account.py b/deltachat-rpc-client/src/deltachat_rpc_client/account.py index 86ee9addb..b1e9fea63 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/account.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/account.py @@ -20,7 +20,7 @@ class Account: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.manager.rpc async def wait_for_event(self) -> AttrDict: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index 443927b38..7dac7d2f4 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -22,7 +22,7 @@ class Chat: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.account._rpc async def delete(self) -> None: @@ -218,8 +218,8 @@ class Chat: async def get_locations( self, contact: Optional[Contact] = None, - timestamp_from: Optional[datetime] = None, - timestamp_to: Optional[datetime] = None, + timestamp_from: Optional["datetime"] = None, + timestamp_to: Optional["datetime"] = None, ) -> List[AttrDict]: """Get list of location snapshots for the given contact in the given timespan.""" time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0 diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/client.py b/deltachat-rpc-client/src/deltachat_rpc_client/client.py index 1f1a14506..6f816e5de 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/client.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/client.py @@ -39,7 +39,7 @@ class Client: def __init__( self, - account: Account, + account: "Account", hooks: Optional[Iterable[Tuple[Callable, Union[type, EventFilter]]]] = None, logger: Optional[logging.Logger] = None, ) -> None: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/contact.py b/deltachat-rpc-client/src/deltachat_rpc_client/contact.py index 9e9417790..efb3e9297 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/contact.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/contact.py @@ -21,7 +21,7 @@ class Contact: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.account._rpc async def block(self) -> None: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py b/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py index aea4ccd1f..c2cecd60d 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py @@ -13,7 +13,7 @@ class DeltaChat: This is the root of the object oriented API. """ - def __init__(self, rpc: Rpc) -> None: + def __init__(self, rpc: "Rpc") -> None: self.rpc = rpc async def add_account(self) -> Account: diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/events.py b/deltachat-rpc-client/src/deltachat_rpc_client/events.py index 0330c87d9..4896527b9 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/events.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/events.py @@ -82,7 +82,7 @@ class RawEvent(EventFilter): return (self.types, self.func) == (other.types, other.func) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.types and event.type not in self.types: return False return await self._call_func(event) @@ -120,7 +120,7 @@ class NewMessage(EventFilter): command: Optional[str] = None, is_bot: Optional[bool] = False, is_info: Optional[bool] = None, - func: Optional[Callable[[AttrDict], bool]] = None, + func: Optional[Callable[["AttrDict"], bool]] = None, ) -> None: super().__init__(func=func) self.is_bot = is_bot @@ -159,7 +159,7 @@ class NewMessage(EventFilter): ) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.is_bot is not None and self.is_bot != event.message_snapshot.is_bot: return False if self.is_info is not None and self.is_info != event.message_snapshot.is_info: @@ -201,7 +201,7 @@ class MemberListChanged(EventFilter): return (self.added, self.func) == (other.added, other.func) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.added is not None and self.added != event.member_added: return False return await self._call_func(event) @@ -233,7 +233,7 @@ class GroupImageChanged(EventFilter): return (self.deleted, self.func) == (other.deleted, other.func) return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: if self.deleted is not None and self.deleted != event.image_deleted: return False return await self._call_func(event) @@ -258,7 +258,7 @@ class GroupNameChanged(EventFilter): return self.func == other.func return False - async def filter(self, event: AttrDict) -> bool: + async def filter(self, event: "AttrDict") -> bool: return await self._call_func(event) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/message.py b/deltachat-rpc-client/src/deltachat_rpc_client/message.py index 224931ba1..5ec30961a 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/message.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/message.py @@ -18,7 +18,7 @@ class Message: id: int @property - def _rpc(self) -> Rpc: + def _rpc(self) -> "Rpc": return self.account._rpc async def send_reaction(self, *reaction: str): From 38a62d92bad08a703413f0be3e993180a4e289b4 Mon Sep 17 00:00:00 2001 From: bjoern Date: Tue, 21 Feb 2023 13:36:36 +0100 Subject: [PATCH 12/30] mention speedup of #4065 in CHANGELOG (#4073) the speedup of #4065 is larger than measured first running `cargo test`, first we thought there is only a slight speedup from 13.5 seconds to 12.5 seconds on a m1pro. however, there is one test that does 11 seconds of sleep() (test_modify_chat_disordered) as this test is not run very first, this slows down things overall. skipping this test, speedup is from 13.5 seconds to 9.5 seconds - 28% faster - and this is something we should mention in the changelog :) (this pr does not remove or change the slow test. i think, due to the number of cores, this is not needed until someone has a machine where the other tests run in 2 seconds or so) --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 621d24ac1..38795a38a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ ### Changes - use transaction in `Contact::add_or_lookup()` #4059 - Organize the connection pool as a stack rather than a queue to ensure that - connection page cache is reused more often. #4065 + connection page cache is reused more often. + This speeds up tests by 28%, real usage will have lower speedup. #4065 - Use transaction in `update_blocked_mailinglist_contacts`. #4058 - Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055 From e86fbf5855a3560e59b2eac8daf0d7b08d62220a Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 13:09:57 +0000 Subject: [PATCH 13/30] ci: stop using deprecated Ubuntu 18.04 GitHub Actions fails running 18.04 jobs currently: --- .github/workflows/jsonrpc-client-npm-package.yml | 2 +- .github/workflows/node-package.yml | 10 +++++----- .github/workflows/node-tests.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index 7d7f0145e..b7f2f34d9 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -9,7 +9,7 @@ on: jobs: pack-module: name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat" - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Install tree run: sudo apt install tree diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index b737ca00c..4ae546397 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, macos-latest, windows-latest] + os: [ubuntu-20.04, macos-latest, windows-latest] steps: - name: Checkout uses: actions/checkout@v3 @@ -65,7 +65,7 @@ jobs: pack-module: needs: prebuild name: Package deltachat-node and upload to download.delta.chat - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Install tree run: sudo apt install tree @@ -99,7 +99,7 @@ jobs: - name: Download Ubuntu prebuild uses: actions/download-artifact@v1 with: - name: ubuntu-18.04 + name: ubuntu-20.04 - name: Download macOS prebuild uses: actions/download-artifact@v1 with: @@ -111,11 +111,11 @@ jobs: - shell: bash run: | mkdir node/prebuilds - tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C node/prebuilds + tar -xvzf ubuntu-20.04/ubuntu-20.04.tar.gz -C node/prebuilds tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds tree node/prebuilds - rm -rf ubuntu-18.04 macos-latest windows-latest + rm -rf ubuntu-20.04 macos-latest windows-latest - name: Install dependencies without running scripts run: | npm install --ignore-scripts diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 73e8d0cc4..7836f9b33 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, macos-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - name: Checkout uses: actions/checkout@v3 From 15c9efaa95e2b20bb594298d973c059e737a84bd Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 20 Feb 2023 15:55:52 -0300 Subject: [PATCH 14/30] Update provider/update.py according to the `struct Provider` changes --- src/provider/update.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/provider/update.py b/src/provider/update.py index b67e9fb59..304b80e74 100755 --- a/src/provider/update.py +++ b/src/provider/update.py @@ -40,6 +40,23 @@ def file2url(f): return "https://providers.delta.chat/" + f +def process_opt(data): + if not "opt" in data: + return "Default::default()" + opt = "ProviderOptions {\n" + opt_data = data.get("opt", "") + for key in opt_data: + value = str(opt_data[key]) + if key == "max_smtp_rcpt_to": + value = "Some(" + value + ")" + if value in {"True", "False"}: + value = value.lower() + opt += " " + key + ": " + value + ",\n" + opt += " ..Default::default()\n" + opt += " }" + return opt + + def process_config_defaults(data): if not "config_defaults" in data: return "None" @@ -106,14 +123,9 @@ def process_data(data, file): server += (" Server { protocol: " + protocol.capitalize() + ", socket: " + socket.capitalize() + ", hostname: \"" + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern.capitalize() + " },\n") + opt = process_opt(data) config_defaults = process_config_defaults(data) - strict_tls = data.get("strict_tls", True) - strict_tls = "true" if strict_tls else "false" - - max_smtp_rcpt_to = data.get("max_smtp_rcpt_to", 0) - max_smtp_rcpt_to = "Some(" + str(max_smtp_rcpt_to) + ")" if max_smtp_rcpt_to != 0 else "None" - oauth2 = data.get("oauth2", "") oauth2 = "Some(Oauth2Authorizer::" + camel(oauth2) + ")" if oauth2 != "" else "None" @@ -128,9 +140,8 @@ def process_data(data, file): provider += " after_login_hint: \"" + after_login_hint + "\",\n" provider += " overview_page: \"" + file2url(file) + "\",\n" provider += " server: vec![\n" + server + " ],\n" + provider += " opt: " + opt + ",\n" provider += " config_defaults: " + config_defaults + ",\n" - provider += " strict_tls: " + strict_tls + ",\n" - provider += " max_smtp_rcpt_to: " + max_smtp_rcpt_to + ",\n" provider += " oauth2_authorizer: " + oauth2 + ",\n" provider += "});\n\n" else: @@ -174,7 +185,9 @@ if __name__ == "__main__": "use crate::provider::Protocol::*;\n" "use crate::provider::Socket::*;\n" "use crate::provider::UsernamePattern::*;\n" - "use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status};\n" + "use crate::provider::{\n" + " Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status,\n" + "};\n" "use std::collections::HashMap;\n\n" "use once_cell::sync::Lazy;\n\n") From 45abaff275358db612cf49f5238265dad8f36239 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 14:40:56 +0000 Subject: [PATCH 15/30] Update provider database --- CHANGELOG.md | 1 + src/provider/data.rs | 176 +++++++++++++++++++++++++------------------ 2 files changed, 103 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38795a38a..c0fa82590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ This speeds up tests by 28%, real usage will have lower speedup. #4065 - Use transaction in `update_blocked_mailinglist_contacts`. #4058 - Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055 +- Updated provider database. ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/src/provider/data.rs b/src/provider/data.rs index 673c2b815..73eb49634 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -1,15 +1,14 @@ // file generated by src/provider/update.py -use std::collections::HashMap; - -use once_cell::sync::Lazy; - use crate::provider::Protocol::*; use crate::provider::Socket::*; use crate::provider::UsernamePattern::*; use crate::provider::{ Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status, }; +use std::collections::HashMap; + +use once_cell::sync::Lazy; // 163.md: 163.com static P_163: Lazy = Lazy::new(|| Provider { @@ -34,9 +33,9 @@ static P_163: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // aktivix.org.md: aktivix.org @@ -62,9 +61,9 @@ static P_AKTIVIX_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // aol.md: aol.com @@ -79,9 +78,9 @@ static P_AOL: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.aol.com", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.aol.com", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -108,9 +107,9 @@ static P_ARCOR_DE: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // autistici.org.md: autistici.org @@ -136,9 +135,9 @@ static P_AUTISTICI_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // blindzeln.org.md: delta.blinzeln.de, delta.blindzeln.org @@ -164,9 +163,9 @@ static P_BLINDZELN_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // bluewin.ch.md: bluewin.ch @@ -192,9 +191,9 @@ static P_BLUEWIN_CH: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // buzon.uy.md: buzon.uy @@ -220,9 +219,9 @@ static P_BUZON_UY: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // chello.at.md: chello.at @@ -248,9 +247,9 @@ static P_CHELLO_AT: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // comcast.md: xfinity.com, comcast.net @@ -261,9 +260,9 @@ static P_COMCAST: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/comcast", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // dismail.de.md: dismail.de @@ -274,9 +273,9 @@ static P_DISMAIL_DE: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/dismail-de", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // disroot.md: disroot.org @@ -302,9 +301,9 @@ static P_DISROOT: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // e.email.md: e.email @@ -330,9 +329,9 @@ static P_E_EMAIL: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // espiv.net.md: espiv.net @@ -343,9 +342,9 @@ static P_ESPIV_NET: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/espiv-net", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // example.com.md: example.com, example.org, example.net @@ -360,9 +359,9 @@ static P_EXAMPLE_COM: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.example.com", port: 1337, username_pattern: Email }, Server { protocol: Smtp, socket: Starttls, hostname: "smtp.example.com", port: 1337, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -390,9 +389,9 @@ static P_FASTMAIL: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // firemail.de.md: firemail.at, firemail.de @@ -405,9 +404,9 @@ static P_FIREMAIL_DE: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/firemail-de", server: vec![ ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -419,6 +418,7 @@ static P_FIVE_CHAT: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/five-chat", server: vec![], + opt: Default::default(), config_defaults: Some(vec![ ConfigDefault { key: Config::BccSelf, @@ -434,7 +434,6 @@ static P_FIVE_CHAT: Lazy = Lazy::new(|| Provider { }, ]), oauth2_authorizer: None, - opt: Default::default(), }); // freenet.de.md: freenet.de @@ -449,9 +448,9 @@ static P_FREENET_DE: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "mx.freenet.de", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Starttls, hostname: "mx.freenet.de", port: 587, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -467,12 +466,12 @@ static P_GMAIL: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.gmail.com", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.gmail.com", port: 465, username_pattern: Email }, ], - config_defaults: None, - oauth2_authorizer: Some(Oauth2Authorizer::Gmail), opt: ProviderOptions { delete_to_trash: true, ..Default::default() }, + config_defaults: None, + oauth2_authorizer: Some(Oauth2Authorizer::Gmail), } }); @@ -506,9 +505,9 @@ static P_GMX_NET: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, ec.hermes.radio, ec1.hermes.radio, ec2.hermes.radio, ec3.hermes.radio, ec4.hermes.radio, ec5.hermes.radio, ec6.hermes.radio, ec7.hermes.radio, ec8.hermes.radio, ec9.hermes.radio, ec10.hermes.radio, ec11.hermes.radio, ec12.hermes.radio, ec13.hermes.radio, ec14.hermes.radio, ec15.hermes.radio, hermes.radio @@ -519,6 +518,10 @@ static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/hermes-radio", server: vec![], + opt: ProviderOptions { + strict_tls: false, + ..Default::default() + }, config_defaults: Some(vec![ ConfigDefault { key: Config::MdnsEnabled, @@ -534,10 +537,6 @@ static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { }, ]), oauth2_authorizer: None, - opt: ProviderOptions { - strict_tls: false, - ..Default::default() - }, }); // hey.com.md: hey.com @@ -550,9 +549,9 @@ static P_HEY_COM: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/hey-com", server: vec![ ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -564,9 +563,9 @@ static P_I_UA: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/i-ua", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // i3.net.md: i3.net @@ -577,9 +576,9 @@ static P_I3_NET: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/i3-net", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // icloud.md: icloud.com, me.com, mac.com @@ -605,9 +604,9 @@ static P_ICLOUD: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // infomaniak.com.md: ik.me @@ -633,12 +632,12 @@ static P_INFOMANIAK_COM: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], - config_defaults: None, - oauth2_authorizer: None, opt: ProviderOptions { max_smtp_rcpt_to: Some(10), ..Default::default() }, + config_defaults: None, + oauth2_authorizer: None, }); // kolst.com.md: kolst.com @@ -649,9 +648,9 @@ static P_KOLST_COM: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/kolst-com", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // kontent.com.md: kontent.com @@ -662,9 +661,9 @@ static P_KONTENT_COM: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/kontent-com", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // mail.de.md: mail.de @@ -690,9 +689,9 @@ static P_MAIL_DE: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // mail.ru.md: mail.ru, inbox.ru, internet.ru, bk.ru, list.ru @@ -707,9 +706,9 @@ static P_MAIL_RU: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.ru", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.ru", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -736,9 +735,9 @@ static P_MAIL2TOR: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // mailbox.org.md: mailbox.org, secure.mailbox.org @@ -764,9 +763,9 @@ static P_MAILBOX_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // mailo.com.md: mailo.com @@ -792,9 +791,9 @@ static P_MAILO_COM: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // nauta.cu.md: nauta.cu @@ -820,6 +819,11 @@ static P_NAUTA_CU: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: ProviderOptions { + max_smtp_rcpt_to: Some(20), + strict_tls: false, + ..Default::default() + }, config_defaults: Some(vec![ ConfigDefault { key: Config::DeleteServerAfter, @@ -847,11 +851,6 @@ static P_NAUTA_CU: Lazy = Lazy::new(|| Provider { }, ]), oauth2_authorizer: None, - opt: ProviderOptions { - strict_tls: false, - max_smtp_rcpt_to: Some(20), - ..Default::default() - }, }); // naver.md: naver.com @@ -877,9 +876,9 @@ static P_NAVER: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // nubo.coop.md: nubo.coop @@ -905,9 +904,9 @@ static P_NUBO_COOP: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com, outlook.de @@ -933,9 +932,9 @@ static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // ouvaton.coop.md: ouvaton.org @@ -961,9 +960,9 @@ static P_OUVATON_COOP: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us @@ -974,6 +973,13 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/posteo", server: vec![ + Server { + protocol: Imap, + socket: Ssl, + hostname: "posteo.de", + port: 993, + username_pattern: Email, + }, Server { protocol: Imap, socket: Starttls, @@ -981,6 +987,13 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { port: 143, username_pattern: Email, }, + Server { + protocol: Smtp, + socket: Ssl, + hostname: "posteo.de", + port: 465, + username_pattern: Email, + }, Server { protocol: Smtp, socket: Starttls, @@ -989,9 +1002,9 @@ static P_POSTEO: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // protonmail.md: protonmail.com, protonmail.ch, pm.me @@ -1004,9 +1017,9 @@ static P_PROTONMAIL: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/protonmail", server: vec![ ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1022,9 +1035,9 @@ static P_QQ: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.qq.com", port: 993, username_pattern: Emaillocalpart }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.qq.com", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1051,9 +1064,9 @@ static P_RISEUP_NET: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // rogers.com.md: rogers.com @@ -1064,9 +1077,22 @@ static P_ROGERS_COM: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/rogers-com", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, +}); + +// sonic.md: sonic.net +static P_SONIC: Lazy = Lazy::new(|| Provider { + id: "sonic", + status: Status::Ok, + before_login_hint: "", + after_login_hint: "", + overview_page: "https://providers.delta.chat/sonic", + server: vec![], opt: Default::default(), + config_defaults: None, + oauth2_authorizer: None, }); // systemausfall.org.md: systemausfall.org, solidaris.me @@ -1092,9 +1118,9 @@ static P_SYSTEMAUSFALL_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // systemli.org.md: systemli.org @@ -1120,9 +1146,9 @@ static P_SYSTEMLI_ORG: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // t-online.md: t-online.de, magenta.de @@ -1137,9 +1163,9 @@ static P_T_ONLINE: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "secureimap.t-online.de", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "securesmtp.t-online.de", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1173,6 +1199,7 @@ static P_TESTRUN: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: Some(vec![ ConfigDefault { key: Config::BccSelf, @@ -1188,7 +1215,6 @@ static P_TESTRUN: Lazy = Lazy::new(|| Provider { }, ]), oauth2_authorizer: None, - opt: Default::default(), }); // tiscali.it.md: tiscali.it @@ -1214,9 +1240,9 @@ static P_TISCALI_IT: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // tutanota.md: tutanota.com, tutanota.de, tutamail.com, tuta.io, keemail.me @@ -1229,9 +1255,9 @@ static P_TUTANOTA: Lazy = Lazy::new(|| { overview_page: "https://providers.delta.chat/tutanota", server: vec![ ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1243,9 +1269,9 @@ static P_UKR_NET: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/ukr-net", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // undernet.uy.md: undernet.uy @@ -1271,9 +1297,9 @@ static P_UNDERNET_UY: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // vfemail.md: vfemail.net @@ -1284,9 +1310,9 @@ static P_VFEMAIL: Lazy = Lazy::new(|| Provider { after_login_hint: "", overview_page: "https://providers.delta.chat/vfemail", server: vec![], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // vivaldi.md: vivaldi.net @@ -1312,9 +1338,9 @@ static P_VIVALDI: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // vodafone.de.md: vodafone.de, vodafonemail.de @@ -1340,9 +1366,9 @@ static P_VODAFONE_DE: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // web.de.md: web.de, email.de, flirt.ms, hallo.ms, kuss.ms, love.ms, magic.ms, singles.ms, cool.ms, kanzler.ms, okay.ms, party.ms, pop.ms, stars.ms, techno.ms, clever.ms, deutschland.ms, genial.ms, ich.ms, online.ms, smart.ms, wichtig.ms, action.ms, fussball.ms, joker.ms, planet.ms, power.ms @@ -1358,9 +1384,9 @@ static P_WEB_DE: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Starttls, hostname: "imap.web.de", port: 143, username_pattern: Emaillocalpart }, Server { protocol: Smtp, socket: Starttls, hostname: "smtp.web.de", port: 587, username_pattern: Emaillocalpart }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1376,9 +1402,9 @@ static P_YAHOO: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: Email }, Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: Email }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1405,9 +1431,9 @@ static P_YANDEX_RU: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: Some(Oauth2Authorizer::Yandex), - opt: Default::default(), }); // yggmail.md: yggmail @@ -1422,11 +1448,11 @@ static P_YGGMAIL: Lazy = Lazy::new(|| { Server { protocol: Imap, socket: Plain, hostname: "localhost", port: 1143, username_pattern: Email }, Server { protocol: Smtp, socket: Plain, hostname: "localhost", port: 1025, username_pattern: Email }, ], + opt: Default::default(), config_defaults: Some(vec![ ConfigDefault { key: Config::MvboxMove, value: "0" }, ]), oauth2_authorizer: None, - opt: Default::default(), } }); @@ -1453,9 +1479,9 @@ static P_ZIGGO_NL: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); // zoho.md: zohomail.eu, zohomail.com, zoho.com @@ -1481,9 +1507,9 @@ static P_ZOHO: Lazy = Lazy::new(|| Provider { username_pattern: Email, }, ], + opt: Default::default(), config_defaults: None, oauth2_authorizer: None, - opt: Default::default(), }); pub(crate) static PROVIDER_DATA: Lazy> = Lazy::new(|| { @@ -1776,6 +1802,7 @@ pub(crate) static PROVIDER_DATA: Lazy> ("foxmail.com", &*P_QQ), ("riseup.net", &*P_RISEUP_NET), ("rogers.com", &*P_ROGERS_COM), + ("sonic.net", &*P_SONIC), ("systemausfall.org", &*P_SYSTEMAUSFALL_ORG), ("solidaris.me", &*P_SYSTEMAUSFALL_ORG), ("systemli.org", &*P_SYSTEMLI_ORG), @@ -1900,6 +1927,7 @@ pub(crate) static PROVIDER_IDS: Lazy> = ("qq", &*P_QQ), ("riseup.net", &*P_RISEUP_NET), ("rogers.com", &*P_ROGERS_COM), + ("sonic", &*P_SONIC), ("systemausfall.org", &*P_SYSTEMAUSFALL_ORG), ("systemli.org", &*P_SYSTEMLI_ORG), ("t-online", &*P_T_ONLINE), @@ -1924,4 +1952,4 @@ pub(crate) static PROVIDER_IDS: Lazy> = }); pub static PROVIDER_UPDATED: Lazy = - Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 1, 6).unwrap()); + Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 21).unwrap()); From df1c1addfbd2375641fbb70d4de7afdf1dd4e79f Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 23:24:08 +0000 Subject: [PATCH 16/30] ci: add action to build deltachat-rpc-server --- .github/workflows/deltachat-rpc-server.yml | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/deltachat-rpc-server.yml diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml new file mode 100644 index 000000000..cddb470eb --- /dev/null +++ b/.github/workflows/deltachat-rpc-server.yml @@ -0,0 +1,47 @@ +# Manually triggered action to build deltachat-rpc-server binaries. + +name: Build deltachat-rpc-server binaries + +on: + workflow_dispatch: + +jobs: + build_server: + name: Build deltachat-rpc-server + strategy: + matrix: + - os: ubuntu-22.04 + artifact: gnu-linux-x86_64 + path: "target/release/deltachat-rpc-server" + target: x86_64-unknown-linux-gnu + + - os: ubuntu-22.04 + artifact: gnu-linux-i686 + path: "target/release/deltachat-rpc-server" + target: i686-unknown-linux-gnu + + - os: windows-latest + artifact: win32.exe + path: "target/release/deltachat-rpc-server.exe" + target: i686-pc-windows-msvc + + - os: windows-latest + artifact: win64.exe + path: "target/release/deltachat-rpc-server.exe" + target: x86_64-pc-windows-msvc + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + + - name: Setup rust target + run: rustup target add ${{ matrix.target }} + + - name: Build + run: cargo build --release -p deltachat-rpc-server + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-${{ matrix.artfiact }} + path: ${{ matrix.path }} From a616c69f9a637c1b4b24691f91db26d4c0e0b81f Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 23:40:45 +0000 Subject: [PATCH 17/30] ci: add missing include to deltachat-rpc-server.yml --- .github/workflows/deltachat-rpc-server.yml | 33 +++++++++++----------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index cddb470eb..8979bf033 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -10,25 +10,26 @@ jobs: name: Build deltachat-rpc-server strategy: matrix: - - os: ubuntu-22.04 - artifact: gnu-linux-x86_64 - path: "target/release/deltachat-rpc-server" - target: x86_64-unknown-linux-gnu + include: + - os: ubuntu-22.04 + artifact: gnu-linux-x86_64 + path: "target/release/deltachat-rpc-server" + target: x86_64-unknown-linux-gnu - - os: ubuntu-22.04 - artifact: gnu-linux-i686 - path: "target/release/deltachat-rpc-server" - target: i686-unknown-linux-gnu + - os: ubuntu-22.04 + artifact: gnu-linux-i686 + path: "target/release/deltachat-rpc-server" + target: i686-unknown-linux-gnu - - os: windows-latest - artifact: win32.exe - path: "target/release/deltachat-rpc-server.exe" - target: i686-pc-windows-msvc + - os: windows-latest + artifact: win32.exe + path: "target/release/deltachat-rpc-server.exe" + target: i686-pc-windows-msvc - - os: windows-latest - artifact: win64.exe - path: "target/release/deltachat-rpc-server.exe" - target: x86_64-pc-windows-msvc + - os: windows-latest + artifact: win64.exe + path: "target/release/deltachat-rpc-server.exe" + target: x86_64-pc-windows-msvc runs-on: ${{ matrix.os }} steps: From 877b3551ae0ef5c1fa8d6d1ecdf2451b1a96ca57 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 23:54:20 +0000 Subject: [PATCH 18/30] ci: add --target option when build deltachat-rpc-server --- .github/workflows/deltachat-rpc-server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 8979bf033..5242f74c1 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -39,7 +39,7 @@ jobs: run: rustup target add ${{ matrix.target }} - name: Build - run: cargo build --release -p deltachat-rpc-server + run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} - name: Upload binary uses: actions/upload-artifact@v3 From 178e67a262b968aa9f1032c4834a53609fdc602c Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:00:32 +0000 Subject: [PATCH 19/30] Add "vendored" feature to deltachat-rpc-server --- .github/workflows/deltachat-rpc-server.yml | 2 +- deltachat-rpc-server/Cargo.toml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 5242f74c1..95d4382c5 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -39,7 +39,7 @@ jobs: run: rustup target add ${{ matrix.target }} - name: Build - run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} + run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored - name: Upload binary uses: actions/upload-artifact@v3 diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 619841fc9..1722198cd 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -23,3 +23,7 @@ serde_json = "1.0.91" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.25.0", features = ["io-std"] } yerpc = { version = "0.4.0", features = ["anyhow_expose"] } + +[features] +default = ["vendored"] +vendored = ["deltachat/vendored"] From 79212bee133a666806c13c6da1ad03110a770af9 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:03:40 +0000 Subject: [PATCH 20/30] Add "vendored" feature to deltachat-jsonrpc --- deltachat-jsonrpc/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 2ad9f2b83..e2958a4b2 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -36,5 +36,6 @@ tokio = { version = "1.25.0", features = ["full", "rt-multi-thread"] } [features] -default = [] +default = ["vendored"] webserver = ["env_logger", "axum", "tokio/full", "yerpc/support-axum"] +vendored = ["deltachat/vendored"] From 57f221dcc91f7cb94376146499acd38b73a7760e Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:04:42 +0000 Subject: [PATCH 21/30] Enable "vendored" feature on "deltachat-jsonrpc" from "deltachat-rpc-server" --- deltachat-rpc-server/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 1722198cd..a332e21b5 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -13,7 +13,7 @@ categories = ["cryptography", "std", "email"] name = "deltachat-rpc-server" [dependencies] -deltachat-jsonrpc = { path = "../deltachat-jsonrpc" } +deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false } anyhow = "1" env_logger = { version = "0.10.0" } @@ -26,4 +26,4 @@ yerpc = { version = "0.4.0", features = ["anyhow_expose"] } [features] default = ["vendored"] -vendored = ["deltachat/vendored"] +vendored = ["deltachat-jsonrpc/vendored"] From 7bfff6c87c2f608941e880ba1492c96784ce9f52 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:11:45 +0000 Subject: [PATCH 22/30] ci: disable fail-fast on deltachat-rpc-server.yml builds --- .github/workflows/deltachat-rpc-server.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 95d4382c5..29f40b491 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -9,6 +9,7 @@ jobs: build_server: name: Build deltachat-rpc-server strategy: + fail-fast: false matrix: include: - os: ubuntu-22.04 From 439c57e3aca0439b304dcfef676e160810ff706e Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:32:30 +0000 Subject: [PATCH 23/30] ci: fix typo in deltachat-rpc-server.yml --- .github/workflows/deltachat-rpc-server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 29f40b491..809d77d8f 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -45,5 +45,5 @@ jobs: - name: Upload binary uses: actions/upload-artifact@v3 with: - name: deltachat-rpc-server-${{ matrix.artfiact }} + name: deltachat-rpc-server-${{ matrix.artifact }} path: ${{ matrix.path }} From edec80a917b63b92990ab33a6096089ec1d36a7c Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:35:35 +0000 Subject: [PATCH 24/30] ci: attempt to fix binary path for artifact upload --- .github/workflows/deltachat-rpc-server.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 809d77d8f..acc91ee3e 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -14,22 +14,22 @@ jobs: include: - os: ubuntu-22.04 artifact: gnu-linux-x86_64 - path: "target/release/deltachat-rpc-server" + path: deltachat-rpc-server target: x86_64-unknown-linux-gnu - os: ubuntu-22.04 artifact: gnu-linux-i686 - path: "target/release/deltachat-rpc-server" + path: deltachat-rpc-server target: i686-unknown-linux-gnu - os: windows-latest artifact: win32.exe - path: "target/release/deltachat-rpc-server.exe" + path: deltachat-rpc-server.exe target: i686-pc-windows-msvc - os: windows-latest artifact: win64.exe - path: "target/release/deltachat-rpc-server.exe" + path: deltachat-rpc-server.exe target: x86_64-pc-windows-msvc runs-on: ${{ matrix.os }} @@ -46,4 +46,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: deltachat-rpc-server-${{ matrix.artifact }} - path: ${{ matrix.path }} + path: target/${{ matrix.target}}/release/${{ matrix.path }} From df96fbdcefca10dc12c7245e692e9dadc810c0d5 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 00:37:17 +0000 Subject: [PATCH 25/30] ci: error if deltachat-rpc-server artifact cannot be uploaded --- .github/workflows/deltachat-rpc-server.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index acc91ee3e..6f5e5166b 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -47,3 +47,4 @@ jobs: with: name: deltachat-rpc-server-${{ matrix.artifact }} path: target/${{ matrix.target}}/release/${{ matrix.path }} + if-no-files-found: error From 7f5217fc8708d4064a7c964382447b1578514ff6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 01:07:17 +0000 Subject: [PATCH 26/30] ci: do not try to cross-compile deltachat-rpc-server on i686 It does not work because vendored OpenSSL compilation fails --- .github/workflows/deltachat-rpc-server.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 6f5e5166b..9f8f83982 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -17,11 +17,6 @@ jobs: path: deltachat-rpc-server target: x86_64-unknown-linux-gnu - - os: ubuntu-22.04 - artifact: gnu-linux-i686 - path: deltachat-rpc-server - target: i686-unknown-linux-gnu - - os: windows-latest artifact: win32.exe path: deltachat-rpc-server.exe From 999a9550f5314479572b0f84d8542b0fd56569f7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 05:08:13 +0000 Subject: [PATCH 27/30] Remove explicit native-tls dependency --- Cargo.lock | 1 - Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 836a44375..7e0219a99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -864,7 +864,6 @@ dependencies = [ "libc", "log", "mailparse", - "native-tls", "num-derive", "num-traits", "num_cpus", diff --git a/Cargo.toml b/Cargo.toml index ecbab5a88..867a10b72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,6 @@ kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } libc = "0.2" mailparse = "0.14" -native-tls = "0.2" num_cpus = "1.15" num-derive = "0.3" num-traits = "0.2" From d3f4654d4b5d620cdb2c9926eee911116c35c74b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Mon, 20 Feb 2023 09:29:38 -0800 Subject: [PATCH 28/30] python: replace pkg_resources with importlib.metadata Use of pkg_resources is discouraged in favor of importlib.resources, importlib.metadata, and their backports. --- python/pyproject.toml | 1 + python/src/deltachat/__init__.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 4aa6666db..227fb73d2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ dependencies = [ "cffi>=1.0.0", "imap-tools", + "importlib_metadata;python_version<'3.8'", "pluggy", "requests", ] diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 048d8e3a7..3d2734cca 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -1,6 +1,9 @@ import sys -from pkg_resources import DistributionNotFound, get_distribution +if sys.version_info >= (3, 8): + from importlib.metadata import PackageNotFoundError, version +else: + from importlib_metadata import PackageNotFoundError, version from . import capi, events, hookspec # noqa from .account import Account, get_core_info # noqa @@ -11,8 +14,8 @@ from .hookspec import account_hookimpl, global_hookimpl # noqa from .message import Message # noqa try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: + __version__ = version(__name__) +except PackageNotFoundError: # package is not installed __version__ = "0.0.0.dev0-unknown" From 42a18d4d0d19893a8e5c21e6033cd957a64e89c7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 11:26:51 +0000 Subject: [PATCH 29/30] Unpin ruff version False positive is fixed in the latest version. --- deltachat-rpc-client/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-rpc-client/tox.ini b/deltachat-rpc-client/tox.ini index 5437e5fce..7f74daa37 100644 --- a/deltachat-rpc-client/tox.ini +++ b/deltachat-rpc-client/tox.ini @@ -22,7 +22,7 @@ deps = skipsdist = True skip_install = True deps = - ruff==0.0.247 + ruff black commands = black --check --diff src/ examples/ tests/ From c7b7fbaf784655820a22edfb3f2ff7834334defb Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 13:53:02 +0000 Subject: [PATCH 30/30] ci: use minimal profile for rustup Avoid installing unnecessary documentation. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf8185b22..449161d7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Install rustfmt and clippy - run: rustup toolchain install $RUSTUP_TOOLCHAIN --component rustfmt --component clippy + run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy - name: Cache rust cargo artifacts uses: swatinem/rust-cache@v2 - name: Run rustfmt @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@master - name: Install Rust ${{ matrix.rust }} - run: rustup toolchain install ${{ matrix.rust }} + run: rustup toolchain install --profile minimal ${{ matrix.rust }} - run: rustup override set ${{ matrix.rust }} - name: Cache rust cargo artifacts