From e653531934a8e6d1ef45c3f39202c364ba518f95 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:36:56 +0000 Subject: [PATCH 01/77] cargo: bump base64 from 0.13.1 to 0.20.0 Bumps [base64](https://github.com/marshallpierce/rust-base64) from 0.13.1 to 0.20.0. - [Release notes](https://github.com/marshallpierce/rust-base64/releases) - [Changelog](https://github.com/marshallpierce/rust-base64/blob/master/RELEASE-NOTES.md) - [Commits](https://github.com/marshallpierce/rust-base64/compare/v0.13.1...v0.20.0) --- updated-dependencies: - dependency-name: base64 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 +++++++- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff69d5210..fb9565f1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "base64ct" version = "1.5.1" @@ -877,7 +883,7 @@ dependencies = [ "async-smtp", "async_zip", "backtrace", - "base64 0.13.1", + "base64 0.20.0", "bitflags", "chrono", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 1c9d7230e..6e34c46eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ trust-dns-resolver = "0.22" tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar backtrace = "0.3" -base64 = "0.13" +base64 = "0.20" bitflags = "1.3" chrono = { version = "0.4", default-features=false, features = ["clock", "std"] } dirs = { version = "4", optional=true } From ac0fbaad2119f0e02dfceb60ae96deea418eea8f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Jan 2023 23:51:59 +0000 Subject: [PATCH 02/77] cargo: bump quick-xml from 0.26.0 to 0.27.1 Bumps [quick-xml](https://github.com/tafia/quick-xml) from 0.26.0 to 0.27.1. - [Release notes](https://github.com/tafia/quick-xml/releases) - [Changelog](https://github.com/tafia/quick-xml/blob/master/Changelog.md) - [Commits](https://github.com/tafia/quick-xml/compare/v0.26.0...v0.27.1) --- updated-dependencies: - dependency-name: quick-xml dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14e347901..7b22fdca2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2739,9 +2739,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 4cfd9d809..91d56d4af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ once_cell = "1.17.0" percent-encoding = "2.2" pgp = { version = "0.9", default-features = false } pretty_env_logger = { version = "0.4", optional = true } -quick-xml = "0.26" +quick-xml = "0.27" r2d2 = "0.8" r2d2_sqlite = "0.20" rand = "0.8" From d873f88b5646971060ca9efb49b9f76c45c7dda5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jan 2023 00:32:24 +0000 Subject: [PATCH 03/77] cargo: bump serde from 1.0.148 to 1.0.152 Bumps [serde](https://github.com/serde-rs/serde) from 1.0.148 to 1.0.152. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.148...v1.0.152) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 373223d0b..ddbd3c710 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3184,18 +3184,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.148" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.148" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", From 2b4e32d2cf7460a6f16eb29365464c5566bef5dd Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 4 Jan 2023 14:45:05 +0000 Subject: [PATCH 04/77] Remove unused `KeyType` from `DcKey` trait It always equals Self. --- src/key.rs | 24 ++++++++---------------- src/keyring.rs | 2 +- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/key.rs b/src/key.rs index e2fbe7c82..82ea25667 100644 --- a/src/key.rs +++ b/src/key.rs @@ -30,17 +30,13 @@ pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; /// [SignedSecretKey] types and makes working with them a little /// easier in the deltachat world. pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone { - type KeyType: Serialize + Deserializable + KeyTrait + Clone; - /// Create a key from some bytes. - fn from_slice(bytes: &[u8]) -> Result { - Ok(::from_bytes(Cursor::new( - bytes, - ))?) + fn from_slice(bytes: &[u8]) -> Result { + Ok(::from_bytes(Cursor::new(bytes))?) } /// Create a key from a base64 string. - fn from_base64(data: &str) -> Result { + fn from_base64(data: &str) -> Result { // strip newlines and other whitespace let cleaned: String = data.split_whitespace().collect(); let bytes = base64::decode(cleaned.as_bytes())?; @@ -51,15 +47,15 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone { /// /// Returns the key and a map of any headers which might have been set in /// the ASCII-armored representation. - fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap)> { + fn from_asc(data: &str) -> Result<(Self, BTreeMap)> { let bytes = data.as_bytes(); - Self::KeyType::from_armor_single(Cursor::new(bytes)).context("rPGP error") + Self::from_armor_single(Cursor::new(bytes)).context("rPGP error") } /// Load the users' default key from the database. fn load_self<'a>( context: &'a Context, - ) -> Pin> + 'a + Send>>; + ) -> Pin> + 'a + Send>>; /// Serialise the key as bytes. fn to_bytes(&self) -> Vec { @@ -92,11 +88,9 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone { } impl DcKey for SignedPublicKey { - type KeyType = SignedPublicKey; - fn load_self<'a>( context: &'a Context, - ) -> Pin> + 'a + Send>> { + ) -> Pin> + 'a + Send>> { Box::pin(async move { let addr = context.get_primary_self_addr().await?; match context @@ -143,11 +137,9 @@ impl DcKey for SignedPublicKey { } impl DcKey for SignedSecretKey { - type KeyType = SignedSecretKey; - fn load_self<'a>( context: &'a Context, - ) -> Pin> + 'a + Send>> { + ) -> Pin> + 'a + Send>> { Box::pin(async move { match context .sql diff --git a/src/keyring.rs b/src/keyring.rs index 192b8de80..fa5e9b5f4 100644 --- a/src/keyring.rs +++ b/src/keyring.rs @@ -19,7 +19,7 @@ where impl Keyring where - T: DcKey, + T: DcKey, { /// New empty keyring. pub fn new() -> Keyring { From 754c7324f51c0d33e1263c16d74d59ed161389cf Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 4 Jan 2023 14:37:12 +0000 Subject: [PATCH 05/77] Print more anyhow errors with their causes --- src/chat.rs | 12 ++++++------ src/context.rs | 4 ++-- src/download.rs | 2 +- src/html.rs | 2 +- src/job.rs | 10 +++++----- src/location.rs | 4 ++-- src/message.rs | 4 ++-- src/mimeparser.rs | 18 +++++++++--------- src/quota.rs | 4 ++-- src/receive_imf.rs | 27 +++++++++++++++------------ src/sql.rs | 8 ++++---- 11 files changed, 49 insertions(+), 46 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 4de92e070..8a5ee0bb2 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1129,7 +1129,7 @@ impl Chat { } } Err(err) => { - error!(context, "faild to load contacts for {}: {:?}", chat.id, err); + error!(context, "faild to load contacts for {}: {:#}", chat.id, err); } } chat.name = chat_name; @@ -2146,7 +2146,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result attach_selfavatar, Err(err) => { - warn!(context, "job: cannot get selfavatar-state: {}", err); + warn!(context, "job: cannot get selfavatar-state: {:#}", err); false } }; @@ -2208,27 +2208,27 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result error!(self, "Failed to start IO: {}", err), + Err(err) => error!(self, "Failed to start IO: {:#}", err), Ok(scheduler) => *lock = Some(scheduler), } } @@ -499,7 +499,7 @@ impl Context { match &*s { RunningState::Running { cancel_sender } => { if let Err(err) = cancel_sender.send(()).await { - warn!(self, "could not cancel ongoing: {:?}", err); + warn!(self, "could not cancel ongoing: {:#}", err); } info!(self, "Signaling the ongoing process to stop ASAP.",); *s = RunningState::ShallStop; diff --git a/src/download.rs b/src/download.rs index de127c7ae..015eb56f1 100644 --- a/src/download.rs +++ b/src/download.rs @@ -132,7 +132,7 @@ impl Job { /// Called in response to `Action::DownloadMsg`. pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status { if let Err(err) = imap.prepare(context).await { - warn!(context, "download: could not connect: {:?}", err); + warn!(context, "download: could not connect: {:#}", err); return Status::RetryNow; } diff --git a/src/html.rs b/src/html.rs index ac59793fc..279995292 100644 --- a/src/html.rs +++ b/src/html.rs @@ -250,7 +250,7 @@ impl MsgId { if !rawmime.is_empty() { match HtmlMsgParser::from_bytes(context, &rawmime).await { Err(err) => { - warn!(context, "get_html: parser error: {}", err); + warn!(context, "get_html: parser error: {:#}", err); Ok(None) } Ok(parser) => Ok(Some(parser.html)), diff --git a/src/job.rs b/src/job.rs index e3b513fed..689a7cade 100644 --- a/src/job.rs +++ b/src/job.rs @@ -157,7 +157,7 @@ impl Job { /// Synchronizes UIDs for all folders. async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status { if let Err(err) = imap.prepare(context).await { - warn!(context, "could not connect: {:?}", err); + warn!(context, "could not connect: {:#}", err); return Status::RetryLater; } @@ -246,7 +246,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ time_offset ); job.save(context).await.unwrap_or_else(|err| { - error!(context, "failed to save job: {}", err); + error!(context, "failed to save job: {:#}", err); }); } else { info!( @@ -254,7 +254,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ "remove job {} as it exhausted {} retries", job, JOB_RETRIES ); job.delete(context).await.unwrap_or_else(|err| { - error!(context, "failed to delete job: {}", err); + error!(context, "failed to delete job: {:#}", err); }); } } @@ -269,7 +269,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ } job.delete(context).await.unwrap_or_else(|err| { - error!(context, "failed to delete job: {}", err); + error!(context, "failed to delete job: {:#}", err); }); } } @@ -403,7 +403,7 @@ LIMIT 1; Ok(job) => return Ok(job), Err(err) => { // Remove invalid job from the DB - info!(context, "cleaning up job, because of {}", err); + info!(context, "cleaning up job, because of {:#}", err); // TODO: improve by only doing a single query let id = context diff --git a/src/location.rs b/src/location.rs index bff558912..dc6bf3af6 100644 --- a/src/location.rs +++ b/src/location.rs @@ -337,7 +337,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64 ContactId::SELF, ] ).await { - warn!(context, "failed to store location {:?}", err); + warn!(context, "failed to store location {:#}", err); } else { info!(context, "stored location for chat {}", chat_id); continue_streaming = true; @@ -638,7 +638,7 @@ pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receive loop { let next_event = match maybe_send_locations(context).await { Err(err) => { - warn!(context, "maybe_send_locations failed: {}", err); + warn!(context, "maybe_send_locations failed: {:#}", err); Some(60) // Retry one minute later. } Ok(next_event) => next_event, diff --git a/src/message.rs b/src/message.rs index d05b8b366..e8cc3a936 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1687,7 +1687,7 @@ pub async fn get_unblocked_msg_cnt(context: &Context) -> usize { { Ok(res) => res, Err(err) => { - error!(context, "get_unblocked_msg_cnt() failed. {}", err); + error!(context, "get_unblocked_msg_cnt() failed. {:#}", err); 0 } } @@ -1707,7 +1707,7 @@ pub async fn get_request_msg_cnt(context: &Context) -> usize { { Ok(res) => res, Err(err) => { - error!(context, "get_request_msg_cnt() failed. {}", err); + error!(context, "get_request_msg_cnt() failed. {:#}", err); 0 } } diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 843053930..e7a2ea40e 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -251,7 +251,7 @@ impl MimeMessage { } Ok(None) => (Ok(mail), HashSet::new(), false), Err(err) => { - warn!(context, "decryption failed: {}", err); + warn!(context, "decryption failed: {:#}", err); (Err(err), HashSet::new(), false) } }; @@ -380,7 +380,7 @@ impl MimeMessage { typ: Viewtype::Text, msg_raw: Some(txt.clone()), msg: txt, - error: Some(format!("Decrypting failed: {}", err)), + error: Some(format!("Decrypting failed: {:#}", err)), ..Default::default() }; parser.parts.push(part); @@ -680,7 +680,7 @@ impl MimeMessage { Err(err) => { warn!( context, - "Could not save decoded avatar to blob file: {}", err + "Could not save decoded avatar to blob file: {:#}", err ); None } @@ -987,7 +987,7 @@ impl MimeMessage { let decoded_data = match mail.get_body() { Ok(decoded_data) => decoded_data, Err(err) => { - warn!(context, "Invalid body parsed {:?}", err); + warn!(context, "Invalid body parsed {:#}", err); // Note that it's not always an error - might be no data return Ok(false); } @@ -1007,7 +1007,7 @@ impl MimeMessage { let decoded_data = match mail.get_body() { Ok(decoded_data) => decoded_data, Err(err) => { - warn!(context, "Invalid body parsed {:?}", err); + warn!(context, "Invalid body parsed {:#}", err); // Note that it's not always an error - might be no data return Ok(false); } @@ -1139,7 +1139,7 @@ impl MimeMessage { if filename.starts_with("location") || filename.starts_with("message") { let parsed = location::Kml::parse(decoded_data) .map_err(|err| { - warn!(context, "failed to parse kml part: {}", err); + warn!(context, "failed to parse kml part: {:#}", err); }) .ok(); if filename.starts_with("location") { @@ -1157,7 +1157,7 @@ impl MimeMessage { self.sync_items = context .parse_sync_items(serialized) .map_err(|err| { - warn!(context, "failed to parse sync data: {}", err); + warn!(context, "failed to parse sync data: {:#}", err); }) .ok(); return Ok(()); @@ -1179,7 +1179,7 @@ impl MimeMessage { Err(err) => { error!( context, - "Could not add blob for mime part {}, error {}", filename, err + "Could not add blob for mime part {}, error {:#}", filename, err ); return Ok(()); } @@ -1224,7 +1224,7 @@ impl MimeMessage { Err(err) => { warn!( context, - "PGP key attachment is not an ASCII-armored file: {}", err, + "PGP key attachment is not an ASCII-armored file: {:#}", err ); return Ok(false); } diff --git a/src/quota.rs b/src/quota.rs index eba52899f..9b233a349 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -134,7 +134,7 @@ impl Context { /// Called in response to `Action::UpdateRecentQuota`. pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result { if let Err(err) = imap.prepare(self).await { - warn!(self, "could not connect: {:?}", err); + warn!(self, "could not connect: {:#}", err); return Ok(Status::RetryNow); } @@ -162,7 +162,7 @@ impl Context { self.set_config(Config::QuotaExceeding, None).await?; } } - Err(err) => warn!(self, "cannot get highest quota usage: {:?}", err), + Err(err) => warn!(self, "cannot get highest quota usage: {:#}", err), } } diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 41bb20ff9..e8ad015af 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -102,7 +102,7 @@ pub(crate) async fn receive_imf_inner( let mut mime_parser = match MimeMessage::from_bytes_with_partial(context, imf_raw, is_partial_download).await { Err(err) => { - warn!(context, "receive_imf: can't parse MIME: {}", err); + warn!(context, "receive_imf: can't parse MIME: {:#}", err); let msg_ids; if !rfc724_mid.starts_with(GENERATED_PREFIX) { let row_id = context @@ -253,7 +253,7 @@ pub(crate) async fn receive_imf_inner( if from_id == ContactId::SELF { if mime_parser.was_encrypted() { if let Err(err) = context.execute_sync_items(sync_items).await { - warn!(context, "receive_imf cannot execute sync items: {}", err); + warn!(context, "receive_imf cannot execute sync items: {:#}", err); } } else { warn!(context, "sync items are not encrypted."); @@ -268,7 +268,7 @@ pub(crate) async fn receive_imf_inner( .receive_status_update(from_id, insert_msg_id, status_update) .await { - warn!(context, "receive_imf cannot update status: {}", err); + warn!(context, "receive_imf cannot update status: {:#}", err); } } @@ -290,7 +290,10 @@ pub(crate) async fn receive_imf_inner( context.emit_event(EventType::ChatModified(chat_id)); } Err(err) => { - warn!(context, "receive_imf cannot update profile image: {}", err); + warn!( + context, + "receive_imf cannot update profile image: {:#}", err + ); } }; } @@ -317,7 +320,7 @@ pub(crate) async fn receive_imf_inner( ) .await { - warn!(context, "cannot update contact status: {}", err); + warn!(context, "cannot update contact status: {:#}", err); } } @@ -495,7 +498,7 @@ async fn add_parts( securejoin_seen = false; } Err(err) => { - warn!(context, "Error in Secure-Join message handling: {}", err); + warn!(context, "Error in Secure-Join message handling: {:#}", err); chat_id = Some(DC_CHAT_ID_TRASH); securejoin_seen = true; } @@ -730,7 +733,7 @@ async fn add_parts( chat_id = None; } Err(err) => { - warn!(context, "Error in Secure-Join watching: {}", err); + warn!(context, "Error in Secure-Join watching: {:#}", err); chat_id = Some(DC_CHAT_ID_TRASH); } } @@ -870,7 +873,7 @@ async fn add_parts( Err(err) => { warn!( context, - "can't parse ephemeral timer \"{}\": {}", value, err + "can't parse ephemeral timer \"{}\": {:#}", value, err ); EphemeralTimer::Disabled } @@ -926,7 +929,7 @@ async fn add_parts( { warn!( context, - "failed to modify timer for chat {}: {}", chat_id, err + "failed to modify timer for chat {}: {:#}", chat_id, err ); } else { info!( @@ -975,7 +978,7 @@ async fn add_parts( if chat.is_protected() || new_status.is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id, to_ids).await { - warn!(context, "verification problem: {}", err); + warn!(context, "verification problem: {:#}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } else { @@ -1487,7 +1490,7 @@ async fn create_or_lookup_group( let create_protected = if mime_parser.get_header(HeaderDef::ChatVerified).is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id, to_ids).await { - warn!(context, "verification problem: {}", err); + warn!(context, "verification problem: {:#}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } @@ -1685,7 +1688,7 @@ async fn apply_group_changes( if mime_parser.get_header(HeaderDef::ChatVerified).is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id, to_ids).await { - warn!(context, "verification problem: {}", err); + warn!(context, "verification problem: {:#}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } diff --git a/src/sql.rs b/src/sql.rs index f24008390..1203a276e 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -626,26 +626,26 @@ pub async fn housekeeping(context: &Context) -> Result<()> { if let Err(err) = remove_unused_files(context).await { warn!( context, - "Housekeeping: cannot remove unusued files: {}", err + "Housekeeping: cannot remove unusued files: {:#}", err ); } if let Err(err) = start_ephemeral_timers(context).await { warn!( context, - "Housekeeping: cannot start ephemeral timers: {}", err + "Housekeeping: cannot start ephemeral timers: {:#}", err ); } if let Err(err) = prune_tombstones(&context.sql).await { warn!( context, - "Housekeeping: Cannot prune message tombstones: {}", err + "Housekeeping: Cannot prune message tombstones: {:#}", err ); } if let Err(err) = deduplicate_peerstates(&context.sql).await { - warn!(context, "Failed to deduplicate peerstates: {}", err) + warn!(context, "Failed to deduplicate peerstates: {:#}", err) } context.schedule_quota_update().await?; From f69acaa13db838b30dc3dd76f843bb449eb19779 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 4 Jan 2023 15:55:54 +0000 Subject: [PATCH 06/77] Add more logging and improve errors around folder selection --- src/imap.rs | 13 ++++++++++--- src/imap/select_folder.rs | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index cb3843ee0..84871cb89 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -554,7 +554,10 @@ impl Imap { folder: &str, ) -> Result { let session = self.session.as_mut().context("no session")?; - let newly_selected = session.select_or_create_folder(context, folder).await?; + let newly_selected = session + .select_or_create_folder(context, folder) + .await + .with_context(|| format!("failed to select or create folder {}", folder))?; let mailbox = session .selected_mailbox .as_mut() @@ -564,8 +567,12 @@ impl Imap { .uid_validity .with_context(|| format!("No UIDVALIDITY for folder {}", folder))?; - let old_uid_validity = get_uidvalidity(context, folder).await?; - let old_uid_next = get_uid_next(context, folder).await?; + let old_uid_validity = get_uidvalidity(context, folder) + .await + .with_context(|| format!("failed to get old UID validity for folder {}", folder))?; + let old_uid_next = get_uid_next(context, folder) + .await + .with_context(|| format!("failed to get old UID NEXT for folder {}", folder))?; if new_uid_validity == old_uid_validity { let new_emails = if newly_selected == NewlySelected::No { diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index 3f04f41a2..facf04ffc 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -109,13 +109,14 @@ impl ImapSession { Ok(newly_selected) => Ok(newly_selected), Err(err) => match err { Error::NoFolder(..) => { + info!(context, "Failed to select folder {} because it does not exist, trying to create it.", folder); self.create(folder).await.with_context(|| { format!("Couldn't select folder ('{}'), then create() failed", err) })?; - Ok(self.select_folder(context, Some(folder)).await?) + Ok(self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {}", folder))?) } - _ => Err(err.into()), + _ => Err(err).with_context(|| format!("failed to select folder {} with error other than NO, not trying to create it", folder)), }, } } From ea81d08c0159f648bc91c83ed3bc5230cc02da30 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 5 Jan 2023 23:15:09 +0000 Subject: [PATCH 07/77] ci: use default Rust toolchain for JSON-RPC tests Default preinstalled toolchain is currently at 1.65.0, there is no need to overwrite it with 1.66.0 for these tests. --- .github/workflows/jsonrpc.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/jsonrpc.yml b/.github/workflows/jsonrpc.yml index e84985229..de05f50d6 100644 --- a/.github/workflows/jsonrpc.yml +++ b/.github/workflows/jsonrpc.yml @@ -19,11 +19,6 @@ jobs: uses: actions/setup-node@v3 with: node-version: 16.x - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - name: Add Rust cache uses: Swatinem/rust-cache@v2 - name: npm install From d8f5e818805874d665ebee421da9bb78671fb73f Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 5 Jan 2023 23:32:43 +0000 Subject: [PATCH 08/77] jsonrpc: increase account request timeout to 60 seconds --- deltachat-jsonrpc/typescript/test/online.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/typescript/test/online.ts b/deltachat-jsonrpc/typescript/test/online.ts index 4bf7181c9..06c4c8954 100644 --- a/deltachat-jsonrpc/typescript/test/online.ts +++ b/deltachat-jsonrpc/typescript/test/online.ts @@ -12,7 +12,7 @@ describe("online tests", function () { let accountId1: number, accountId2: number; before(async function () { - this.timeout(12000); + this.timeout(60000); if (!process.env.DCC_NEW_TMP_EMAIL) { if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) { console.error( From 10349b7be4f6a4d123f5787b25e8184737201eaa Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 00:39:40 +0000 Subject: [PATCH 09/77] Update provider database --- CHANGELOG.md | 1 + src/provider/data.rs | 82 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ea187ef2..ae2eb0ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Buffer IMAP client writes #3888 - move `DC_CHAT_ID_ARCHIVED_LINK` to the top of chat lists and make `dc_get_fresh_msg_cnt()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3918 +- Update provider database ### API-Changes - jsonrpc: add python API for webxdc updates #3872 diff --git a/src/provider/data.rs b/src/provider/data.rs index 3b8b4b642..24dbdf6ef 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -526,7 +526,7 @@ static P_GMX_NET: Lazy = Lazy::new(|| Provider { oauth2_authorizer: None, }); -// hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, hermes.radio +// hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, ec.hermes.radio, ec1.hermes.radio, ec2.hermes.radio, ec3.hermes.radio, ec4.hermes.radio, ec5.hermes.radio, ec6.hermes.radio, ec7.hermes.radio, ec8.hermes.radio, ec9.hermes.radio, ec10.hermes.radio, ec11.hermes.radio, ec12.hermes.radio, ec13.hermes.radio, ec14.hermes.radio, ec15.hermes.radio, hermes.radio static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { id: "hermes.radio", status: Status::Ok, @@ -902,6 +902,35 @@ static P_NAVER: Lazy = Lazy::new(|| Provider { oauth2_authorizer: None, }); +// nubo.coop.md: nubo.coop +static P_NUBO_COOP: Lazy = Lazy::new(|| Provider { + id: "nubo.coop", + status: Status::Ok, + before_login_hint: "", + after_login_hint: "", + overview_page: "https://providers.delta.chat/nubo-coop", + server: vec![ + Server { + protocol: Imap, + socket: Ssl, + hostname: "mail.nubo.coop", + port: 993, + username_pattern: Email, + }, + Server { + protocol: Smtp, + socket: Ssl, + hostname: "mail.nubo.coop", + port: 465, + username_pattern: Email, + }, + ], + config_defaults: None, + strict_tls: true, + max_smtp_rcpt_to: None, + oauth2_authorizer: None, +}); + // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com, outlook.de static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { id: "outlook.com", @@ -931,6 +960,35 @@ static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { oauth2_authorizer: None, }); +// ouvaton.coop.md: ouvaton.org +static P_OUVATON_COOP: Lazy = Lazy::new(|| Provider { + id: "ouvaton.coop", + status: Status::Ok, + before_login_hint: "", + after_login_hint: "", + overview_page: "https://providers.delta.chat/ouvaton-coop", + server: vec![ + Server { + protocol: Imap, + socket: Ssl, + hostname: "imap.ouvaton.coop", + port: 993, + username_pattern: Email, + }, + Server { + protocol: Smtp, + socket: Ssl, + hostname: "smtp.ouvaton.coop", + port: 465, + username_pattern: Email, + }, + ], + config_defaults: None, + strict_tls: true, + max_smtp_rcpt_to: None, + oauth2_authorizer: None, +}); + // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us static P_POSTEO: Lazy = Lazy::new(|| Provider { id: "posteo", @@ -1659,6 +1717,22 @@ pub(crate) static PROVIDER_DATA: Lazy> ("ka13.hermes.radio", &*P_HERMES_RADIO), ("ka14.hermes.radio", &*P_HERMES_RADIO), ("ka15.hermes.radio", &*P_HERMES_RADIO), + ("ec.hermes.radio", &*P_HERMES_RADIO), + ("ec1.hermes.radio", &*P_HERMES_RADIO), + ("ec2.hermes.radio", &*P_HERMES_RADIO), + ("ec3.hermes.radio", &*P_HERMES_RADIO), + ("ec4.hermes.radio", &*P_HERMES_RADIO), + ("ec5.hermes.radio", &*P_HERMES_RADIO), + ("ec6.hermes.radio", &*P_HERMES_RADIO), + ("ec7.hermes.radio", &*P_HERMES_RADIO), + ("ec8.hermes.radio", &*P_HERMES_RADIO), + ("ec9.hermes.radio", &*P_HERMES_RADIO), + ("ec10.hermes.radio", &*P_HERMES_RADIO), + ("ec11.hermes.radio", &*P_HERMES_RADIO), + ("ec12.hermes.radio", &*P_HERMES_RADIO), + ("ec13.hermes.radio", &*P_HERMES_RADIO), + ("ec14.hermes.radio", &*P_HERMES_RADIO), + ("ec15.hermes.radio", &*P_HERMES_RADIO), ("hermes.radio", &*P_HERMES_RADIO), ("hey.com", &*P_HEY_COM), ("i.ua", &*P_I_UA), @@ -1681,12 +1755,14 @@ pub(crate) static PROVIDER_DATA: Lazy> ("mailo.com", &*P_MAILO_COM), ("nauta.cu", &*P_NAUTA_CU), ("naver.com", &*P_NAVER), + ("nubo.coop", &*P_NUBO_COOP), ("hotmail.com", &*P_OUTLOOK_COM), ("outlook.com", &*P_OUTLOOK_COM), ("office365.com", &*P_OUTLOOK_COM), ("outlook.com.tr", &*P_OUTLOOK_COM), ("live.com", &*P_OUTLOOK_COM), ("outlook.de", &*P_OUTLOOK_COM), + ("ouvaton.org", &*P_OUVATON_COOP), ("posteo.de", &*P_POSTEO), ("posteo.af", &*P_POSTEO), ("posteo.at", &*P_POSTEO), @@ -1861,7 +1937,9 @@ pub(crate) static PROVIDER_IDS: Lazy> = ("mailo.com", &*P_MAILO_COM), ("nauta.cu", &*P_NAUTA_CU), ("naver", &*P_NAVER), + ("nubo.coop", &*P_NUBO_COOP), ("outlook.com", &*P_OUTLOOK_COM), + ("ouvaton.coop", &*P_OUVATON_COOP), ("posteo", &*P_POSTEO), ("protonmail", &*P_PROTONMAIL), ("qq", &*P_QQ), @@ -1891,4 +1969,4 @@ pub(crate) static PROVIDER_IDS: Lazy> = }); pub static PROVIDER_UPDATED: Lazy = - Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2022, 7, 5).unwrap()); + Lazy::new(|| chrono::NaiveDate::from_ymd(2023, 1, 6)); From 3986bb6c4e4d3598b6472cf6f6afd3ef2c096adc Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 00:44:15 +0000 Subject: [PATCH 10/77] Fix provider update script --- src/provider/data.rs | 2 +- src/provider/update.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/provider/data.rs b/src/provider/data.rs index 24dbdf6ef..8c9516272 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -1969,4 +1969,4 @@ pub(crate) static PROVIDER_IDS: Lazy> = }); pub static PROVIDER_UPDATED: Lazy = - Lazy::new(|| chrono::NaiveDate::from_ymd(2023, 1, 6)); + Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 1, 6).unwrap()); diff --git a/src/provider/update.py b/src/provider/update.py index 9031e85b4..b67e9fb59 100755 --- a/src/provider/update.py +++ b/src/provider/update.py @@ -190,6 +190,6 @@ if __name__ == "__main__": now = datetime.datetime.utcnow() out_all += "pub static PROVIDER_UPDATED: Lazy = "\ - "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n" + "Lazy::new(|| chrono::NaiveDate::from_ymd_opt("+str(now.year)+", "+str(now.month)+", "+str(now.day)+").unwrap());\n" print(out_all) From eba96eb9047048d30b57db59042c3c10fe386e4d Mon Sep 17 00:00:00 2001 From: bjoern Date: Fri, 6 Jan 2023 11:22:30 +0100 Subject: [PATCH 11/77] mark all archived read (#3919) * let marknoticed_chat() work for DC_CHAT_ID_ARCHIVED_LINK * fix test_util::get_last_msg() - the first position may be the archive-link if 'adding specials' is allowed * add a test for the archived-link message counter * update CHANGELOG --- CHANGELOG.md | 1 + src/chat.rs | 159 ++++++++++++++++++++++++++++++++++++++++------ src/test_utils.rs | 4 +- 3 files changed, 141 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae2eb0ff3..580a3c60f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Buffer IMAP client writes #3888 - move `DC_CHAT_ID_ARCHIVED_LINK` to the top of chat lists and make `dc_get_fresh_msg_cnt()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3918 +- make `dc_marknoticed_chat()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3919 - Update provider database ### API-Changes diff --git a/src/chat.rs b/src/chat.rs index 8a5ee0bb2..d22cec77d 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2431,31 +2431,63 @@ pub(crate) async fn marknoticed_chat_if_older_than( } /// Marks all messages in the chat as noticed. +/// If the given chat-id is the archive-link, marks all messages in all archived chats as noticed. pub async fn marknoticed_chat(context: &Context, chat_id: ChatId) -> Result<()> { // "WHERE" below uses the index `(state, hidden, chat_id)`, see get_fresh_msg_cnt() for reasoning // the additional SELECT statement may speed up things as no write-blocking is needed. - let exists = context - .sql - .exists( - "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;", - paramsv![MessageState::InFresh, chat_id], - ) - .await?; - if !exists { - return Ok(()); - } + if chat_id.is_archived_link() { + let chat_ids_in_archive = context + .sql + .query_map( + "SELECT DISTINCT(m.chat_id) FROM msgs m + LEFT JOIN chats c ON m.chat_id=c.id + WHERE m.state=10 AND m.hidden=0 AND m.chat_id>9 AND c.blocked=0 AND c.archived=1", + paramsv![], + |row| row.get::<_, ChatId>(0), + |ids| ids.collect::, _>>().map_err(Into::into) + ) + .await?; + if chat_ids_in_archive.is_empty() { + return Ok(()); + } - context - .sql - .execute( - "UPDATE msgs - SET state=? - WHERE state=? - AND hidden=0 - AND chat_id=?;", - paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id], - ) - .await?; + context + .sql + .execute( + &format!( + "UPDATE msgs SET state=13 WHERE state=10 AND hidden=0 AND chat_id IN ({});", + sql::repeat_vars(chat_ids_in_archive.len()) + ), + rusqlite::params_from_iter(&chat_ids_in_archive), + ) + .await?; + for chat_id_in_archive in chat_ids_in_archive { + context.emit_event(EventType::MsgsNoticed(chat_id_in_archive)); + } + } else { + let exists = context + .sql + .exists( + "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;", + paramsv![MessageState::InFresh, chat_id], + ) + .await?; + if !exists { + return Ok(()); + } + + context + .sql + .execute( + "UPDATE msgs + SET state=? + WHERE state=? + AND hidden=0 + AND chat_id=?;", + paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id], + ) + .await?; + } context.emit_event(EventType::MsgsNoticed(chat_id)); @@ -4460,6 +4492,91 @@ mod tests { Ok(()) } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_archive_fresh_msgs() -> Result<()> { + let t = TestContext::new_alice().await; + + async fn msg_from(t: &TestContext, name: &str, num: u32) -> Result<()> { + receive_imf( + t, + format!( + "From: {}@example.net\n\ + To: alice@example.org\n\ + Message-ID: <{}@example.org>\n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2022 19:37:57 +0000\n\ + \n\ + hello\n", + name, num + ) + .as_bytes(), + false, + ) + .await?; + Ok(()) + } + + // receive some messages in archived+muted chats + msg_from(&t, "bob", 1).await?; + let bob_chat_id = t.get_last_msg().await.get_chat_id(); + bob_chat_id.accept(&t).await?; + set_muted(&t, bob_chat_id, MuteDuration::Forever).await?; + bob_chat_id + .set_visibility(&t, ChatVisibility::Archived) + .await?; + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 0); + + msg_from(&t, "bob", 2).await?; + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1); + + msg_from(&t, "bob", 3).await?; + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1); + + msg_from(&t, "claire", 4).await?; + let claire_chat_id = t.get_last_msg().await.get_chat_id(); + claire_chat_id.accept(&t).await?; + set_muted(&t, claire_chat_id, MuteDuration::Forever).await?; + claire_chat_id + .set_visibility(&t, ChatVisibility::Archived) + .await?; + msg_from(&t, "claire", 5).await?; + msg_from(&t, "claire", 6).await?; + msg_from(&t, "claire", 7).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 3); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2); + + // mark one of the archived+muted chats as noticed: check that the archive-link counter is changed as well + marknoticed_chat(&t, claire_chat_id).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1); + + // receive some more messages + msg_from(&t, "claire", 8).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 1); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(t.get_fresh_msgs().await?.len(), 0); + + msg_from(&t, "dave", 9).await?; + let dave_chat_id = t.get_last_msg().await.get_chat_id(); + dave_chat_id.accept(&t).await?; + assert_eq!(dave_chat_id.get_fresh_msg_cnt(&t).await?, 1); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(t.get_fresh_msgs().await?.len(), 1); + + // mark the archived-link as noticed: check that the real chats are noticed as well + marknoticed_chat(&t, DC_CHAT_ID_ARCHIVED_LINK).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(dave_chat_id.get_fresh_msg_cnt(&t).await?, 1); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(t.get_fresh_msgs().await?.len(), 1); + + Ok(()) + } + async fn get_chats_from_chat_list(ctx: &Context, listflags: usize) -> Vec { let chatlist = Chatlist::try_load(ctx, listflags, None, None) .await diff --git a/src/test_utils.rs b/src/test_utils.rs index ecf219d5c..8204602e8 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -21,7 +21,7 @@ use crate::chat::{self, Chat, ChatId}; use crate::chatlist::Chatlist; use crate::config::Config; use crate::constants::Chattype; -use crate::constants::{DC_GCM_ADDDAYMARKER, DC_MSG_ID_DAYMARKER}; +use crate::constants::{DC_GCL_NO_SPECIALS, DC_GCM_ADDDAYMARKER, DC_MSG_ID_DAYMARKER}; use crate::contact::{Contact, ContactId, Modifier, Origin}; use crate::context::Context; use crate::events::{Event, EventType, Events}; @@ -502,7 +502,7 @@ impl TestContext { /// Gets the most recent message over all chats. pub async fn get_last_msg(&self) -> Message { - let chats = Chatlist::try_load(&self.ctx, 0, None, None) + let chats = Chatlist::try_load(&self.ctx, DC_GCL_NO_SPECIALS, None, None) .await .expect("failed to load chatlist"); // 0 is correct in the next line (as opposed to `chats.len() - 1`, which would be the last element): From f4c674fa98c93e2ec5727e51337aba0f9d317356 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 5 Jan 2023 22:59:12 +0000 Subject: [PATCH 12/77] python: set reasonable timeouts for account requests `requests` library does not have a timeout at all by default. --- .../src/deltachat_rpc_client/pytestplugin.py | 5 ++++- python/src/deltachat/testplugin.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py index 4b8562af7..6bfd446d1 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py @@ -12,8 +12,11 @@ from .rpc import Rpc async def get_temp_credentials() -> dict: url = os.getenv("DCC_NEW_TMP_EMAIL") assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set" + + # Replace default 5 minute timeout with a 1 minute timeout. + timeout = aiohttp.ClientTimeout(total=60) async with aiohttp.ClientSession() as session: - async with session.post(url) as response: + async with session.post(url, timeout=timeout) as response: return json.loads(await response.text()) diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 335e384a5..ed92917b6 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -176,7 +176,7 @@ class TestProcess: try: yield self._configlist[index] except IndexError: - res = requests.post(liveconfig_opt) + res = requests.post(liveconfig_opt, timeout=60) if res.status_code != 200: pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text)) d = res.json() From 6e40fd8000fd39a283eb888c759d272c3548455e Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 02:12:36 +0000 Subject: [PATCH 13/77] message: derive `Ord` for `MessageState` --- src/message.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index e8cc3a936..35c69f9b2 100644 --- a/src/message.rs +++ b/src/message.rs @@ -586,7 +586,7 @@ impl Message { } pub fn is_sent(&self) -> bool { - self.state as i32 >= MessageState::OutDelivered as i32 + self.state >= MessageState::OutDelivered } pub fn is_forwarded(&self) -> bool { @@ -899,6 +899,8 @@ impl Message { Copy, PartialEq, Eq, + PartialOrd, + Ord, FromPrimitive, ToPrimitive, ToSql, From ecc7758788651c0c2df22e757b87ae85be47adda Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 31 Dec 2022 12:14:27 +0000 Subject: [PATCH 14/77] Add documentation to `contact` module --- src/contact.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/contact.rs b/src/contact.rs index baf74b4ac..612b0eee8 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -1,7 +1,5 @@ //! Contacts module -#![allow(missing_docs)] - use std::cmp::Reverse; use std::collections::BinaryHeap; use std::convert::{TryFrom, TryInto}; @@ -48,12 +46,18 @@ const SEEN_RECENTLY_SECONDS: i64 = 600; pub struct ContactId(u32); impl ContactId { + /// Undefined contact. Used as a placeholder for trashed messages. pub const UNDEFINED: ContactId = ContactId::new(0); + /// The owner of the account. /// /// The email-address is set by `set_config` using "addr". pub const SELF: ContactId = ContactId::new(1); + + /// ID of the contact for info messages. pub const INFO: ContactId = ContactId::new(2); + + /// ID of the contact for device messages. pub const DEVICE: ContactId = ContactId::new(5); const LAST_SPECIAL: ContactId = ContactId::new(9); @@ -177,6 +181,8 @@ pub struct Contact { )] #[repr(u32)] pub enum Origin { + /// Unknown origin. Can be used as a minimum origin to specify that the caller does not care + /// about origin of the contact. Unknown = 0, /// The contact is a mailing list address, needed to unblock mailing lists @@ -257,12 +263,13 @@ pub(crate) enum Modifier { Created, } +/// Verification status of the contact. #[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)] #[repr(u8)] pub enum VerifiedStatus { /// Contact is not verified. Unverified = 0, - // TODO: is this a thing? + /// SELF has verified the fingerprint of a contact. Currently unused. Verified = 1, /// SELF and contact have verified their fingerprints in both directions; in the UI typically checkmarks are shown. BidirectVerified = 2, @@ -275,6 +282,7 @@ impl Default for VerifiedStatus { } impl Contact { + /// Loads a contact snapshot from the database. pub async fn load_from_db(context: &Context, contact_id: ContactId) -> Result { let mut contact = context .sql @@ -847,6 +855,7 @@ impl Contact { Ok(()) } + /// Returns number of blocked contacts. pub async fn get_blocked_cnt(context: &Context) -> Result { let count = context .sql @@ -1138,7 +1147,7 @@ impl Contact { Ok(VerifiedStatus::Unverified) } - /// Return the address that verified the given contact + /// Returns the address that verified the given contact. pub async fn get_verifier_addr( context: &Context, contact_id: &ContactId, @@ -1150,6 +1159,7 @@ impl Contact { .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned()))) } + /// Returns the ContactId that verified the given contact. pub async fn get_verifier_id( context: &Context, contact_id: &ContactId, @@ -1162,7 +1172,7 @@ impl Contact { } } - /// Return the ContactId that verified the given contact + /// Returns the number of real (i.e. non-special) contacts in the database. pub async fn get_real_cnt(context: &Context) -> Result { if !context.sql.is_open().await { return Ok(0); @@ -1178,6 +1188,7 @@ impl Contact { Ok(count) } + /// Returns true if a contact with this ID exists. pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result { if contact_id.is_special() { return Ok(false); @@ -1193,6 +1204,7 @@ impl Contact { Ok(exists) } + /// Updates the origin of the contact, but only if new origin is higher than the current one. pub async fn scaleup_origin_by_id( context: &Context, contact_id: ContactId, @@ -1453,6 +1465,7 @@ fn cat_fingerprint( } } +/// Compares two email addresses, normalizing them beforehand. pub fn addr_cmp(addr1: &str, addr2: &str) -> bool { let norm1 = addr_normalize(addr1).to_lowercase(); let norm2 = addr_normalize(addr2).to_lowercase(); From 8e65e794bca56ca5dbe8aadbecc23dcd508e543e Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 28 Dec 2022 23:03:38 +0000 Subject: [PATCH 15/77] format-flowed: make quotes round-trip --- CHANGELOG.md | 1 + format-flowed/src/lib.rs | 76 ++++++++++++++++++------- fuzz/fuzz_targets/fuzz_format_flowed.rs | 5 +- src/message.rs | 5 ++ 4 files changed, 62 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 580a3c60f..bf0e1b21b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Fix STARTTLS connection and add a test for it #3907 - Trigger reconnection when failing to fetch existing messages #3911 - Do not retry fetching existing messages after failure, prevents infinite reconnection loop #3913 +- Ensure format=flowed formatting is always reversible on the receiver side #3880 ## 1.104.0 diff --git a/format-flowed/src/lib.rs b/format-flowed/src/lib.rs index 52b6827e5..10f9400b5 100644 --- a/format-flowed/src/lib.rs +++ b/format-flowed/src/lib.rs @@ -24,7 +24,7 @@ fn format_line_flowed(line: &str, prefix: &str) -> String { let mut result = String::new(); let mut buffer = prefix.to_string(); - let mut after_space = false; + let mut after_space = prefix.ends_with(' '); for c in line.chars() { if c == ' ' { @@ -55,7 +55,7 @@ fn format_line_flowed(line: &str, prefix: &str) -> String { result + &buffer } -/// Returns text formatted according to RFC 3767 (format=flowed). +/// Returns text formatted according to RFC 3676 (format=flowed). /// /// This function accepts text separated by LF, but returns text /// separated by CRLF. @@ -70,23 +70,20 @@ pub fn format_flowed(text: &str) -> String { result += "\r\n"; } - let line_no_prefix = line - .strip_prefix('>') - .map(|line| line.strip_prefix(' ').unwrap_or(line)); - let is_quote = line_no_prefix.is_some(); - let line = line_no_prefix.unwrap_or(line).trim_end(); - let prefix = if is_quote { "> " } else { "" }; + let line = line.trim_end(); + let quote_depth = line.chars().take_while(|&c| c == '>').count(); + let (prefix, mut line) = line.split_at(quote_depth); - if prefix.len() + line.len() > 78 { - result += &format_line_flowed(line, prefix); - } else { - result += prefix; - if prefix.is_empty() && (line.starts_with('>') || line.starts_with(' ')) { - // Space stuffing, see RFC 3676 - result.push(' '); + let mut prefix = prefix.to_string(); + + if quote_depth > 0 { + if let Some(s) = line.strip_prefix(' ') { + line = s; + prefix += " "; } - result += line; } + + result += &format_line_flowed(line, &prefix); } result @@ -111,9 +108,6 @@ pub fn format_flowed_quote(text: &str) -> String { /// /// Lines must be separated by single LF. /// -/// Quote processing is not supported, it is assumed that they are -/// deleted during simplification. -/// /// Signature separator line is not processed here, it is assumed to /// be stripped beforehand. pub fn unformat_flowed(text: &str, delsp: bool) -> String { @@ -121,6 +115,12 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String { let mut skip_newline = true; for line in text.split('\n') { + let line = if !result.is_empty() && skip_newline { + line.trim_start_matches('>') + } else { + line + }; + // Revert space-stuffing let line = line.strip_prefix(' ').unwrap_or(line); @@ -150,8 +150,20 @@ mod tests { #[test] fn test_format_flowed() { + let text = ""; + assert_eq!(format_flowed(text), ""); + let text = "Foo bar baz"; - assert_eq!(format_flowed(text), "Foo bar baz"); + assert_eq!(format_flowed(text), text); + + let text = ">Foo bar"; + assert_eq!(format_flowed(text), text); + + let text = "> Foo bar"; + assert_eq!(format_flowed(text), text); + + let text = ">\n\nA"; + assert_eq!(format_flowed(text), ">\r\n\r\nA"); let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\ \n\ @@ -165,17 +177,33 @@ mod tests { let text = "> A quote"; assert_eq!(format_flowed(text), "> A quote"); + let text = "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A"; + assert_eq!( + format_flowed(text), + "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > \r\n> A" + ); + // Test space stuffing of wrapped lines let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\ > \n\ > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."; let expected = "> This is the Autocrypt Setup Message used to transfer your key between \r\n\ > clients.\r\n\ - > \r\n\ + >\r\n\ > To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\ > client and enter the setup code presented on the generating device."; assert_eq!(format_flowed(text), expected); + let text = ">> This is the Autocrypt Setup Message used to transfer your key between clients.\n\ + >> \n\ + >> To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."; + let expected = ">> This is the Autocrypt Setup Message used to transfer your key between \r\n\ + >> clients.\r\n\ + >>\r\n\ + >> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\ + >> client and enter the setup code presented on the generating device."; + assert_eq!(format_flowed(text), expected); + // Test space stuffing of spaces. let text = " Foo bar baz"; assert_eq!(format_flowed(text), " Foo bar baz"); @@ -202,6 +230,12 @@ mod tests { let text = " Foo bar"; let expected = " Foo bar"; assert_eq!(unformat_flowed(text, false), expected); + + let text = + "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > \n> A"; + let expected = + "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A"; + assert_eq!(unformat_flowed(text, false), expected); } #[test] diff --git a/fuzz/fuzz_targets/fuzz_format_flowed.rs b/fuzz/fuzz_targets/fuzz_format_flowed.rs index f44bebf94..8f779a468 100644 --- a/fuzz/fuzz_targets/fuzz_format_flowed.rs +++ b/fuzz/fuzz_targets/fuzz_format_flowed.rs @@ -10,10 +10,7 @@ fn round_trip(input: &str) -> String { fn main() { check!().for_each(|data: &[u8]| { if let Ok(input) = std::str::from_utf8(data.into()) { - let mut input = input.to_string(); - - // Only consider inputs that don't contain quotes. - input.retain(|c| c != '>'); + let input = input.trim().to_string(); // Only consider inputs that are the result of unformatting format=flowed text. // At least this means that lines don't contain any trailing whitespace. diff --git a/src/message.rs b/src/message.rs index 35c69f9b2..678415378 100644 --- a/src/message.rs +++ b/src/message.rs @@ -2390,6 +2390,11 @@ mod tests { let received = bob.recv_msg(&sent).await; assert_eq!(received.text.as_deref(), Some(text)); + let text = "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A"; + let sent = alice.send_text(chat.id, text).await; + let received = bob.recv_msg(&sent).await; + assert_eq!(received.text.as_deref(), Some(text)); + let python_program = "\ def hello(): return 'Hello, world!'"; From 9eb7a84d832d2f4f4495b0ca909159ca7954b7b7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 16:34:49 +0000 Subject: [PATCH 16/77] Use TestContextManager for test_format_flowed_round_trip test --- src/message.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/message.rs b/src/message.rs index 678415378..5535666bd 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1889,7 +1889,7 @@ mod tests { use crate::chatlist::Chatlist; use crate::receive_imf::receive_imf; use crate::test_utils as test; - use crate::test_utils::TestContext; + use crate::test_utils::{TestContext, TestContextManager}; use super::*; @@ -2376,8 +2376,9 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_format_flowed_round_trip() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + let bob = tcm.bob().await; let chat = alice.create_chat(&bob).await; let text = " Foo bar"; From 4f8593f46a053876d62445aac39a6a66762985d0 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 21:58:47 +0000 Subject: [PATCH 17/77] Update base64 API usage --- src/tools.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/tools.rs b/src/tools.rs index b766dc73e..b02ed2fea 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -277,6 +277,12 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time /// - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header /// - the group-id should be a string with the characters [a-zA-Z0-9\-_] pub(crate) fn create_id() -> String { + const URL_SAFE_ENGINE: base64::engine::fast_portable::FastPortable = + base64::engine::fast_portable::FastPortable::from( + &base64::alphabet::URL_SAFE, + base64::engine::fast_portable::NO_PAD, + ); + // ThreadRng implements CryptoRng trait and is supposed to be cryptographically secure. let mut rng = thread_rng(); @@ -285,7 +291,7 @@ pub(crate) fn create_id() -> String { rng.fill(&mut arr[..]); // Take 11 base64 characters containing 66 random bits. - base64::encode_config(arr, base64::URL_SAFE) + base64::encode_engine(arr, &URL_SAFE_ENGINE) .chars() .take(11) .collect() From 3df5f6110c22c810b8910bf69e8eb6893056bf4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Jan 2023 23:10:29 +0000 Subject: [PATCH 18/77] cargo: bump tokio from 1.23.0 to 1.23.1 Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.23.0 to 1.23.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.23.0...tokio-1.23.1) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- deltachat-jsonrpc/Cargo.toml | 4 ++-- deltachat-rpc-server/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb0d51376..5b2c268a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3544,9 +3544,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" dependencies = [ "autocfg", "bytes", diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index dbe8b2d17..49c322197 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -23,7 +23,7 @@ futures = { version = "0.3.25" } serde_json = "1.0.89" yerpc = { version = "^0.3.1", features = ["anyhow_expose"] } typescript-type-def = { version = "0.5.5", features = ["json_value"] } -tokio = { version = "1.23.0" } +tokio = { version = "1.23.1" } sanitize-filename = "0.4" walkdir = "2.3.2" @@ -32,7 +32,7 @@ axum = { version = "0.6.1", optional = true, features = ["ws"] } env_logger = { version = "0.10.0", optional = true } [dev-dependencies] -tokio = { version = "1.23.0", features = ["full", "rt-multi-thread"] } +tokio = { version = "1.23.1", features = ["full", "rt-multi-thread"] } [features] diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index b6f10996d..2a7ce674c 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -21,5 +21,5 @@ futures-lite = "1.12.0" log = "0.4" serde_json = "1.0.89" serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.23.0", features = ["io-std"] } +tokio = { version = "1.23.1", features = ["io-std"] } yerpc = { version = "0.3.1", features = ["anyhow_expose"] } From 087b4289e5443758b2b28d73d089fa69be0ecfb5 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 7 Jan 2023 00:34:47 +0100 Subject: [PATCH 19/77] jsonrpc: add fresh message count to the archive-link chatlistitem variant (#3920) this is a followup to #3918 I went with option "C" from my comment: https://github.com/deltachat/deltachat-core-rust/pull/3918#issuecomment-1371224339 - Archive link is (still) very different from a normal chat, so most of the options would be empty / not relevant - avatar path is not needed as desktop renders it differently anyway, it's not really a chat like saved messages or device messages where it made more sense for the core to supply the icon, vs. using the svg directly. - translating the string in the coreas stock-string is more annoying than doing it from the ui, especially when this special pseudo chat has different rendering anyway so also no need to provide a name property --- CHANGELOG.md | 1 + deltachat-jsonrpc/src/api/types/chat_list.rs | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf0e1b21b..37715b863 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ ### API-Changes - jsonrpc: add python API for webxdc updates #3872 +- jsonrpc: add fresh message count to ChatListItemFetchResult::ArchiveLink - Add ffi functions to retrieve `verified by` information #3786 ### Fixes diff --git a/deltachat-jsonrpc/src/api/types/chat_list.rs b/deltachat-jsonrpc/src/api/types/chat_list.rs index 45933770f..83b84dd3d 100644 --- a/deltachat-jsonrpc/src/api/types/chat_list.rs +++ b/deltachat-jsonrpc/src/api/types/chat_list.rs @@ -48,12 +48,10 @@ pub enum ChatListItemFetchResult { dm_chat_contact: Option, was_seen_recently: bool, }, - ArchiveLink, #[serde(rename_all = "camelCase")] - Error { - id: u32, - error: String, - }, + ArchiveLink { fresh_message_counter: usize }, + #[serde(rename_all = "camelCase")] + Error { id: u32, error: String }, } pub(crate) async fn get_chat_list_item_by_id( @@ -66,8 +64,12 @@ pub(crate) async fn get_chat_list_item_by_id( _ => Some(MsgId::new(entry.1)), }; + let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?; + if chat_id.is_archived_link() { - return Ok(ChatListItemFetchResult::ArchiveLink); + return Ok(ChatListItemFetchResult::ArchiveLink { + fresh_message_counter, + }); } let chat = Chat::load_from_db(ctx, chat_id).await?; @@ -111,7 +113,6 @@ pub(crate) async fn get_chat_list_item_by_id( (None, false) }; - let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?; let color = color_int_to_hex_string(chat.get_color(ctx).await?); Ok(ChatListItemFetchResult::ChatListItem { From 58ba107d011032f7567c083f67d536613c27916e Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 19:22:57 +0000 Subject: [PATCH 20/77] Resultify Message::get_filebytes() --- CHANGELOG.md | 1 + deltachat-ffi/src/lib.rs | 2 ++ deltachat-jsonrpc/src/api/types/message.rs | 2 +- src/chat.rs | 2 +- src/message.rs | 13 +++++++------ src/mimefactory.rs | 2 +- src/mimeparser.rs | 2 +- src/tools.rs | 10 ++++------ 8 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37715b863..b28312450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - jsonrpc: add python API for webxdc updates #3872 - jsonrpc: add fresh message count to ChatListItemFetchResult::ArchiveLink - Add ffi functions to retrieve `verified by` information #3786 +- resultify `Message::get_filebytes()` #3925 ### Fixes - Do not add an error if the message is encrypted but not signed #3860 diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 764f1acad..7de6e4cd8 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -3309,6 +3309,8 @@ pub unsafe extern "C" fn dc_msg_get_filebytes(msg: *mut dc_msg_t) -> u64 { let ctx = &*ffi_msg.context; block_on(ffi_msg.message.get_filebytes(ctx)) + .unwrap_or_log_default(ctx, "Cannot get file size") + .unwrap_or_default() } #[no_mangle] diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index c7e9cfe8d..0f35f0c6f 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -105,7 +105,7 @@ impl MessageObject { let sender_contact = Contact::load_from_db(context, message.get_from_id()).await?; let sender = ContactObject::try_from_dc_contact(context, sender_contact).await?; - let file_bytes = message.get_filebytes(context).await; + let file_bytes = message.get_filebytes(context).await?.unwrap_or_default(); let override_sender_name = message.get_override_sender_name(); let webxdc_info = if message.get_viewtype() == Viewtype::Webxdc { diff --git a/src/chat.rs b/src/chat.rs index d22cec77d..618319cb1 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -5202,7 +5202,7 @@ mod tests { assert_eq!(msg.get_filename(), Some(filename.to_string())); assert_eq!(msg.get_width(), w); assert_eq!(msg.get_height(), h); - assert!(msg.get_filebytes(&bob).await > 250); + assert!(msg.get_filebytes(&bob).await?.unwrap() > 250); Ok(()) } diff --git a/src/message.rs b/src/message.rs index 5535666bd..b1581137f 100644 --- a/src/message.rs +++ b/src/message.rs @@ -494,11 +494,12 @@ impl Message { .map(|name| name.to_string_lossy().to_string()) } - pub async fn get_filebytes(&self, context: &Context) -> u64 { - match self.param.get_path(Param::File, context) { - Ok(Some(path)) => get_filebytes(context, &path).await, - Ok(None) => 0, - Err(_) => 0, + /// Returns the size of the file in bytes, if applicable. + pub async fn get_filebytes(&self, context: &Context) -> Result> { + if let Some(path) = self.param.get_path(Param::File, context)? { + Ok(Some(get_filebytes(context, &path).await?)) + } else { + Ok(None) } } @@ -1102,7 +1103,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { } if let Some(path) = msg.get_file(context) { - let bytes = get_filebytes(context, &path).await; + let bytes = get_filebytes(context, &path).await?; ret += &format!("\nFile: {}, {}, bytes\n", path.display(), bytes); } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 43f3e5886..5d7d51fe8 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1432,7 +1432,7 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool async fn is_file_size_okay(context: &Context, msg: &Message) -> Result { match msg.param.get_path(Param::File, context)? { Some(path) => { - let bytes = get_filebytes(context, &path).await; + let bytes = get_filebytes(context, &path).await?; Ok(bytes <= UPPER_LIMIT_FILE_SIZE) } None => Ok(false), diff --git a/src/mimeparser.rs b/src/mimeparser.rs index e7a2ea40e..c83709673 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -3145,7 +3145,7 @@ On 2020-10-25, Bob wrote: assert_eq!(msg.is_dc_message, MessengerMessage::No); assert_eq!(msg.chat_blocked, Blocked::Request); assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.get_filebytes(&t).await, 2115); + assert_eq!(msg.get_filebytes(&t).await.unwrap().unwrap(), 2115); assert!(msg.get_file(&t).is_some()); assert_eq!(msg.get_filename().unwrap(), "avatar64x64.png"); assert_eq!(msg.get_width(), 64); diff --git a/src/tools.rs b/src/tools.rs index b02ed2fea..e9c34f5e1 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -364,12 +364,10 @@ pub(crate) fn get_abs_path(context: &Context, path: impl AsRef) -> PathBuf } } -pub(crate) async fn get_filebytes(context: &Context, path: impl AsRef) -> u64 { +pub(crate) async fn get_filebytes(context: &Context, path: impl AsRef) -> Result { let path_abs = get_abs_path(context, &path); - match fs::metadata(&path_abs).await { - Ok(meta) => meta.len(), - Err(_err) => 0, - } + let meta = fs::metadata(&path_abs).await?; + Ok(meta.len()) } pub(crate) async fn delete_file(context: &Context, path: impl AsRef) -> bool { @@ -1055,7 +1053,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; .is_ok()); assert!(file_exist!(context, "$BLOBDIR/foobar")); assert!(!file_exist!(context, "$BLOBDIR/foobarx")); - assert_eq!(get_filebytes(context, "$BLOBDIR/foobar").await, 7); + assert_eq!(get_filebytes(context, "$BLOBDIR/foobar").await.unwrap(), 7); let abs_path = context .get_blobdir() From f2f211908dc7c7e73b60e8c0f36c471342d6a136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Jan 2023 15:58:34 +0000 Subject: [PATCH 21/77] cargo: bump serde_json from 1.0.89 to 1.0.91 Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.89 to 1.0.91. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.89...v1.0.91) --- updated-dependencies: - dependency-name: serde_json dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b2c268a9..0c0c3b778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3210,9 +3210,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa", "ryu", diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 49c322197..36e543b3e 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -20,7 +20,7 @@ tempfile = "3.3.0" log = "0.4" async-channel = { version = "1.8.0" } futures = { version = "0.3.25" } -serde_json = "1.0.89" +serde_json = "1.0.91" yerpc = { version = "^0.3.1", features = ["anyhow_expose"] } typescript-type-def = { version = "0.5.5", features = ["json_value"] } tokio = { version = "1.23.1" } diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 2a7ce674c..b6f839dd9 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -19,7 +19,7 @@ anyhow = "1" env_logger = { version = "0.10.0" } futures-lite = "1.12.0" log = "0.4" -serde_json = "1.0.89" +serde_json = "1.0.91" serde = { version = "1.0", features = ["derive"] } tokio = { version = "1.23.1", features = ["io-std"] } yerpc = { version = "0.3.1", features = ["anyhow_expose"] } From 15674111b440b816814ebf8544abf7052c61ca54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Jan 2023 16:39:18 +0000 Subject: [PATCH 22/77] Bump tokio from 1.23.0 to 1.24.1 in /fuzz Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.23.0 to 1.24.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.23.0...tokio-1.24.1) --- updated-dependencies: - dependency-name: tokio dependency-type: indirect ... Signed-off-by: dependabot[bot] --- fuzz/Cargo.lock | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index b2f3f5f36..46118154d 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -233,6 +233,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "base64ct" version = "1.5.3" @@ -702,7 +708,7 @@ dependencies = [ "async-smtp", "async_zip", "backtrace", - "base64 0.13.1", + "base64 0.20.0", "bitflags", "chrono", "deltachat_derive", @@ -719,7 +725,7 @@ dependencies = [ "kamadak-exif", "lettre_email", "libc", - "mailparse", + "mailparse 0.14.0", "native-tls", "num-derive", "num-traits", @@ -764,7 +770,7 @@ dependencies = [ "bolero", "deltachat", "format-flowed", - "mailparse", + "mailparse 0.13.8", ] [[package]] @@ -1691,6 +1697,17 @@ dependencies = [ "quoted_printable", ] +[[package]] +name = "mailparse" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b56570f5f8c0047260d1c8b5b331f62eb9c660b9dd4071a8c46f8c7d3f280aa" +dependencies = [ + "charset", + "data-encoding", + "quoted_printable", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -1887,9 +1904,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "opaque-debug" @@ -2195,9 +2212,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" dependencies = [ "memchr", ] @@ -2846,9 +2863,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", From 847611aed49a76959f484537cbd6cd5f4a1362cf Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sat, 7 Jan 2023 23:37:54 +0100 Subject: [PATCH 23/77] improve error of node tests (#3928) remove unused imports and fail without revealing the token --- node/test/test.js | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/node/test/test.js b/node/test/test.js index 3d6c93de9..d76566b04 100644 --- a/node/test/test.js +++ b/node/test/test.js @@ -1,33 +1,29 @@ // @ts-check -import DeltaChat, { Message } from '../dist' -import binding from '../binding' +import DeltaChat from '../dist' -import { deepEqual, deepStrictEqual, strictEqual } from 'assert' +import { deepStrictEqual, strictEqual } from 'assert' import chai, { expect } from 'chai' import chaiAsPromised from 'chai-as-promised' import { EventId2EventName, C } from '../dist/constants' import { join } from 'path' -import { mkdtempSync, statSync } from 'fs' -import { tmpdir } from 'os' +import { statSync } from 'fs' import { Context } from '../dist/context' +import fetch from 'node-fetch' chai.use(chaiAsPromised) -chai.config.truncateThreshold = 0; // Do not truncate assertion errors. +chai.config.truncateThreshold = 0 // Do not truncate assertion errors. async function createTempUser(url) { - const fetch = require('node-fetch') - async function postData(url = '') { // Default options are marked with * const response = await fetch(url, { method: 'POST', // *GET, POST, PUT, DELETE, etc. - mode: 'cors', // no-cors, *cors, same-origin - cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached - credentials: 'same-origin', // include, *same-origin, omit headers: { 'cache-control': 'no-cache', }, - referrerPolicy: 'no-referrer', // no-referrer, *client }) + if (!response.ok) { + throw new Error('request failed: ' + response.body.read()) + } return response.json() // parses JSON response into native JavaScript objects } From 872cd10b8b3ec112ef2a9c4d4033e42a411d9084 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 8 Jan 2023 09:58:53 +0000 Subject: [PATCH 24/77] Update tokio to 1.24.1 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c0c3b778..9afad7882 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3544,9 +3544,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.1" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", From 120a7a3090e46727b21c518ffb618cc643ffff82 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 8 Jan 2023 16:18:08 +0000 Subject: [PATCH 25/77] Prepare 1.105.0 release --- CHANGELOG.md | 9 +++++++++ Cargo.lock | 8 ++++---- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- package.json | 2 +- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b28312450..d0ca83889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### Changes + +### API-Changes + +### Fixes + + +## 1.105.0 + ### Changes - Validate signatures in try_decrypt() even if the message isn't encrypted #3859 - Don't parse the message again after detached signatures validation #3862 diff --git a/Cargo.lock b/Cargo.lock index 9afad7882..abe7cd0d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -873,7 +873,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deltachat" -version = "1.104.0" +version = "1.105.0" dependencies = [ "ansi_term", "anyhow", @@ -947,7 +947,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.104.0" +version = "1.105.0" dependencies = [ "anyhow", "async-channel", @@ -969,7 +969,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.104.0" +version = "1.105.0" dependencies = [ "anyhow", "deltachat-jsonrpc", @@ -992,7 +992,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.104.0" +version = "1.105.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index e8f36abe3..549d49dbe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.104.0" +version = "1.105.0" edition = "2021" license = "MPL-2.0" rust-version = "1.63" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index b6e18901b..c3c3dbdf1 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.104.0" +version = "1.105.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 36e543b3e..63677512c 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.104.0" +version = "1.105.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 9937dbabb..9c1df45e3 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -48,5 +48,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.104.0" + "version": "1.105.0" } \ No newline at end of file diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index b6f839dd9..a45572b74 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.104.0" +version = "1.105.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 3122efe66..09b9e179d 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.104.0" + "version": "1.105.0" } \ No newline at end of file From 0e849609f419843860493909e6fc669c7c4b78df Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 9 Jan 2023 16:20:56 +0000 Subject: [PATCH 26/77] ci: python: do not try to upload source distributions They are not useful because you still need a statically linked rust library besides the C FFI module, and they are not built by tox 4 without changes. --- scripts/concourse/docs_wheels.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml index d902947bc..bd2074d4b 100644 --- a/scripts/concourse/docs_wheels.yml +++ b/scripts/concourse/docs_wheels.yml @@ -104,9 +104,6 @@ jobs: outputs: - name: py-docs path: ./python/doc/_build/ - # Source packages - - name: py-dist - path: ./python/.docker-tox/dist/ # Binary wheels - name: py-wheels path: ./python/.docker-tox/wheelhouse/ @@ -145,7 +142,6 @@ jobs: config: inputs: - name: py-wheels - - name: py-dist image_resource: type: registry-image source: @@ -162,7 +158,6 @@ jobs: devpi use https://m.devpi.net/dc/master devpi login ((devpi.login)) --password ((devpi.password)) devpi upload py-wheels/*manylinux201* - devpi upload py-dist/* - name: python-aarch64 plan: From 07f2e28eca7ccd3e44d5e7a39b059dcce17563a8 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Mon, 9 Jan 2023 22:12:33 +0100 Subject: [PATCH 27/77] fix: only send contact changed event for recently seen if it is relevant (#3938) (no events for contacts that are already offline again) This dramatically speeds up replay after backup import on desktop. --- CHANGELOG.md | 2 +- src/contact.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ca83889..cf4f53026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ ### API-Changes ### Fixes - +- fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938 ## 1.105.0 diff --git a/src/contact.rs b/src/contact.rs index 612b0eee8..4f2189467 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -1416,6 +1416,7 @@ pub(crate) async fn update_last_seen( ) .await? > 0 + && timestamp > time() - SEEN_RECENTLY_SECONDS { context.interrupt_recently_seen(contact_id, timestamp).await; } From 8d119713bce74d14e19e40e6a315cbc36d677d67 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 9 Jan 2023 22:46:32 +0100 Subject: [PATCH 28/77] Print chats when a test failed (#3937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Print chats after a test failed again E.g. ``` ========== Chats of bob: ========== Single#Chat#10: alice@example.org [alice@example.org] -------------------------------------------------------------------------------- Msg#10: (Contact#Contact#10): hellooo [FRESH] Msg#11: (Contact#Contact#10): hellooo without mailing list [FRESH] -------------------------------------------------------------------------------- ========== Chats of alice: ========== Single#Chat#10: bob@example.net [bob@example.net] -------------------------------------------------------------------------------- Msg#10: Me (Contact#Contact#Self): hellooo √ Msg#11: Me (Contact#Contact#Self): hellooo without mailing list √ -------------------------------------------------------------------------------- ``` I found this very useful sometimes, so, let's try to re-introduce it (it was removed in #3449) * Add failing test for TestContext::drop() * Do not panic in TestContext::drop() if runtime is dropped Co-authored-by: link2xt --- src/test_utils.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index 8204602e8..cf19dafd8 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -14,6 +14,7 @@ use chat::ChatItem; use once_cell::sync::Lazy; use rand::Rng; use tempfile::{tempdir, TempDir}; +use tokio::runtime::Handle; use tokio::sync::RwLock; use tokio::task; @@ -263,7 +264,6 @@ impl TestContext { Self::builder().configure_fiona().build().await } - #[allow(dead_code)] /// Print current chat state. pub async fn print_chats(&self) { println!("\n========== Chats of {}: ==========", self.name()); @@ -702,6 +702,19 @@ impl Deref for TestContext { } } +impl Drop for TestContext { + fn drop(&mut self) { + task::block_in_place(move || { + if let Ok(handle) = Handle::try_current() { + // Print the chats if runtime still exists. + handle.block_on(async move { + self.print_chats().await; + }); + } + }); + } +} + pub enum LogEvent { /// Logged event. Event(Event), @@ -1079,4 +1092,12 @@ mod tests { bob.ctx.emit_event(EventType::Info("there".into())); // panic!("Both fail"); } + + /// Checks that dropping the `TestContext` after the runtime does not panic, + /// e.g. that `TestContext::drop` does not assume the runtime still exists. + #[test] + fn test_new_test_context() { + let runtime = tokio::runtime::Runtime::new().expect("unable to create tokio runtime"); + runtime.block_on(TestContext::new()); + } } From 2b8923931ed64839c446ebbb1789263c6f1bec84 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 10 Jan 2023 00:00:23 +0000 Subject: [PATCH 29/77] Add more context to IMAP errors --- src/imap.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 84871cb89..94d4223d2 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -698,9 +698,11 @@ impl Imap { let old_uid_next = get_uid_next(context, folder).await?; let msgs = if fetch_existing_msgs { - self.prefetch_existing_msgs().await? + self.prefetch_existing_msgs() + .await + .context("prefetch_existing_msgs")? } else { - self.prefetch(old_uid_next).await? + self.prefetch(old_uid_next).await.context("prefetch")? }; let read_cnt = msgs.len(); @@ -763,7 +765,7 @@ impl Imap { fetch_response.flags(), show_emails, ) - .await? + .await.context("prefetch_should_download")? { match download_limit { Some(download_limit) => uids_fetch.push(( @@ -799,7 +801,8 @@ impl Imap { fetch_partially, fetch_existing_msgs, ) - .await?; + .await + .context("fetch_many_msgs")?; received_msgs.extend(received_msgs_in_batch); largest_uid_fetched = max( largest_uid_fetched, From 01fe88e3371ef66ffa2f11776aaad0cd99324ce7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 10 Jan 2023 01:36:05 +0000 Subject: [PATCH 30/77] Save modified .toml if absolute paths were replaced with relative Previously this required adding or removing an account. --- CHANGELOG.md | 1 + src/accounts.rs | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf4f53026..942ac2a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixes - fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938 +- Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943 ## 1.105.0 diff --git a/src/accounts.rs b/src/accounts.rs index 2380de04e..8de0a7da8 100644 --- a/src/accounts.rs +++ b/src/accounts.rs @@ -367,13 +367,20 @@ impl Config { // Previous versions of the core stored absolute paths in account config. // Convert them to relative paths. + let mut modified = false; for account in &mut inner.accounts { if let Ok(new_dir) = account.dir.strip_prefix(dir) { account.dir = new_dir.to_path_buf(); + modified = true; } } - Ok(Config { file, inner }) + let config = Self { file, inner }; + if modified { + config.sync().await?; + } + + Ok(config) } /// Loads all accounts defined in the configuration file. From 68cd8dbc3e605731913247c218eaf168e38f6ab3 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 10 Jan 2023 14:18:40 +0100 Subject: [PATCH 31/77] Only send IncomingMsgBunch if there are more than 0 new messages (#3941) --- CHANGELOG.md | 1 + src/imap.rs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 942ac2a86..b754ae5ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Changes +- Only send IncomingMsgBunch if there are more than 0 new messages #3941 ### API-Changes diff --git a/src/imap.rs b/src/imap.rs index 94d4223d2..8bf71b4c3 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -828,11 +828,13 @@ impl Imap { info!(context, "{} mails read from \"{}\".", read_cnt, folder); - let msg_ids = received_msgs + let msg_ids: Vec = received_msgs .iter() .flat_map(|m| m.msg_ids.clone()) .collect(); - context.emit_event(EventType::IncomingMsgBunch { msg_ids }); + if !msg_ids.is_empty() { + context.emit_event(EventType::IncomingMsgBunch { msg_ids }); + } chat::mark_old_messages_as_noticed(context, received_msgs).await?; From 5ae6c2539413212608dec8880bd734778e1740e3 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 10 Jan 2023 12:41:17 +0000 Subject: [PATCH 32/77] Remove aheader module dependency on mailparse --- src/aheader.rs | 31 +++---------------------------- src/decrypt.rs | 24 ++++++++++++++++++++---- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/src/aheader.rs b/src/aheader.rs index 9899c1978..ac9a54f60 100644 --- a/src/aheader.rs +++ b/src/aheader.rs @@ -4,11 +4,9 @@ use anyhow::{bail, Context as _, Error, Result}; use std::collections::BTreeMap; +use std::fmt; use std::str::FromStr; -use std::{fmt, str}; -use crate::contact::addr_cmp; -use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{DcKey, SignedPublicKey}; /// Possible values for encryption preference @@ -36,7 +34,7 @@ impl fmt::Display for EncryptPreference { } } -impl str::FromStr for EncryptPreference { +impl FromStr for EncryptPreference { type Err = Error; fn from_str(s: &str) -> Result { @@ -69,29 +67,6 @@ impl Aheader { prefer_encrypt, } } - - /// Tries to parse Autocrypt header. - /// - /// If there is none, returns None. If the header is present but cannot be parsed, returns an - /// error. - pub fn from_headers( - wanted_from: &str, - headers: &[mailparse::MailHeader<'_>], - ) -> Result> { - if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) { - let header = Self::from_str(&value)?; - if !addr_cmp(&header.addr, wanted_from) { - bail!( - "Autocrypt header address {:?} is not {:?}", - header.addr, - wanted_from - ); - } - Ok(Some(header)) - } else { - Ok(None) - } - } } impl fmt::Display for Aheader { @@ -118,7 +93,7 @@ impl fmt::Display for Aheader { } } -impl str::FromStr for Aheader { +impl FromStr for Aheader { type Err = Error; fn from_str(s: &str) -> Result { diff --git a/src/decrypt.rs b/src/decrypt.rs index 68bcba737..b7f831ab0 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -1,6 +1,7 @@ //! End-to-end decryption support. use std::collections::HashSet; +use std::str::FromStr; use anyhow::Result; use mailparse::ParsedMail; @@ -13,7 +14,6 @@ use crate::context::Context; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey}; use crate::keyring::Keyring; -use crate::log::LogExt; use crate::peerstate::Peerstate; use crate::pgp; @@ -72,9 +72,25 @@ pub(crate) async fn prepare_decryption( }); } - let autocrypt_header = Aheader::from_headers(from, &mail.headers) - .ok_or_log_msg(context, "Failed to parse Autocrypt header") - .flatten(); + let autocrypt_header = + if let Some(autocrypt_header_value) = mail.headers.get_header_value(HeaderDef::Autocrypt) { + match Aheader::from_str(&autocrypt_header_value) { + Ok(header) if addr_cmp(&header.addr, from) => Some(header), + Ok(header) => { + warn!( + context, + "Autocrypt header address {:?} is not {:?}.", header.addr, from + ); + None + } + Err(err) => { + warn!(context, "Failed to parse Autocrypt header: {:#}.", err); + None + } + } + } else { + None + }; let dkim_results = handle_authres(context, mail, from, message_time).await?; From e215b4d919ced5631c16067061bff942bb6f35e8 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 9 Jan 2023 22:28:06 +0000 Subject: [PATCH 33/77] Return Option from Contact::add_or_lookup() This allows to distinguish exceptions, such as database errors, from invalid user input. For example, if the From: field of the message does not look like an email address, the mail should be ignored. But if there is a database failure while writing a new contact for the address, this error should be bubbled up. --- CHANGELOG.md | 2 ++ python/tests/test_1_online.py | 14 +++++++++ src/chat.rs | 12 +++++-- src/contact.rs | 56 ++++++++++++++++++++++++--------- src/imap.rs | 36 +++++++++++++++------ src/mimefactory.rs | 1 + src/peerstate.rs | 13 +++++--- src/qr.rs | 22 +++++++++---- src/reaction.rs | 1 + src/receive_imf.rs | 59 ++++++++++++++++++++++++++--------- src/receive_imf/tests.rs | 7 ++++- src/securejoin.rs | 3 +- src/test_utils.rs | 6 +++- 13 files changed, 176 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b754ae5ed..6383f3f7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ### Fixes - fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938 - Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943 +- Do not treat invalid email addresses as an exception #3942 + ## 1.105.0 diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 07da89134..954812d3b 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -939,6 +939,20 @@ def test_dont_show_emails(acfactory, lp): ac1.get_config("configured_addr") ), ) + ac1.direct_imap.append( + "Spam", + """ + From: delta + Subject: subj + To: {} + Message-ID: + Content-Type: text/plain; charset=utf-8 + + Unknown & malformed message in Spam + """.format( + ac1.get_config("configured_addr") + ), + ) ac1.direct_imap.append( "Spam", """ diff --git a/src/chat.rs b/src/chat.rs index 618319cb1..8a2fb2a42 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -4693,7 +4693,9 @@ mod tests { assert!(!shall_attach_selfavatar(&t, chat_id).await?); let (contact_id, _) = - Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::IncomingUnknownTo).await?; + Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::IncomingUnknownTo) + .await? + .unwrap(); add_contact_to_chat(&t, chat_id, contact_id).await?; assert!(!shall_attach_selfavatar(&t, chat_id).await?); t.set_config(Config::Selfavatar, None).await?; // setting to None also forces re-sending @@ -4940,7 +4942,9 @@ mod tests { bob.set_config(Config::ShowEmails, Some("2")).await?; let (contact_id, _) = - Contact::add_or_lookup(&alice, "", "bob@example.net", Origin::ManuallyCreated).await?; + Contact::add_or_lookup(&alice, "", "bob@example.net", Origin::ManuallyCreated) + .await? + .unwrap(); let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?; let alice_chat = Chat::load_from_db(&alice, alice_chat_id).await?; @@ -5651,7 +5655,9 @@ mod tests { async fn test_create_for_contact_with_blocked() -> Result<()> { let t = TestContext::new().await; let (contact_id, _) = - Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::ManuallyCreated).await?; + Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::ManuallyCreated) + .await? + .unwrap(); // create a blocked chat let chat_id_orig = diff --git a/src/contact.rs b/src/contact.rs index 4f2189467..b0e33a52c 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -376,12 +376,14 @@ impl Contact { /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event. pub async fn create(context: &Context, name: &str, addr: &str) -> Result { let name = improve_single_line_input(name); - ensure!(!addr.is_empty(), "Cannot create contact with empty address"); let (name, addr) = sanitize_name_and_addr(&name, addr); let (contact_id, sth_modified) = - Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated).await?; + Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated) + .await + .context("add_or_lookup")? + .with_context(|| format!("can't create a contact with address {:?}", addr))?; let blocked = Contact::is_blocked_load(context, contact_id).await?; match sth_modified { Modifier::None => {} @@ -466,12 +468,14 @@ impl Contact { /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name". /// /// Returns the contact_id and a `Modifier` value indicating if a modification occurred. + /// + /// Returns None if the contact with such address cannot exist. pub(crate) async fn add_or_lookup( context: &Context, name: &str, addr: &str, mut origin: Origin, - ) -> Result<(ContactId, Modifier)> { + ) -> Result> { let mut sth_modified = Modifier::None; ensure!(!addr.is_empty(), "Can not add_or_lookup empty address"); @@ -480,7 +484,7 @@ impl Contact { let addr = addr_normalize(addr).to_string(); if context.is_self_addr(&addr).await? { - return Ok((ContactId::SELF, sth_modified)); + return Ok(Some((ContactId::SELF, sth_modified))); } if !may_be_valid_addr(&addr) { @@ -490,7 +494,7 @@ impl Contact { addr, if !name.is_empty() { name } else { "" }, ); - bail!("Bad address supplied: {:?}", addr); + return Ok(None); } let mut name = name; @@ -653,7 +657,7 @@ impl Contact { } } - Ok((ContactId::new(row_id), sth_modified)) + Ok(Some((ContactId::new(row_id), sth_modified))) } /// Add a number of contacts. @@ -686,7 +690,10 @@ impl Contact { "Failed to add address {} from address book: {}", addr, err ); } - Ok((_, modified)) => { + Ok(None) => { + warn!(context, "Cannot create contact with address {}.", addr); + } + Ok(Some((_, modified))) => { if modified != Modifier::None { modify_cnt += 1 } @@ -1697,7 +1704,8 @@ mod tests { "user@example.org", Origin::IncomingReplyTo, ) - .await?; + .await? + .unwrap(); assert_ne!(id, ContactId::UNDEFINED); let contact = Contact::load_from_db(&context.ctx, id).await.unwrap(); @@ -1725,7 +1733,8 @@ mod tests { "user@example.org", Origin::ManuallyCreated, ) - .await?; + .await? + .unwrap(); assert_eq!(contact_bob_id, id); assert_eq!(modified, Modifier::Modified); let contact = Contact::load_from_db(&context.ctx, id).await.unwrap(); @@ -1775,6 +1784,7 @@ mod tests { let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo) .await + .unwrap() .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); @@ -1790,6 +1800,7 @@ mod tests { let (contact_id_test, sth_modified) = Contact::add_or_lookup(&t, "Real one", " one@eins.org ", Origin::ManuallyCreated) .await + .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); @@ -1802,6 +1813,7 @@ mod tests { let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo) .await + .unwrap() .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::None); @@ -1819,6 +1831,7 @@ mod tests { Origin::IncomingUnknownFrom, ) .await + .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); @@ -1830,6 +1843,7 @@ mod tests { let (contact_id_test, sth_modified) = Contact::add_or_lookup(&t, "schnucki", "three@drei.sam", Origin::ManuallyCreated) .await + .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); @@ -1842,6 +1856,7 @@ mod tests { let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo) .await + .unwrap() .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::None); @@ -1979,7 +1994,8 @@ mod tests { // Create Bob contact let (contact_id, _) = Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await?; + .await? + .unwrap(); let chat = alice .create_chat_with_contact("Bob", "bob@example.net") .await; @@ -2055,6 +2071,7 @@ mod tests { let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom) .await + .unwrap() .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Created); @@ -2067,6 +2084,7 @@ mod tests { let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom) .await + .unwrap() .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); @@ -2089,6 +2107,7 @@ mod tests { let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom) .await + .unwrap() .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); @@ -2118,6 +2137,7 @@ mod tests { Origin::IncomingUnknownFrom, ) .await + .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); @@ -2134,6 +2154,7 @@ mod tests { Origin::IncomingUnknownFrom, ) .await + .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); @@ -2154,7 +2175,8 @@ mod tests { // Incoming message from Bob. let (contact_id, sth_modified) = Contact::add_or_lookup(&t, "Bob", "bob@example.org", Origin::IncomingUnknownFrom) - .await?; + .await? + .unwrap(); assert_eq!(sth_modified, Modifier::Created); let contact = Contact::load_from_db(&t, contact_id).await?; assert_eq!(contact.get_display_name(), "Bob"); @@ -2162,7 +2184,8 @@ mod tests { // Incoming message from someone else with "Not Bob" in the "To:" field. let (contact_id_same, sth_modified) = Contact::add_or_lookup(&t, "Not Bob", "bob@example.org", Origin::IncomingUnknownTo) - .await?; + .await? + .unwrap(); assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await?; @@ -2171,7 +2194,8 @@ mod tests { // Incoming message from Bob, changing the name back. let (contact_id_same, sth_modified) = Contact::add_or_lookup(&t, "Bob", "bob@example.org", Origin::IncomingUnknownFrom) - .await?; + .await? + .unwrap(); assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); // This was None until the bugfix let contact = Contact::load_from_db(&t, contact_id).await?; @@ -2312,7 +2336,8 @@ mod tests { let (contact_bob_id, _modified) = Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await?; + .await? + .unwrap(); let encrinfo = Contact::get_encrinfo(&alice, contact_bob_id).await?; assert_eq!(encrinfo, "No encryption"); @@ -2471,7 +2496,8 @@ CCCB 5AA9 F6E1 141C 9431 let (contact_id, _) = Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await?; + .await? + .unwrap(); let contact = Contact::load_from_db(&alice, contact_id).await?; assert_eq!(contact.last_seen(), 0); diff --git a/src/imap.rs b/src/imap.rs index 8bf71b4c3..c2bcd1f7a 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -1737,7 +1737,19 @@ async fn should_move_out_of_spam( }; // No chat found. let (from_id, blocked_contact, _origin) = - from_field_to_contact_id(context, &from, true).await?; + match from_field_to_contact_id(context, &from, true) + .await + .context("from_field_to_contact_id")? + { + Some(res) => res, + None => { + warn!( + context, + "Contact with From address {:?} cannot exist, not moving out of spam", from + ); + return Ok(false); + } + }; if blocked_contact { // Contact is blocked, leave the message in spam. return Ok(false); @@ -2027,7 +2039,10 @@ pub(crate) async fn prefetch_should_download( None => return Ok(false), }; let (_from_id, blocked_contact, origin) = - from_field_to_contact_id(context, &from, true).await?; + match from_field_to_contact_id(context, &from, true).await? { + Some(res) => res, + None => return Ok(false), + }; // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact. // (prevent_rename is the last argument of from_field_to_contact_id()) @@ -2347,14 +2362,14 @@ async fn add_all_recipients_as_contacts( .await .with_context(|| format!("could not select {}", mailbox))?; - let contacts = imap + let recipients = imap .get_all_recipients(context) .await .context("could not get recipients")?; let mut any_modified = false; - for contact in contacts { - let display_name_normalized = contact + for recipient in recipients { + let display_name_normalized = recipient .display_name .as_ref() .map(|s| normalize_name(s)) @@ -2363,17 +2378,20 @@ async fn add_all_recipients_as_contacts( match Contact::add_or_lookup( context, &display_name_normalized, - &contact.addr, + &recipient.addr, Origin::OutgoingTo, ) - .await + .await? { - Ok((_, modified)) => { + Some((_, modified)) => { if modified != Modifier::None { any_modified = true; } } - Err(e) => warn!(context, "Could not add recipient: {}", e), + None => warn!( + context, + "Could not add contact for recipient with address {:?}", recipient.addr + ), } } if any_modified { diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 5d7d51fe8..bde304869 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1821,6 +1821,7 @@ mod tests { Contact::add_or_lookup(&t, "Dave", "dave@example.com", Origin::ManuallyCreated) .await .unwrap() + .unwrap() .0; let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap(); diff --git a/src/peerstate.rs b/src/peerstate.rs index 2699bf6eb..6224118d9 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -542,14 +542,17 @@ impl Peerstate { if (chat.typ == Chattype::Group && chat.is_protected()) || chat.typ == Chattype::Broadcast { - chat::remove_from_chat_contacts_table(context, *chat_id, contact_id).await?; - - let (new_contact_id, _) = + if let Some((new_contact_id, _)) = Contact::add_or_lookup(context, "", new_addr, Origin::IncomingUnknownFrom) + .await? + { + chat::remove_from_chat_contacts_table(context, *chat_id, contact_id) + .await?; + chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]) .await?; - chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]).await?; - context.emit_event(EventType::ChatModified(*chat_id)); + context.emit_event(EventType::ChatModified(*chat_id)); + } } } diff --git a/src/qr.rs b/src/qr.rs index 2813c762d..c1e250211 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -221,10 +221,15 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { .context("Can't load peerstate")?; if let (Some(addr), Some(invitenumber), Some(authcode)) = (&addr, invitenumber, authcode) { - let contact_id = Contact::add_or_lookup(context, &name, addr, Origin::UnhandledQrScan) + let (contact_id, _) = Contact::add_or_lookup(context, &name, addr, Origin::UnhandledQrScan) .await - .map(|(id, _)| id) - .with_context(|| format!("failed to add or lookup contact for address {:?}", addr))?; + .with_context(|| format!("failed to add or lookup contact for address {:?}", addr))? + .with_context(|| { + format!( + "do not want to lookup contact for invalid address {:?}", + addr + ) + })?; if let (Some(grpid), Some(grpname)) = (grpid, grpname) { if context @@ -287,10 +292,13 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { } } else if let Some(addr) = addr { if let Some(peerstate) = peerstate { - let contact_id = + let (contact_id, _) = Contact::add_or_lookup(context, &name, &peerstate.addr, Origin::UnhandledQrScan) .await - .map(|(id, _)| id)?; + .context("add_or_lookup")? + .with_context(|| { + format!("Not looking up contact for address {:?}", &peerstate.addr) + })?; let chat = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request) .await .context("Failed to create (new) chat for contact")?; @@ -620,7 +628,9 @@ impl Qr { draft: Option, ) -> Result { let (contact_id, _) = - Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan).await?; + Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan) + .await? + .context("QR code does not contain a valid address")?; Ok(Qr::Addr { contact_id, draft }) } } diff --git a/src/reaction.rs b/src/reaction.rs index 72a096030..6777c5375 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -368,6 +368,7 @@ Can we chat at 1pm pacific, today?" let bob_id = Contact::add_or_lookup(&alice, "", "bob@example.net", Origin::ManuallyCreated) .await? + .unwrap() .0; let bob_reaction = reactions.get(bob_id); assert!(bob_reaction.is_empty()); // Bob has not reacted to message yet. diff --git a/src/receive_imf.rs b/src/receive_imf.rs index e8ad015af..c22cfb348 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -170,7 +170,16 @@ pub(crate) async fn receive_imf_inner( // If this is a mailing list email (i.e. list_id_header is some), don't change the displayname because in // a mailing list the sender displayname sometimes does not belong to the sender email address. let (from_id, _from_id_blocked, incoming_origin) = - from_field_to_contact_id(context, &mime_parser.from, prevent_rename).await?; + match from_field_to_contact_id(context, &mime_parser.from, prevent_rename).await? { + Some(contact_id_res) => contact_id_res, + None => { + warn!( + context, + "receive_imf: From field does not contain an acceptable address" + ); + return Ok(None); + } + }; let incoming = from_id != ContactId::SELF; @@ -373,22 +382,28 @@ pub async fn from_field_to_contact_id( context: &Context, from: &SingleInfo, prevent_rename: bool, -) -> Result<(ContactId, bool, Origin)> { +) -> Result> { let display_name = if prevent_rename { Some("") } else { from.display_name.as_deref() }; - let from_id = add_or_lookup_contact_by_addr( + let from_id = if let Some(from_id) = add_or_lookup_contact_by_addr( context, display_name, &from.addr, Origin::IncomingUnknownFrom, ) - .await?; + .await + .context("add_or_lookup_contact_by_addr")? + { + from_id + } else { + return Ok(None); + }; if from_id == ContactId::SELF { - Ok((ContactId::SELF, false, Origin::OutgoingBcc)) + Ok(Some((ContactId::SELF, false, Origin::OutgoingBcc))) } else { let mut from_id_blocked = false; let mut incoming_origin = Origin::Unknown; @@ -396,7 +411,7 @@ pub async fn from_field_to_contact_id( from_id_blocked = contact.blocked; incoming_origin = contact.origin; } - Ok((from_id, from_id_blocked, incoming_origin)) + Ok(Some((from_id, from_id_blocked, incoming_origin))) } } @@ -1928,8 +1943,11 @@ async fn apply_mailinglist_changes( } let listid = &chat.grpid; - let (contact_id, _) = - Contact::add_or_lookup(context, "", list_post, Origin::Hidden).await?; + let contact_id = + match Contact::add_or_lookup(context, "", list_post, Origin::Hidden).await? { + Some((contact_id, _)) => contact_id, + None => return Ok(()), + }; let mut contact = Contact::load_from_db(context, contact_id).await?; if contact.param.get(Param::ListId) != Some(listid) { contact.param.set(Param::ListId, listid); @@ -2268,28 +2286,39 @@ async fn add_or_lookup_contacts_by_address_list( continue; } let display_name = info.display_name.as_deref(); - contact_ids - .insert(add_or_lookup_contact_by_addr(context, display_name, addr, origin).await?); + if let Some(contact_id) = + add_or_lookup_contact_by_addr(context, display_name, addr, origin).await? + { + contact_ids.insert(contact_id); + } else { + warn!(context, "Contact with address {:?} cannot exist.", addr); + } } Ok(contact_ids.into_iter().collect::>()) } /// Add contacts to database on receiving messages. +/// +/// Returns `None` if the address can't be a valid email address. async fn add_or_lookup_contact_by_addr( context: &Context, display_name: Option<&str>, addr: &str, origin: Origin, -) -> Result { +) -> Result> { if context.is_self_addr(addr).await? { - return Ok(ContactId::SELF); + return Ok(Some(ContactId::SELF)); } let display_name_normalized = display_name.map(normalize_name).unwrap_or_default(); - let (row_id, _modified) = - Contact::add_or_lookup(context, &display_name_normalized, addr, origin).await?; - Ok(row_id) + if let Some((contact_id, _modified)) = + Contact::add_or_lookup(context, &display_name_normalized, addr, origin).await? + { + Ok(Some(contact_id)) + } else { + Ok(None) + } } #[cfg(test)] diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index a92c8aead..12a3901b0 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -429,6 +429,7 @@ async fn test_escaped_recipients() { Contact::add_or_lookup(&t, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom) .await .unwrap() + .unwrap() .0; receive_imf( @@ -471,6 +472,7 @@ async fn test_cc_to_contact() { Contact::add_or_lookup(&t, "garabage", "carl@host.tld", Origin::IncomingUnknownFrom) .await .unwrap() + .unwrap() .0; receive_imf( @@ -2058,6 +2060,7 @@ async fn test_duplicate_message() -> Result<()> { Origin::IncomingUnknownFrom, ) .await? + .unwrap() .0; let first_message = b"Received: from [127.0.0.1] @@ -2111,6 +2114,7 @@ async fn test_ignore_footer_status_from_mailinglist() -> Result<()> { t.set_config(Config::ShowEmails, Some("2")).await?; let bob_id = Contact::add_or_lookup(&t, "", "bob@example.net", Origin::IncomingUnknownCc) .await? + .unwrap() .0; let bob = Contact::load_from_db(&t, bob_id).await?; assert_eq!(bob.get_status(), ""); @@ -2529,7 +2533,8 @@ Second thread."#; "fiona@example.net", Origin::IncomingUnknownTo, ) - .await?; + .await? + .unwrap(); chat::add_contact_to_chat(&alice, alice_first_msg.chat_id, alice_fiona_contact_id).await?; let alice_first_invite = alice.pop_sent_msg().await; diff --git a/src/securejoin.rs b/src/securejoin.rs index 77d743c17..f335f2e27 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -1006,7 +1006,8 @@ mod tests { "bob@example.net", Origin::ManuallyCreated, ) - .await?; + .await? + .unwrap(); let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id).await?; assert_eq!( contact_bob.is_verified(&alice.ctx).await?, diff --git a/src/test_utils.rs b/src/test_utils.rs index cf19dafd8..4ae732591 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -529,7 +529,11 @@ impl TestContext { let (contact_id, modified) = Contact::add_or_lookup(self, &name, &addr, Origin::MailinglistAddress) .await - .unwrap(); + .expect("add_or_lookup") + .expect(&format!( + "contact with address {:?} cannot be created", + &addr + )); match modified { Modifier::None => (), Modifier::Modified => warn!(&self.ctx, "Contact {} modified by TestContext", &addr), From 1cde09c312f4dc2e72ee492548f4edab4a1a1752 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 02:37:01 +0000 Subject: [PATCH 34/77] Add missing documentation to the `message` module --- src/message.rs | 112 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 8 deletions(-) diff --git a/src/message.rs b/src/message.rs index b1581137f..63bf1450f 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1,7 +1,5 @@ //! # Messages and their identifiers. -#![allow(missing_docs)] - use std::collections::BTreeSet; use std::path::{Path, PathBuf}; @@ -237,11 +235,18 @@ impl Default for MessengerMessage { /// If you want an update, you have to recreate the object. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct Message { + /// Message ID. pub(crate) id: MsgId, + + /// `From:` contact ID. pub(crate) from_id: ContactId, + + /// ID of the first contact in the `To:` header. pub(crate) to_id: ContactId, pub(crate) chat_id: ChatId, pub(crate) viewtype: Viewtype, + + /// State of the message. pub(crate) state: MessageState, pub(crate) download_state: DownloadState, pub(crate) hidden: bool, @@ -263,6 +268,7 @@ pub struct Message { } impl Message { + /// Creates a new message with given view type. pub fn new(viewtype: Viewtype) -> Self { Message { viewtype, @@ -270,6 +276,7 @@ impl Message { } } + /// Loads message with given ID from the database. pub async fn load_from_db(context: &Context, id: MsgId) -> Result { ensure!( !id.is_special(), @@ -366,6 +373,12 @@ impl Message { Ok(msg) } + /// Returns the MIME type of an attached file if it exists. + /// + /// If the MIME type is not known, the function guesses the MIME type + /// from the extension. `application/octet-stream` is used as a fallback + /// if MIME type is not known, but `None` is only returned if no file + /// is attached. pub fn get_filemime(&self) -> Option { if let Some(m) = self.param.get(Param::MimeType) { return Some(m.to_string()); @@ -380,11 +393,12 @@ impl Message { None } + /// Returns the full path to the file associated with a message. pub fn get_file(&self, context: &Context) -> Option { self.param.get_path(Param::File, context).unwrap_or(None) } - pub async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> { + pub(crate) async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> { if self.viewtype.has_file() { let file_param = self.param.get_path(Param::File, context)?; if let Some(path_and_filename) = file_param { @@ -442,6 +456,8 @@ impl Message { self.param.set_float(Param::SetLongitude, longitude); } + /// Returns the message timestamp for display in the UI + /// as a unix timestamp in seconds. pub fn get_timestamp(&self) -> i64 { if 0 != self.timestamp_sent { self.timestamp_sent @@ -450,10 +466,12 @@ impl Message { } } + /// Returns the message ID. pub fn get_id(&self) -> MsgId { self.id } + /// Returns the ID of the contact who wrote the message. pub fn get_from_id(&self) -> ContactId { self.from_id } @@ -463,30 +481,40 @@ impl Message { self.chat_id } + /// Returns the type of the message. pub fn get_viewtype(&self) -> Viewtype { self.viewtype } + /// Returns the state of the message. pub fn get_state(&self) -> MessageState { self.state } + /// Returns the message receive time as a unix timestamp in seconds. pub fn get_received_timestamp(&self) -> i64 { self.timestamp_rcvd } + /// Returns the timestamp of the message for sorting. pub fn get_sort_timestamp(&self) -> i64 { self.timestamp_sort } + /// Returns the text of the message. pub fn get_text(&self) -> Option { self.text.as_ref().map(|s| s.to_string()) } + /// Returns message subject. pub fn get_subject(&self) -> &str { &self.subject } + /// Returns base file name without the path. + /// The base file name includes the extension. + /// + /// To get the full path, use [`Self::get_file()`]. pub fn get_filename(&self) -> Option { self.param .get(Param::File) @@ -503,18 +531,22 @@ impl Message { } } + /// Returns width of associated image or video file. pub fn get_width(&self) -> i32 { self.param.get_int(Param::Width).unwrap_or_default() } + /// Returns height of associated image or video file. pub fn get_height(&self) -> i32 { self.param.get_int(Param::Height).unwrap_or_default() } + /// Returns duration of associated audio or video file. pub fn get_duration(&self) -> i32 { self.param.get_int(Param::Duration).unwrap_or_default() } + /// Returns true if padlock indicating message encryption should be displayed in the UI. pub fn get_showpadlock(&self) -> bool { self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0 } @@ -524,10 +556,12 @@ impl Message { self.param.get_bool(Param::Bot).unwrap_or_default() } + /// Return the ephemeral timer duration for a message. pub fn get_ephemeral_timer(&self) -> EphemeralTimer { self.ephemeral_timer } + /// Returns the timestamp of the epehemeral message removal. pub fn get_ephemeral_timestamp(&self) -> i64 { self.ephemeral_timestamp } @@ -565,6 +599,7 @@ impl Message { // C-data in the Java code (i.e. a `long` storing a C pointer) // - We can't make a param `SenderDisplayname` for messages as sometimes the display name of a contact changes, and we want to show // the same display name over all messages from the same sender. + /// Returns the name that should be shown over the message instead of the contact display ame. pub fn get_override_sender_name(&self) -> Option { self.param .get(Param::OverrideSenderDisplayname) @@ -573,11 +608,15 @@ impl Message { // Exposing this function over the ffi instead of get_override_sender_name() would mean that at least Android Java code has // to handle raw C-data (as it is done for msg_get_summary()) - pub fn get_sender_name(&self, contact: &Contact) -> String { + pub(crate) fn get_sender_name(&self, contact: &Contact) -> String { self.get_override_sender_name() .unwrap_or_else(|| contact.get_display_name().to_string()) } + /// Returns true if a message has a deviating timestamp. + /// + /// A message has a deviating timestamp when it is sent on + /// another day as received/sorted by. pub fn has_deviating_timestamp(&self) -> bool { let cnv_to_local = gm2local_offset(); let sort_timestamp = self.get_sort_timestamp() + cnv_to_local; @@ -586,14 +625,18 @@ impl Message { sort_timestamp / 86400 != send_timestamp / 86400 } + /// Returns true if the message was successfully delivered to the outgoing server or even + /// received a read receipt. pub fn is_sent(&self) -> bool { self.state >= MessageState::OutDelivered } + /// Returns true if the message is a forwarded message. pub fn is_forwarded(&self) -> bool { 0 != self.param.get_int(Param::Forwarded).unwrap_or_default() } + /// Returns true if the message is an informational message. pub fn is_info(&self) -> bool { let cmd = self.param.get_cmd(); self.from_id == ContactId::INFO @@ -601,10 +644,12 @@ impl Message { || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage } + /// Returns the type of an informational message. pub fn get_info_type(&self) -> SystemMessage { self.param.get_cmd() } + /// Returns true if the message is a system message. pub fn is_system_message(&self) -> bool { let cmd = self.param.get_cmd(); cmd != SystemMessage::Unknown @@ -622,6 +667,7 @@ impl Message { self.viewtype.has_file() && self.state == MessageState::OutPreparing } + /// Returns true if the message is an Autocrypt Setup Message. pub fn is_setupmessage(&self) -> bool { if self.viewtype != Viewtype::File { return false; @@ -630,6 +676,9 @@ impl Message { self.param.get_cmd() == SystemMessage::AutocryptSetupMessage } + /// Returns the first characters of the setup code. + /// + /// This is used to pre-fill the first entry field of the setup code. pub async fn get_setupcodebegin(&self, context: &Context) -> Option { if !self.is_setupmessage() { return None; @@ -650,7 +699,7 @@ impl Message { // add room to a webrtc_instance as defined by the corresponding config-value; // the result may still be prefixed by the type - pub fn create_webrtc_instance(instance: &str, room: &str) -> String { + pub(crate) fn create_webrtc_instance(instance: &str, room: &str) -> String { let (videochat_type, mut url) = Message::parse_webrtc_instance(instance); // make sure, there is a scheme in the url @@ -707,6 +756,7 @@ impl Message { } } + /// Returns videochat URL if the message is a videochat invitation. pub fn get_videochat_url(&self) -> Option { if self.viewtype == Viewtype::VideochatInvitation { if let Some(instance) = self.param.get(Param::WebrtcRoom) { @@ -716,6 +766,7 @@ impl Message { None } + /// Returns videochat type if the message is a videochat invitation. pub fn get_videochat_type(&self) -> Option { if self.viewtype == Viewtype::VideochatInvitation { if let Some(instance) = self.param.get(Param::WebrtcRoom) { @@ -725,10 +776,16 @@ impl Message { None } + /// Sets or unsets message text. pub fn set_text(&mut self, text: Option) { self.text = text; } + /// Sets the file associated with a message. + /// + /// This function does not use the file or check if it exists, + /// the file will only be used when the message is prepared + /// for sending. pub fn set_file(&mut self, file: impl ToString, filemime: Option<&str>) { self.param.set(Param::File, file); if let Some(filemime) = filemime { @@ -746,11 +803,13 @@ impl Message { } } + /// Sets the dimensions of associated image or video file. pub fn set_dimension(&mut self, width: i32, height: i32) { self.param.set_int(Param::Width, width); self.param.set_int(Param::Height, height); } + /// Sets the duration of associated audio or video file. pub fn set_duration(&mut self, duration: i32) { self.param.set_int(Param::Duration, duration); } @@ -760,6 +819,8 @@ impl Message { self.param.set_int(Param::Reaction, 1); } + /// Changes the message width, height or duration, + /// and stores it into the database. pub async fn latefiling_mediasize( &mut self, context: &Context, @@ -824,10 +885,12 @@ impl Message { Ok(()) } + /// Returns quoted message text, if any. pub fn quoted_text(&self) -> Option { self.param.get(Param::Quote).map(|s| s.to_string()) } + /// Returns quoted message, if any. pub async fn quoted_message(&self, context: &Context) -> Result> { if self.param.get(Param::Quote).is_some() && !self.is_forwarded() { return self.parent(context).await; @@ -835,6 +898,10 @@ impl Message { Ok(None) } + /// Returns parent message according to the `In-Reply-To` header + /// if it exists in the database and is not trashed. + /// + /// `References` header is not taken into account. pub async fn parent(&self, context: &Context) -> Result> { if let Some(in_reply_to) = &self.in_reply_to { if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? { @@ -855,6 +922,7 @@ impl Message { self.param.set_int(Param::ForcePlaintext, 1); } + /// Updates `param` column of the message in the database without changing other columns. pub async fn update_param(&self, context: &Context) -> Result<()> { context .sql @@ -894,6 +962,9 @@ impl Message { } } +/// State of the message. +/// For incoming messages, stores the information on whether the message was read or not. +/// For outgoing message, the message could be pending, already delivered or confirmed. #[derive( Debug, Clone, @@ -911,6 +982,7 @@ impl Message { )] #[repr(u32)] pub enum MessageState { + /// Undefined message state. Undefined = 0, /// Incoming *fresh* message. Fresh messages are neither noticed @@ -981,6 +1053,7 @@ impl std::fmt::Display for MessageState { } impl MessageState { + /// Returns true if the message can transition to `OutFailed` state from the current state. pub fn can_fail(self) -> bool { use MessageState::*; matches!( @@ -988,6 +1061,8 @@ impl MessageState { OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed. ) } + + /// Returns true for any outgoing message states. pub fn is_outgoing(self) -> bool { use MessageState::*; matches!( @@ -997,6 +1072,7 @@ impl MessageState { } } +/// Returns detailed message information in a multi-line text form. pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { let msg = Message::load_from_db(context, msg_id).await?; let rawtxt: Option = context @@ -1161,7 +1237,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { Ok(ret) } -pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> { +pub(crate) fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> { let extension: &str = &path.extension()?.to_str()?.to_lowercase(); let info = match extension { // before using viewtype other than Viewtype::File, @@ -1274,6 +1350,9 @@ pub async fn get_mime_headers(context: &Context, msg_id: MsgId) -> Result Result<()> { for msg_id in msg_ids.iter() { let msg = Message::load_from_db(context, *msg_id).await?; @@ -1321,6 +1400,7 @@ async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> Ok(()) } +/// Marks requested messages as seen. pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> { if msg_ids.is_empty() { return Ok(()); @@ -1453,7 +1533,8 @@ pub(crate) async fn update_msg_state( // Context functions to work with messages -pub async fn exists(context: &Context, msg_id: MsgId) -> Result { +/// Returns true if given message ID exists in the database and is not trashed. +pub(crate) async fn exists(context: &Context, msg_id: MsgId) -> Result { if msg_id.is_special() { return Ok(false); } @@ -1470,7 +1551,7 @@ pub async fn exists(context: &Context, msg_id: MsgId) -> Result { } } -pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: &str) { +pub(crate) async fn set_msg_failed(context: &Context, msg_id: MsgId, error: &str) { if let Ok(mut msg) = Message::load_from_db(context, msg_id).await { if msg.state.can_fail() { msg.state = MessageState::OutFailed; @@ -1716,6 +1797,20 @@ pub async fn get_request_msg_cnt(context: &Context) -> usize { } } +/// Estimates the number of messages that will be deleted +/// by the options `delete_device_after` or `delete_server_after`. +/// This is typically used to show the estimated impact to the user +/// before actually enabling deletion of old messages. +/// +/// If `from_server` is true, +/// estimate deletion count for server, +/// otherwise estimate deletion count for device. +/// +/// Count messages older than the given number of `seconds`. +/// +/// Returns the number of messages that are older than the given number of seconds. +/// This includes e-mails downloaded due to the `show_emails` option. +/// Messages in the "saved messages" folder are not counted as they will not be deleted automatically. pub async fn estimate_deletion_cnt( context: &Context, from_server: bool, @@ -1804,6 +1899,7 @@ pub(crate) async fn rfc724_mid_exists( )] #[repr(u32)] pub enum Viewtype { + /// Unknown message type. Unknown = 0, /// Text message. From 554090b03e3b75f5af7a7df496204fed2fa5a2b1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 10 Jan 2023 20:57:14 +0000 Subject: [PATCH 35/77] Prepare 1.106.0 --- CHANGELOG.md | 4 +--- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- package.json | 2 +- 8 files changed, 11 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6383f3f7b..5ef009c26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,10 @@ # Changelog -## Unreleased +## 1.106.0 ### Changes - Only send IncomingMsgBunch if there are more than 0 new messages #3941 -### API-Changes - ### Fixes - fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938 - Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943 diff --git a/Cargo.lock b/Cargo.lock index abe7cd0d1..8bc55496c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -873,7 +873,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deltachat" -version = "1.105.0" +version = "1.106.0" dependencies = [ "ansi_term", "anyhow", @@ -947,7 +947,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.105.0" +version = "1.106.0" dependencies = [ "anyhow", "async-channel", @@ -969,7 +969,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.105.0" +version = "1.106.0" dependencies = [ "anyhow", "deltachat-jsonrpc", @@ -992,7 +992,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.105.0" +version = "1.106.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index 549d49dbe..2ba38945e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.105.0" +version = "1.106.0" edition = "2021" license = "MPL-2.0" rust-version = "1.63" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index c3c3dbdf1..3488ba82b 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.105.0" +version = "1.106.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 63677512c..763548287 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.105.0" +version = "1.106.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 9c1df45e3..480685078 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -48,5 +48,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.105.0" + "version": "1.106.0" } \ No newline at end of file diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index a45572b74..753b99c81 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.105.0" +version = "1.106.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 09b9e179d..25771812e 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.105.0" + "version": "1.106.0" } \ No newline at end of file From 6642083f52120ca1130753bef2ad17789a648f60 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 10 Jan 2023 21:17:30 +0000 Subject: [PATCH 36/77] Clippy fix --- src/test_utils.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index 4ae732591..d8542fe2f 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -530,10 +530,7 @@ impl TestContext { Contact::add_or_lookup(self, &name, &addr, Origin::MailinglistAddress) .await .expect("add_or_lookup") - .expect(&format!( - "contact with address {:?} cannot be created", - &addr - )); + .unwrap_or_else(|| panic!("contact with address {:?} cannot be created", &addr)); match modified { Modifier::None => (), Modifier::Modified => warn!(&self.ctx, "Contact {} modified by TestContext", &addr), From 6d9d31cad13cb0314a94b31c92b474e1d0c57a86 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Wed, 11 Jan 2023 11:55:42 -0300 Subject: [PATCH 37/77] Add timeouts to HTTP requests (#3908) --- CHANGELOG.md | 1 + src/configure/read_url.rs | 2 +- src/http.rs | 12 ++++++++++++ src/lib.rs | 1 + src/oauth2.rs | 11 +++++++++-- src/qr.rs | 2 +- 6 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 src/http.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ef009c26..f36e6cf43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938 - Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943 - Do not treat invalid email addresses as an exception #3942 +- Add timeouts to HTTP requests #3948 ## 1.105.0 diff --git a/src/configure/read_url.rs b/src/configure/read_url.rs index f275025db..b0cdd989f 100644 --- a/src/configure/read_url.rs +++ b/src/configure/read_url.rs @@ -16,7 +16,7 @@ pub async fn read_url(context: &Context, url: &str) -> anyhow::Result { } pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result { - let client = reqwest::Client::new(); + let client = crate::http::get_client()?; let mut url = url.to_string(); // Follow up to 10 http-redirects diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 000000000..4f0c3fb35 --- /dev/null +++ b/src/http.rs @@ -0,0 +1,12 @@ +//! # HTTP module. + +use anyhow::Result; +use std::time::Duration; + +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn get_client() -> Result { + Ok(reqwest::ClientBuilder::new() + .timeout(HTTP_TIMEOUT) + .build()?) +} diff --git a/src/lib.rs b/src/lib.rs index a6e8eb8ae..831f6609e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,6 +66,7 @@ mod decrypt; pub mod download; mod e2ee; pub mod ephemeral; +mod http; mod imap; pub mod imex; mod scheduler; diff --git a/src/oauth2.rs b/src/oauth2.rs index d48cb45a6..ab0b4eef5 100644 --- a/src/oauth2.rs +++ b/src/oauth2.rs @@ -158,7 +158,7 @@ pub async fn get_oauth2_access_token( } // ... and POST - let client = reqwest::Client::new(); + let client = crate::http::get_client()?; let response: Response = match client.post(post_url).form(&post_param).send().await { Ok(resp) => match resp.json().await { @@ -284,7 +284,14 @@ impl Oauth2 { // "verified_email": true, // "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg" // } - let response = match reqwest::get(userinfo_url).await { + let client = match crate::http::get_client() { + Ok(cl) => cl, + Err(err) => { + warn!(context, "failed to get HTTP client: {}", err); + return None; + } + }; + let response = match client.get(userinfo_url).send().await { Ok(response) => response, Err(err) => { warn!(context, "failed to get userinfo: {}", err); diff --git a/src/qr.rs b/src/qr.rs index c1e250211..7cd4fc28c 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -381,7 +381,7 @@ struct CreateAccountErrorResponse { #[allow(clippy::indexing_slicing)] async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> { let url_str = &qr[DCACCOUNT_SCHEME.len()..]; - let response = reqwest::Client::new().post(url_str).send().await?; + let response = crate::http::get_client()?.post(url_str).send().await?; let response_status = response.status(); let response_text = response.text().await.with_context(|| { format!( From 89c8d26968a3a937abd9bb72ae573ab75ac54a8b Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Jan 2023 04:19:28 +0000 Subject: [PATCH 38/77] Add ContactAddress type --- src/chat.rs | 30 ++-- src/contact.rs | 316 +++++++++++++++++++++++++-------------- src/imap.rs | 28 ++-- src/mimefactory.rs | 17 ++- src/peerstate.rs | 32 ++-- src/qr.rs | 48 +++--- src/reaction.rs | 16 +- src/receive_imf.rs | 69 +++++---- src/receive_imf/tests.rs | 55 +++---- src/securejoin.rs | 6 +- src/test_utils.rs | 10 +- 11 files changed, 373 insertions(+), 254 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 8a2fb2a42..cef50d3d1 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -3629,7 +3629,7 @@ mod tests { use crate::chatlist::{get_archived_cnt, Chatlist}; use crate::constants::{DC_GCL_ARCHIVED_ONLY, DC_GCL_NO_SPECIALS}; - use crate::contact::Contact; + use crate::contact::{Contact, ContactAddress}; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; @@ -4692,10 +4692,13 @@ mod tests { let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?; assert!(!shall_attach_selfavatar(&t, chat_id).await?); - let (contact_id, _) = - Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::IncomingUnknownTo) - .await? - .unwrap(); + let (contact_id, _) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("foo@bar.org")?, + Origin::IncomingUnknownTo, + ) + .await?; add_contact_to_chat(&t, chat_id, contact_id).await?; assert!(!shall_attach_selfavatar(&t, chat_id).await?); t.set_config(Config::Selfavatar, None).await?; // setting to None also forces re-sending @@ -4941,10 +4944,8 @@ mod tests { alice.set_config(Config::ShowEmails, Some("2")).await?; bob.set_config(Config::ShowEmails, Some("2")).await?; - let (contact_id, _) = - Contact::add_or_lookup(&alice, "", "bob@example.net", Origin::ManuallyCreated) - .await? - .unwrap(); + let alice_bob_contact = alice.add_or_lookup_contact(&bob).await; + let contact_id = alice_bob_contact.id; let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?; let alice_chat = Chat::load_from_db(&alice, alice_chat_id).await?; @@ -5654,10 +5655,13 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_create_for_contact_with_blocked() -> Result<()> { let t = TestContext::new().await; - let (contact_id, _) = - Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::ManuallyCreated) - .await? - .unwrap(); + let (contact_id, _) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("foo@bar.org")?, + Origin::ManuallyCreated, + ) + .await?; // create a blocked chat let chat_id_orig = diff --git a/src/contact.rs b/src/contact.rs index b0e33a52c..1f5233fbf 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -4,6 +4,7 @@ use std::cmp::Reverse; use std::collections::BinaryHeap; use std::convert::{TryFrom, TryInto}; use std::fmt; +use std::ops::Deref; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; @@ -36,6 +37,51 @@ use crate::{chat, stock_str}; /// Time during which a contact is considered as seen recently. const SEEN_RECENTLY_SECONDS: i64 = 600; +/// Valid contact address. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ContactAddress<'a>(&'a str); + +impl Deref for ContactAddress<'_> { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl AsRef for ContactAddress<'_> { + fn as_ref(&self) -> &str { + self.0 + } +} + +impl fmt::Display for ContactAddress<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl<'a> ContactAddress<'a> { + /// Constructs a new contact address from string, + /// normalizing and validating it. + pub fn new(s: &'a str) -> Result { + let addr = addr_normalize(s); + if !may_be_valid_addr(addr) { + bail!("invalid address {:?}", s); + } + Ok(Self(addr)) + } +} + +/// Allow converting [`ContactAddress`] to an SQLite type. +impl rusqlite::types::ToSql for ContactAddress<'_> { + fn to_sql(&self) -> rusqlite::Result { + let val = rusqlite::types::Value::Text(self.0.to_string()); + let out = rusqlite::types::ToSqlOutput::Owned(val); + Ok(out) + } +} + /// Contact ID, including reserved IDs. /// /// Some contact IDs are reserved to identify special contacts. This @@ -378,12 +424,12 @@ impl Contact { let name = improve_single_line_input(name); let (name, addr) = sanitize_name_and_addr(&name, addr); + let addr = ContactAddress::new(&addr)?; let (contact_id, sth_modified) = - Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated) + Contact::add_or_lookup(context, &name, addr, Origin::ManuallyCreated) .await - .context("add_or_lookup")? - .with_context(|| format!("can't create a contact with address {:?}", addr))?; + .context("add_or_lookup")?; let blocked = Contact::is_blocked_load(context, contact_id).await?; match sth_modified { Modifier::None => {} @@ -473,28 +519,16 @@ impl Contact { pub(crate) async fn add_or_lookup( context: &Context, name: &str, - addr: &str, + addr: ContactAddress<'_>, mut origin: Origin, - ) -> Result> { + ) -> Result<(ContactId, Modifier)> { let mut sth_modified = Modifier::None; ensure!(!addr.is_empty(), "Can not add_or_lookup empty address"); ensure!(origin != Origin::Unknown, "Missing valid origin"); - let addr = addr_normalize(addr).to_string(); - if context.is_self_addr(&addr).await? { - return Ok(Some((ContactId::SELF, sth_modified))); - } - - if !may_be_valid_addr(&addr) { - warn!( - context, - "Bad address \"{}\" for contact \"{}\".", - addr, - if !name.is_empty() { name } else { "" }, - ); - return Ok(None); + return Ok((ContactId::SELF, sth_modified)); } let mut name = name; @@ -555,7 +589,7 @@ impl Contact { || row_authname.is_empty()); row_id = u32::try_from(id)?; - if origin as i32 >= row_origin as i32 && addr != row_addr { + if origin >= row_origin && addr.as_ref() != row_addr { update_addr = true; } if update_name || update_authname || update_addr || origin > row_origin { @@ -657,7 +691,7 @@ impl Contact { } } - Ok(Some((ContactId::new(row_id), sth_modified))) + Ok((ContactId::new(row_id), sth_modified)) } /// Add a number of contacts. @@ -683,21 +717,25 @@ impl Contact { for (name, addr) in split_address_book(addr_book).into_iter() { let (name, addr) = sanitize_name_and_addr(name, addr); let name = normalize_name(&name); - match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await { - Err(err) => { - warn!( - context, - "Failed to add address {} from address book: {}", addr, err - ); - } - Ok(None) => { - warn!(context, "Cannot create contact with address {}.", addr); - } - Ok(Some((_, modified))) => { - if modified != Modifier::None { - modify_cnt += 1 + match ContactAddress::new(&addr) { + Ok(addr) => { + match Contact::add_or_lookup(context, &name, addr, Origin::AddressBook).await { + Ok((_, modified)) => { + if modified != Modifier::None { + modify_cnt += 1 + } + } + Err(err) => { + warn!( + context, + "Failed to add address {} from address book: {}", addr, err + ); + } } } + Err(err) => { + warn!(context, "{:#}.", err); + } } } if modify_cnt > 0 { @@ -1701,11 +1739,10 @@ mod tests { let (id, _modified) = Contact::add_or_lookup( &context.ctx, "bob", - "user@example.org", + ContactAddress::new("user@example.org")?, Origin::IncomingReplyTo, ) - .await? - .unwrap(); + .await?; assert_ne!(id, ContactId::UNDEFINED); let contact = Contact::load_from_db(&context.ctx, id).await.unwrap(); @@ -1730,11 +1767,10 @@ mod tests { let (contact_bob_id, modified) = Contact::add_or_lookup( &context.ctx, "someone", - "user@example.org", + ContactAddress::new("user@example.org")?, Origin::ManuallyCreated, ) - .await? - .unwrap(); + .await?; assert_eq!(contact_bob_id, id); assert_eq!(modified, Modifier::Modified); let contact = Contact::load_from_db(&context.ctx, id).await.unwrap(); @@ -1766,6 +1802,18 @@ mod tests { Ok(()) } + #[test] + fn test_contact_address() -> Result<()> { + let alice_addr = "alice@example.org"; + let contact_address = ContactAddress::new(alice_addr)?; + assert_eq!(contact_address.as_ref(), alice_addr); + + let invalid_addr = "<> foobar"; + assert!(ContactAddress::new(invalid_addr).is_err()); + + Ok(()) + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_add_or_lookup() { // add some contacts, this also tests add_address_book() @@ -1781,11 +1829,14 @@ mod tests { assert_eq!(Contact::add_address_book(&t, book).await.unwrap(), 4); // check first added contact, this modifies authname because it is empty - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo) - .await - .unwrap() - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bla foo", + ContactAddress::new("one@eins.org").unwrap(), + Origin::IncomingUnknownTo, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1797,11 +1848,14 @@ mod tests { assert_eq!(contact.get_name_n_addr(), "Name one (one@eins.org)"); // modify first added contact - let (contact_id_test, sth_modified) = - Contact::add_or_lookup(&t, "Real one", " one@eins.org ", Origin::ManuallyCreated) - .await - .unwrap() - .unwrap(); + let (contact_id_test, sth_modified) = Contact::add_or_lookup( + &t, + "Real one", + ContactAddress::new(" one@eins.org ").unwrap(), + Origin::ManuallyCreated, + ) + .await + .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1810,11 +1864,14 @@ mod tests { assert!(!contact.is_blocked()); // check third added contact (contact without name) - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo) - .await - .unwrap() - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("three@drei.sam").unwrap(), + Origin::IncomingUnknownTo, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::None); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1827,11 +1884,10 @@ mod tests { let (contact_id_test, sth_modified) = Contact::add_or_lookup( &t, "m. serious", - "three@drei.sam", + ContactAddress::new("three@drei.sam").unwrap(), Origin::IncomingUnknownFrom, ) .await - .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); @@ -1840,11 +1896,14 @@ mod tests { assert!(!contact.is_blocked()); // manually edit name of third contact (does not changed authorized name) - let (contact_id_test, sth_modified) = - Contact::add_or_lookup(&t, "schnucki", "three@drei.sam", Origin::ManuallyCreated) - .await - .unwrap() - .unwrap(); + let (contact_id_test, sth_modified) = Contact::add_or_lookup( + &t, + "schnucki", + ContactAddress::new("three@drei.sam").unwrap(), + Origin::ManuallyCreated, + ) + .await + .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1853,11 +1912,14 @@ mod tests { assert!(!contact.is_blocked()); // Fourth contact: - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo) - .await - .unwrap() - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("alice@w.de").unwrap(), + Origin::IncomingUnknownTo, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::None); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1992,10 +2054,13 @@ mod tests { assert!(Contact::delete(&alice, ContactId::SELF).await.is_err()); // Create Bob contact - let (contact_id, _) = - Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await? - .unwrap(); + let (contact_id, _) = Contact::add_or_lookup( + &alice, + "Bob", + ContactAddress::new("bob@example.net")?, + Origin::ManuallyCreated, + ) + .await?; let chat = alice .create_chat_with_contact("Bob", "bob@example.net") .await; @@ -2068,11 +2133,14 @@ mod tests { let t = TestContext::new().await; // incoming mail `From: bob1 ` - this should init authname - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap() - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bob1", + ContactAddress::new("bob@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Created); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -2081,11 +2149,14 @@ mod tests { assert_eq!(contact.get_display_name(), "bob1"); // incoming mail `From: bob2 ` - this should update authname - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap() - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bob2", + ContactAddress::new("bob@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -2104,11 +2175,14 @@ mod tests { assert_eq!(contact.get_display_name(), "bob3"); // incoming mail `From: bob4 ` - this should update authname, manually given name is still "bob3" - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap() - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bob4", + ContactAddress::new("bob@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -2133,11 +2207,10 @@ mod tests { let (contact_id_same, sth_modified) = Contact::add_or_lookup( &t, "claire1", - "claire@example.org", + ContactAddress::new("claire@example.org").unwrap(), Origin::IncomingUnknownFrom, ) .await - .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); @@ -2150,11 +2223,10 @@ mod tests { let (contact_id_same, sth_modified) = Contact::add_or_lookup( &t, "claire2", - "claire@example.org", + ContactAddress::new("claire@example.org").unwrap(), Origin::IncomingUnknownFrom, ) .await - .unwrap() .unwrap(); assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); @@ -2173,29 +2245,38 @@ mod tests { let t = TestContext::new().await; // Incoming message from Bob. - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "Bob", "bob@example.org", Origin::IncomingUnknownFrom) - .await? - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "Bob", + ContactAddress::new("bob@example.org")?, + Origin::IncomingUnknownFrom, + ) + .await?; assert_eq!(sth_modified, Modifier::Created); let contact = Contact::load_from_db(&t, contact_id).await?; assert_eq!(contact.get_display_name(), "Bob"); // Incoming message from someone else with "Not Bob" in the "To:" field. - let (contact_id_same, sth_modified) = - Contact::add_or_lookup(&t, "Not Bob", "bob@example.org", Origin::IncomingUnknownTo) - .await? - .unwrap(); + let (contact_id_same, sth_modified) = Contact::add_or_lookup( + &t, + "Not Bob", + ContactAddress::new("bob@example.org")?, + Origin::IncomingUnknownTo, + ) + .await?; assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await?; assert_eq!(contact.get_display_name(), "Not Bob"); // Incoming message from Bob, changing the name back. - let (contact_id_same, sth_modified) = - Contact::add_or_lookup(&t, "Bob", "bob@example.org", Origin::IncomingUnknownFrom) - .await? - .unwrap(); + let (contact_id_same, sth_modified) = Contact::add_or_lookup( + &t, + "Bob", + ContactAddress::new("bob@example.org")?, + Origin::IncomingUnknownFrom, + ) + .await?; assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); // This was None until the bugfix let contact = Contact::load_from_db(&t, contact_id).await?; @@ -2218,9 +2299,14 @@ mod tests { assert_eq!(contact.get_display_name(), "dave1"); // incoming mail `From: dave2 ` - this should update authname - Contact::add_or_lookup(&t, "dave2", "dave@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap(); + Contact::add_or_lookup( + &t, + "dave2", + ContactAddress::new("dave@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); assert_eq!(contact.get_authname(), "dave2"); assert_eq!(contact.get_name(), "dave1"); @@ -2334,10 +2420,13 @@ mod tests { let encrinfo = Contact::get_encrinfo(&alice, ContactId::DEVICE).await; assert!(encrinfo.is_err()); - let (contact_bob_id, _modified) = - Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await? - .unwrap(); + let (contact_bob_id, _modified) = Contact::add_or_lookup( + &alice, + "Bob", + ContactAddress::new("bob@example.net")?, + Origin::ManuallyCreated, + ) + .await?; let encrinfo = Contact::get_encrinfo(&alice, contact_bob_id).await?; assert_eq!(encrinfo, "No encryption"); @@ -2494,10 +2583,13 @@ CCCB 5AA9 F6E1 141C 9431 async fn test_last_seen() -> Result<()> { let alice = TestContext::new_alice().await; - let (contact_id, _) = - Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await? - .unwrap(); + let (contact_id, _) = Contact::add_or_lookup( + &alice, + "Bob", + ContactAddress::new("bob@example.net")?, + Origin::ManuallyCreated, + ) + .await?; let contact = Contact::load_from_db(&alice, contact_id).await?; assert_eq!(contact.last_seen(), 0); diff --git a/src/imap.rs b/src/imap.rs index c2bcd1f7a..fe8e3a1fb 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -22,7 +22,7 @@ use crate::config::Config; use crate::constants::{ Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION, }; -use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin}; +use crate::contact::{normalize_name, Contact, ContactAddress, ContactId, Modifier, Origin}; use crate::context::Context; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; @@ -2375,23 +2375,23 @@ async fn add_all_recipients_as_contacts( .map(|s| normalize_name(s)) .unwrap_or_default(); - match Contact::add_or_lookup( - context, - &display_name_normalized, - &recipient.addr, - Origin::OutgoingTo, - ) - .await? - { - Some((_, modified)) => { + match ContactAddress::new(&recipient.addr) { + Err(err) => warn!( + context, + "Could not add contact for recipient with address {:?}: {:#}", recipient.addr, err + ), + Ok(recipient_addr) => { + let (_, modified) = Contact::add_or_lookup( + context, + &display_name_normalized, + recipient_addr, + Origin::OutgoingTo, + ) + .await?; if modified != Modifier::None { any_modified = true; } } - None => warn!( - context, - "Could not add contact for recipient with address {:?}", recipient.addr - ), } } if any_modified { diff --git a/src/mimefactory.rs b/src/mimefactory.rs index bde304869..7025ad52c 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1490,7 +1490,7 @@ mod tests { ProtectionStatus, }; use crate::chatlist::Chatlist; - use crate::contact::Origin; + use crate::contact::{ContactAddress, Origin}; use crate::mimeparser::MimeMessage; use crate::receive_imf::receive_imf; use crate::test_utils::{get_chat_msg, TestContext}; @@ -1817,12 +1817,15 @@ mod tests { } async fn first_subject_str(t: TestContext) -> String { - let contact_id = - Contact::add_or_lookup(&t, "Dave", "dave@example.com", Origin::ManuallyCreated) - .await - .unwrap() - .unwrap() - .0; + let contact_id = Contact::add_or_lookup( + &t, + "Dave", + ContactAddress::new("dave@example.com").unwrap(), + Origin::ManuallyCreated, + ) + .await + .unwrap() + .0; let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap(); diff --git a/src/peerstate.rs b/src/peerstate.rs index 6224118d9..761fec50f 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -9,7 +9,7 @@ use crate::aheader::{Aheader, EncryptPreference}; use crate::chat::{self, Chat}; use crate::chatlist::Chatlist; use crate::constants::Chattype; -use crate::contact::{addr_cmp, Contact, Origin}; +use crate::contact::{addr_cmp, Contact, ContactAddress, Origin}; use crate::context::Context; use crate::events::EventType; use crate::key::{DcKey, Fingerprint, SignedPublicKey}; @@ -542,16 +542,30 @@ impl Peerstate { if (chat.typ == Chattype::Group && chat.is_protected()) || chat.typ == Chattype::Broadcast { - if let Some((new_contact_id, _)) = - Contact::add_or_lookup(context, "", new_addr, Origin::IncomingUnknownFrom) - .await? - { - chat::remove_from_chat_contacts_table(context, *chat_id, contact_id) - .await?; - chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]) + match ContactAddress::new(new_addr) { + Ok(new_addr) => { + let (new_contact_id, _) = Contact::add_or_lookup( + context, + "", + new_addr, + Origin::IncomingUnknownFrom, + ) .await?; + chat::remove_from_chat_contacts_table(context, *chat_id, contact_id) + .await?; + chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]) + .await?; - context.emit_event(EventType::ChatModified(*chat_id)); + context.emit_event(EventType::ChatModified(*chat_id)); + } + Err(err) => { + warn!( + context, + "New address {:?} is not vaild, not doing AEAP: {:#}.", + new_addr, + err + ) + } } } } diff --git a/src/qr.rs b/src/qr.rs index 7cd4fc28c..c6b5e18b7 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -14,7 +14,9 @@ use std::collections::BTreeMap; use crate::chat::{self, get_chat_id_by_grpid, ChatIdBlocked}; use crate::config::Config; use crate::constants::Blocked; -use crate::contact::{addr_normalize, may_be_valid_addr, Contact, ContactId, Origin}; +use crate::contact::{ + addr_normalize, may_be_valid_addr, Contact, ContactAddress, ContactId, Origin, +}; use crate::context::Context; use crate::key::Fingerprint; use crate::message::Message; @@ -221,19 +223,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { .context("Can't load peerstate")?; if let (Some(addr), Some(invitenumber), Some(authcode)) = (&addr, invitenumber, authcode) { + let addr = ContactAddress::new(addr)?; let (contact_id, _) = Contact::add_or_lookup(context, &name, addr, Origin::UnhandledQrScan) .await - .with_context(|| format!("failed to add or lookup contact for address {:?}", addr))? - .with_context(|| { - format!( - "do not want to lookup contact for invalid address {:?}", - addr - ) - })?; + .with_context(|| format!("failed to add or lookup contact for address {:?}", addr))?; if let (Some(grpid), Some(grpname)) = (grpid, grpname) { if context - .is_self_addr(addr) + .is_self_addr(&addr) .await .with_context(|| format!("can't check if address {:?} is our address", addr))? { @@ -266,7 +263,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { authcode, }) } - } else if context.is_self_addr(addr).await? { + } else if context.is_self_addr(&addr).await? { if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await { Ok(Qr::WithdrawVerifyContact { contact_id, @@ -292,13 +289,11 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { } } else if let Some(addr) = addr { if let Some(peerstate) = peerstate { + let peerstate_addr = ContactAddress::new(&peerstate.addr)?; let (contact_id, _) = - Contact::add_or_lookup(context, &name, &peerstate.addr, Origin::UnhandledQrScan) + Contact::add_or_lookup(context, &name, peerstate_addr, Origin::UnhandledQrScan) .await - .context("add_or_lookup")? - .with_context(|| { - format!("Not looking up contact for address {:?}", &peerstate.addr) - })?; + .context("add_or_lookup")?; let chat = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request) .await .context("Failed to create (new) chat for contact")?; @@ -538,11 +533,11 @@ async fn decode_mailto(context: &Context, qr: &str) -> Result { }; let addr = normalize_address(addr)?; - let name = "".to_string(); + let name = ""; Qr::from_address( context, name, - addr, + &addr, if draft.is_empty() { None } else { Some(draft) }, ) .await @@ -562,8 +557,8 @@ async fn decode_smtp(context: &Context, qr: &str) -> Result { }; let addr = normalize_address(addr)?; - let name = "".to_string(); - Qr::from_address(context, name, addr, None).await + let name = ""; + Qr::from_address(context, name, &addr, None).await } /// Extract address for the matmsg scheme. @@ -587,8 +582,8 @@ async fn decode_matmsg(context: &Context, qr: &str) -> Result { }; let addr = normalize_address(addr)?; - let name = "".to_string(); - Qr::from_address(context, name, addr, None).await + let name = ""; + Qr::from_address(context, name, &addr, None).await } static VCARD_NAME_RE: Lazy = @@ -617,20 +612,19 @@ async fn decode_vcard(context: &Context, qr: &str) -> Result { bail!("Bad e-mail address"); }; - Qr::from_address(context, name, addr, None).await + Qr::from_address(context, &name, &addr, None).await } impl Qr { pub async fn from_address( context: &Context, - name: String, - addr: String, + name: &str, + addr: &str, draft: Option, ) -> Result { + let addr = ContactAddress::new(addr)?; let (contact_id, _) = - Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan) - .await? - .context("QR code does not contain a valid address")?; + Contact::add_or_lookup(context, name, addr, Origin::UnhandledQrScan).await?; Ok(Qr::Addr { contact_id, draft }) } } diff --git a/src/reaction.rs b/src/reaction.rs index 6777c5375..f858f6172 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -286,11 +286,11 @@ pub async fn get_msg_reactions(context: &Context, msg_id: MsgId) -> Result from_addr, + Err(err) => { + warn!( + context, + "Cannot create a contact for the given From field: {:#}.", err + ); + return Ok(None); + } + }; + + let from_id = add_or_lookup_contact_by_addr( context, display_name, - &from.addr, + from_addr, Origin::IncomingUnknownFrom, ) - .await - .context("add_or_lookup_contact_by_addr")? - { - from_id - } else { - return Ok(None); - }; + .await?; if from_id == ContactId::SELF { Ok(Some((ContactId::SELF, false, Origin::OutgoingBcc))) @@ -1943,11 +1950,15 @@ async fn apply_mailinglist_changes( } let listid = &chat.grpid; - let contact_id = - match Contact::add_or_lookup(context, "", list_post, Origin::Hidden).await? { - Some((contact_id, _)) => contact_id, - None => return Ok(()), - }; + let list_post = match ContactAddress::new(list_post) { + Ok(list_post) => list_post, + Err(err) => { + warn!(context, "Invalid List-Post: {:#}.", err); + return Ok(()); + } + }; + let (contact_id, _) = + Contact::add_or_lookup(context, "", list_post, Origin::Hidden).await?; let mut contact = Contact::load_from_db(context, contact_id).await?; if contact.param.get(Param::ListId) != Some(listid) { contact.param.set(Param::ListId, listid); @@ -1955,7 +1966,7 @@ async fn apply_mailinglist_changes( } if let Some(old_list_post) = chat.param.get(Param::ListPost) { - if list_post != old_list_post { + if list_post.as_ref() != old_list_post { // Apparently the mailing list is using a different List-Post header in each message. // Make the mailing list read-only because we would't know which message the user wants to reply to. chat.param.remove(Param::ListPost); @@ -2286,9 +2297,9 @@ async fn add_or_lookup_contacts_by_address_list( continue; } let display_name = info.display_name.as_deref(); - if let Some(contact_id) = - add_or_lookup_contact_by_addr(context, display_name, addr, origin).await? - { + if let Ok(addr) = ContactAddress::new(addr) { + let contact_id = + add_or_lookup_contact_by_addr(context, display_name, addr, origin).await?; contact_ids.insert(contact_id); } else { warn!(context, "Contact with address {:?} cannot exist.", addr); @@ -2299,26 +2310,20 @@ async fn add_or_lookup_contacts_by_address_list( } /// Add contacts to database on receiving messages. -/// -/// Returns `None` if the address can't be a valid email address. async fn add_or_lookup_contact_by_addr( context: &Context, display_name: Option<&str>, - addr: &str, + addr: ContactAddress<'_>, origin: Origin, -) -> Result> { - if context.is_self_addr(addr).await? { - return Ok(Some(ContactId::SELF)); +) -> Result { + if context.is_self_addr(&addr).await? { + return Ok(ContactId::SELF); } let display_name_normalized = display_name.map(normalize_name).unwrap_or_default(); - if let Some((contact_id, _modified)) = - Contact::add_or_lookup(context, &display_name_normalized, addr, origin).await? - { - Ok(Some(contact_id)) - } else { - Ok(None) - } + let (contact_id, _modified) = + Contact::add_or_lookup(context, &display_name_normalized, addr, origin).await?; + Ok(contact_id) } #[cfg(test)] diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index 12a3901b0..cd5dcd30e 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -425,12 +425,15 @@ async fn test_escaped_recipients() { .await .unwrap(); - let carl_contact_id = - Contact::add_or_lookup(&t, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom) - .await - .unwrap() - .unwrap() - .0; + let carl_contact_id = Contact::add_or_lookup( + &t, + "Carl", + ContactAddress::new("carl@host.tld").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap() + .0; receive_imf( &t, @@ -468,12 +471,15 @@ async fn test_cc_to_contact() { .await .unwrap(); - let carl_contact_id = - Contact::add_or_lookup(&t, "garabage", "carl@host.tld", Origin::IncomingUnknownFrom) - .await - .unwrap() - .unwrap() - .0; + let carl_contact_id = Contact::add_or_lookup( + &t, + "garabage", + ContactAddress::new("carl@host.tld").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap() + .0; receive_imf( &t, @@ -2056,11 +2062,10 @@ async fn test_duplicate_message() -> Result<()> { let bob_contact_id = Contact::add_or_lookup( &alice, "Bob", - "bob@example.org", + ContactAddress::new("bob@example.org").unwrap(), Origin::IncomingUnknownFrom, ) .await? - .unwrap() .0; let first_message = b"Received: from [127.0.0.1] @@ -2112,10 +2117,14 @@ Second signature"; async fn test_ignore_footer_status_from_mailinglist() -> Result<()> { let t = TestContext::new_alice().await; t.set_config(Config::ShowEmails, Some("2")).await?; - let bob_id = Contact::add_or_lookup(&t, "", "bob@example.net", Origin::IncomingUnknownCc) - .await? - .unwrap() - .0; + let bob_id = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("bob@example.net").unwrap(), + Origin::IncomingUnknownCc, + ) + .await? + .0; let bob = Contact::load_from_db(&t, bob_id).await?; assert_eq!(bob.get_status(), ""); assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 0); @@ -2527,14 +2536,8 @@ Second thread."#; // Alice adds Fiona to both ad hoc groups. let fiona = TestContext::new_fiona().await; - let (alice_fiona_contact_id, _) = Contact::add_or_lookup( - &alice, - "Fiona", - "fiona@example.net", - Origin::IncomingUnknownTo, - ) - .await? - .unwrap(); + let alice_fiona_contact = alice.add_or_lookup_contact(&fiona).await; + let alice_fiona_contact_id = alice_fiona_contact.id; chat::add_contact_to_chat(&alice, alice_first_msg.chat_id, alice_fiona_contact_id).await?; let alice_first_invite = alice.pop_sent_msg().await; diff --git a/src/securejoin.rs b/src/securejoin.rs index f335f2e27..420966ceb 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -710,6 +710,7 @@ mod tests { use crate::chat::ProtectionStatus; use crate::chatlist::Chatlist; use crate::constants::{Chattype, DC_GCM_ADDDAYMARKER}; + use crate::contact::ContactAddress; use crate::peerstate::Peerstate; use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; @@ -1003,11 +1004,10 @@ mod tests { let (contact_bob_id, _modified) = Contact::add_or_lookup( &alice.ctx, "Bob", - "bob@example.net", + ContactAddress::new("bob@example.net")?, Origin::ManuallyCreated, ) - .await? - .unwrap(); + .await?; let contact_bob = Contact::load_from_db(&alice.ctx, contact_bob_id).await?; assert_eq!( contact_bob.is_verified(&alice.ctx).await?, diff --git a/src/test_utils.rs b/src/test_utils.rs index d8542fe2f..f5bce1cf3 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -23,7 +23,7 @@ use crate::chatlist::Chatlist; use crate::config::Config; use crate::constants::Chattype; use crate::constants::{DC_GCL_NO_SPECIALS, DC_GCM_ADDDAYMARKER, DC_MSG_ID_DAYMARKER}; -use crate::contact::{Contact, ContactId, Modifier, Origin}; +use crate::contact::{Contact, ContactAddress, ContactId, Modifier, Origin}; use crate::context::Context; use crate::events::{Event, EventType, Events}; use crate::key::{self, DcKey, KeyPair, KeyPairUse}; @@ -523,14 +523,14 @@ impl TestContext { .await .unwrap_or_default() .unwrap_or_default(); - let addr = other.ctx.get_primary_self_addr().await.unwrap(); + let primary_self_addr = other.ctx.get_primary_self_addr().await.unwrap(); + let addr = ContactAddress::new(&primary_self_addr).unwrap(); // MailinglistAddress is the lowest allowed origin, we'd prefer to not modify the // origin when creating this contact. let (contact_id, modified) = - Contact::add_or_lookup(self, &name, &addr, Origin::MailinglistAddress) + Contact::add_or_lookup(self, &name, addr, Origin::MailinglistAddress) .await - .expect("add_or_lookup") - .unwrap_or_else(|| panic!("contact with address {:?} cannot be created", &addr)); + .expect("add_or_lookup"); match modified { Modifier::None => (), Modifier::Modified => warn!(&self.ctx, "Contact {} modified by TestContext", &addr), From 790512d52a01da18eb75e11ed01971f8f3c55e18 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Jan 2023 23:19:07 +0000 Subject: [PATCH 39/77] Reduce code indentation --- src/imap.rs | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index fe8e3a1fb..f8b75574f 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -2375,23 +2375,28 @@ async fn add_all_recipients_as_contacts( .map(|s| normalize_name(s)) .unwrap_or_default(); - match ContactAddress::new(&recipient.addr) { - Err(err) => warn!( - context, - "Could not add contact for recipient with address {:?}: {:#}", recipient.addr, err - ), - Ok(recipient_addr) => { - let (_, modified) = Contact::add_or_lookup( + let recipient_addr = match ContactAddress::new(&recipient.addr) { + Err(err) => { + warn!( context, - &display_name_normalized, - recipient_addr, - Origin::OutgoingTo, - ) - .await?; - if modified != Modifier::None { - any_modified = true; - } + "Could not add contact for recipient with address {:?}: {:#}", + recipient.addr, + err + ); + continue; } + Ok(recipient_addr) => recipient_addr, + }; + + let (_, modified) = Contact::add_or_lookup( + context, + &display_name_normalized, + recipient_addr, + Origin::OutgoingTo, + ) + .await?; + if modified != Modifier::None { + any_modified = true; } } if any_modified { From f460043e8759784eaf65c06f140f32b04d3cb150 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 12 Jan 2023 12:37:41 +0000 Subject: [PATCH 40/77] Add missing documentation to webxdc module --- src/webxdc.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/webxdc.rs b/src/webxdc.rs index 7d4bb4d03..000da4fb6 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -1,7 +1,5 @@ //! # Handle webxdc messages. -#![allow(missing_docs)] - use std::convert::TryFrom; use std::path::Path; @@ -31,6 +29,7 @@ use crate::{chat, EventType}; /// In the future, that may be useful to avoid new Webxdc being loaded on old Delta Chats. const WEBXDC_API_VERSION: u32 = 1; +/// Suffix used to recognize webxdc files. pub const WEBXDC_SUFFIX: &str = "xdc"; const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png"; @@ -55,20 +54,44 @@ const WEBXDC_RECEIVING_LIMIT: u64 = 4194304; #[derive(Debug, Deserialize)] #[non_exhaustive] struct WebxdcManifest { + /// Webxdc name, used on icons or page titles. name: Option, + + /// Minimum API version required to run this webxdc. min_api: Option, + + /// Optional URL of webxdc source code. source_code_url: Option, + + /// If the webxdc requests network access. request_internet_access: Option, } /// Parsed information from WebxdcManifest and fallbacks. #[derive(Debug, Serialize)] pub struct WebxdcInfo { + /// The name of the app. + /// Defaults to filename if not set in the manifest. pub name: String, + + /// Filename of the app icon. pub icon: String, + + /// If the webxdc represents a document and allows to edit it, + /// this is the document name. + /// Otherwise an empty string. pub document: String, + + /// Short description of the webxdc state. + /// For example, "7 votes". pub summary: String, + + /// URL of webxdc source code or an empty string. pub source_code_url: String, + + /// If the webxdc is allowed to access the network. + /// It should request access, be encrypted + /// and sent to self for this. pub internet_access: bool, } From f0e900b885ec8a6a6fd253c6b69f1c57db8924e3 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Jan 2023 23:33:33 +0000 Subject: [PATCH 41/77] Cleanup constants module --- src/constants.rs | 35 +++++------------------------------ src/smtp/send.rs | 6 +++++- 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index dde6219f5..a7c60a8dd 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -68,6 +68,7 @@ impl Default for MediaQuality { } } +/// Type of the key to generate. #[derive( Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql, )] @@ -118,13 +119,13 @@ pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01; pub const DC_GCL_ADD_SELF: u32 = 0x02; // unchanged user avatars are resent to the recipients every some days -pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14; +pub(crate) const DC_RESEND_USER_AVATAR_DAYS: i64 = 14; // warn about an outdated app after a given number of days. // as we use the "provider-db generation date" as reference (that might not be updated very often) // and as not all system get speedy updates, // do not use too small value that will annoy users checking for nonexistant updates. -pub const DC_OUTDATED_WARNING_DAYS: i64 = 365; +pub(crate) const DC_OUTDATED_WARNING_DAYS: i64 = 365; /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again) pub const DC_CHAT_ID_TRASH: ChatId = ChatId::new(3); @@ -169,7 +170,7 @@ pub const DC_MSG_ID_DAYMARKER: u32 = 9; pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9; /// String that indicates that something is left out or truncated. -pub const DC_ELLIPSIS: &str = "[...]"; +pub(crate) const DC_ELLIPSIS: &str = "[...]"; // how many lines desktop can display when fullscreen (fullscreen at zoomlevel 1x) // (taken from "subjective" testing what looks ok) pub const DC_DESIRED_TEXT_LINES: usize = 38; @@ -186,11 +187,6 @@ pub const DC_DESIRED_TEXT_LINE_LEN: usize = 100; /// `char`s), not Unicode Grapheme Clusters. pub const DC_DESIRED_TEXT_LEN: usize = DC_DESIRED_TEXT_LINE_LEN * DC_DESIRED_TEXT_LINES; -// Flags for empty server job - -pub const DC_EMPTY_MVBOX: u32 = 0x01; -pub const DC_EMPTY_INBOX: u32 = 0x02; - // Flags for configuring IMAP and SMTP servers. // These flags are optional // and may be set together with the username, password etc. @@ -220,21 +216,7 @@ pub const BALANCED_IMAGE_SIZE: u32 = 1280; pub const WORSE_IMAGE_SIZE: u32 = 640; // this value can be increased if the folder configuration is changed and must be redone on next program start -pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3; - -// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks. -// this does not affect MIME'e `To:` header. -// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db. -pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50; - -pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)] -#[repr(u8)] -pub enum KeyType { - Public = 0, - Private = 1, -} +pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3; #[cfg(test)] mod tests { @@ -262,13 +244,6 @@ mod tests { assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap()); } - #[test] - fn test_keytype_values() { - // values may be written to disk and must not change - assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap()); - assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap()); - } - #[test] fn test_showemails_values() { // values may be written to disk and must not change diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 829e114b0..6f09d1346 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -4,13 +4,17 @@ use super::Smtp; use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport}; use crate::config::Config; -use crate::constants::DEFAULT_MAX_SMTP_RCPT_TO; use crate::context::Context; use crate::events::EventType; use std::time::Duration; pub type Result = std::result::Result; +// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks. +// this does not affect MIME'e `To:` header. +// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db. +pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Envelope error: {}", _0)] From 0b07dafe771c85a84d74f7d74f5e3060c06d9532 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 12 Jan 2023 17:13:27 +0100 Subject: [PATCH 42/77] add verified-by api to jsonrpc (#3946) also refactor it so that it is not a static method anymore (would have resulted in two load-Contact-from-db-calls in jsonrpc) --- CHANGELOG.md | 5 +++++ deltachat-ffi/src/lib.rs | 15 ++++++--------- deltachat-jsonrpc/src/api/types/contact.rs | 18 ++++++++++++++++++ src/contact.rs | 20 ++++++-------------- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f36e6cf43..1bddc660f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +### API-Changes +- jsonrpc: add verified-by information to `Contact`-Object + ## 1.106.0 ### Changes diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 7de6e4cd8..5ad070e83 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -3975,13 +3975,10 @@ pub unsafe extern "C" fn dc_contact_get_verifier_addr( } let ffi_contact = &*contact; let ctx = &*ffi_contact.context; - block_on(Contact::get_verifier_addr( - ctx, - &ffi_contact.contact.get_id(), - )) - .log_err(ctx, "failed to get verifier for contact") - .unwrap_or_default() - .strdup() + block_on(ffi_contact.contact.get_verifier_addr(ctx)) + .log_err(ctx, "failed to get verifier for contact") + .unwrap_or_default() + .strdup() } #[no_mangle] @@ -3992,12 +3989,12 @@ pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) } let ffi_contact = &*contact; let ctx = &*ffi_contact.context; - let contact_id = block_on(Contact::get_verifier_id(ctx, &ffi_contact.contact.get_id())) + let verifier_contact_id = block_on(ffi_contact.contact.get_verifier_id(ctx)) .log_err(ctx, "failed to get verifier") .unwrap_or_default() .unwrap_or_default(); - contact_id.to_u32() + verifier_contact_id.to_u32() } // dc_lot_t diff --git a/deltachat-jsonrpc/src/api/types/contact.rs b/deltachat-jsonrpc/src/api/types/contact.rs index 4ed4cf435..d67fc7fb1 100644 --- a/deltachat-jsonrpc/src/api/types/contact.rs +++ b/deltachat-jsonrpc/src/api/types/contact.rs @@ -20,6 +20,10 @@ pub struct ContactObject { name_and_addr: String, is_blocked: bool, is_verified: bool, + /// the address that verified this contact + verifier_addr: Option, + /// the id of the contact that verified this contact + verifier_id: Option, /// the contact's last seen timestamp last_seen: i64, was_seen_recently: bool, @@ -36,6 +40,18 @@ impl ContactObject { }; let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified; + let (verifier_addr, verifier_id) = if is_verified { + ( + contact.get_verifier_addr(context).await?, + contact + .get_verifier_id(context) + .await? + .map(|contact_id| contact_id.to_u32()), + ) + } else { + (None, None) + }; + Ok(ContactObject { address: contact.get_addr().to_owned(), color: color_int_to_hex_string(contact.get_color()), @@ -48,6 +64,8 @@ impl ContactObject { name_and_addr: contact.get_name_n_addr(), is_blocked: contact.is_blocked(), is_verified, + verifier_addr, + verifier_id, last_seen: contact.last_seen(), was_seen_recently: contact.was_seen_recently(), }) diff --git a/src/contact.rs b/src/contact.rs index 1f5233fbf..95a713b5b 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -1192,24 +1192,16 @@ impl Contact { Ok(VerifiedStatus::Unverified) } - /// Returns the address that verified the given contact. - pub async fn get_verifier_addr( - context: &Context, - contact_id: &ContactId, - ) -> Result> { - let contact = Contact::load_from_db(context, *contact_id).await?; - - Ok(Peerstate::from_addr(context, contact.get_addr()) + /// Returns the address that verified the contact. + pub async fn get_verifier_addr(&self, context: &Context) -> Result> { + Ok(Peerstate::from_addr(context, self.get_addr()) .await? .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned()))) } - /// Returns the ContactId that verified the given contact. - pub async fn get_verifier_id( - context: &Context, - contact_id: &ContactId, - ) -> Result> { - let verifier_addr = Contact::get_verifier_addr(context, contact_id).await?; + /// Returns the ContactId that verified the contact. + pub async fn get_verifier_id(&self, context: &Context) -> Result> { + let verifier_addr = self.get_verifier_addr(context).await?; if let Some(addr) = verifier_addr { Ok(Contact::lookup_id_by_addr(context, &addr, Origin::AddressBook).await?) } else { From a5354ded3f97519a580f61db14326d658a82500c Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 12 Jan 2023 17:42:19 +0000 Subject: [PATCH 43/77] ci: disable fail-fast This setting is true by default and causes Windows build to cancel when Linux fails due to flaky test and vice versa. Cancelled test then has to be restarted from scratch even though it was not going to fail. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee4abe96c..81c0b0077 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,7 @@ jobs: build_and_test: name: Build and test strategy: + fail-fast: false matrix: include: # Currently used Rust version. From 3b9a48ff5f452b93720c2c229a24729585c54677 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 12 Jan 2023 15:22:11 +0000 Subject: [PATCH 44/77] python: remove "data1=0" from INFO/WARNING/ERROR events display --- python/src/deltachat/events.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/src/deltachat/events.py b/python/src/deltachat/events.py index 72b84a44a..49ef23f96 100644 --- a/python/src/deltachat/events.py +++ b/python/src/deltachat/events.py @@ -30,7 +30,14 @@ class FFIEvent: self.data2 = data2 def __str__(self): - return "{name} data1={data1} data2={data2}".format(**self.__dict__) + if self.name == "DC_EVENT_INFO": + return "INFO {data2}".format(data2=self.data2) + elif self.name == "DC_EVENT_WARNING": + return "WARNING {data2}".format(data2=self.data2) + elif self.name == "DC_EVENT_ERROR": + return "ERROR {data2}".format(data2=self.data2) + else: + return "{name} data1={data1} data2={data2}".format(**self.__dict__) class FFIEventLogger: From 27c6cfc958576ebf074064b1163e9428bcc9a3b6 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 2 Jan 2023 13:09:44 -0300 Subject: [PATCH 45/77] Log messages in info!() if DCC_MIME_DEBUG is set Using println!() leads to reordered output on terminal. Moreover, println!() prints to stdout which is not for logging. --- src/mimefactory.rs | 16 +++++++++++++--- src/mimeparser.rs | 7 +++++-- src/receive_imf.rs | 9 ++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 7025ad52c..04c3543ab 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -722,9 +722,11 @@ impl<'a> MimeFactory<'a> { )); if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { - info!(context, "mimefactory: outgoing message mime:"); - let raw_message = message.clone().build().as_string(); - println!("{}", raw_message); + info!( + context, + "mimefactory: unencrypted message mime-body:\n{}", + message.clone().build().as_string(), + ); } let encrypted = encrypt_helper @@ -782,6 +784,14 @@ impl<'a> MimeFactory<'a> { .into_iter() .fold(outer_message, |message, header| message.header(header)); + if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { + info!( + context, + "mimefactory: outgoing message mime-body:\n{}", + outer_message.clone().build().as_string(), + ); + } + let MimeFactory { last_added_location_id, .. diff --git a/src/mimeparser.rs b/src/mimeparser.rs index c83709673..34318a143 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -244,8 +244,11 @@ impl MimeMessage { mail_raw = raw; let decrypted_mail = mailparse::parse_mail(&mail_raw)?; if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { - info!(context, "decrypted message mime-body:"); - println!("{}", String::from_utf8_lossy(&mail_raw)); + info!( + context, + "decrypted message mime-body:\n{}", + String::from_utf8_lossy(&mail_raw), + ); } (Ok(decrypted_mail), signatures, true) } diff --git a/src/receive_imf.rs b/src/receive_imf.rs index f09941441..971ce7e41 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -94,9 +94,12 @@ pub(crate) async fn receive_imf_inner( ) -> Result> { info!(context, "Receiving message, seen={}...", seen); - if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" { - info!(context, "receive_imf: incoming message mime-body:"); - println!("{}", String::from_utf8_lossy(imf_raw)); + if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { + info!( + context, + "receive_imf: incoming message mime-body:\n{}", + String::from_utf8_lossy(imf_raw), + ); } let mut mime_parser = From d64498884589dd7eda966150500cce24f58286b2 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 2 Jan 2023 13:09:56 -0300 Subject: [PATCH 46/77] Securejoin: Fix adding and handling Autocrypt-Gossip headers (#3836) - If bcc_self is set, gossip headers must be added despite of the number of group members. - If another device observes Secure-Join, instead of looking for Secure-Join-Fingerprint in "vg-member-added"/"vc-contact-confirm" messages it must use keys from Autocrypt-Gossip headers as described in the Countermitm doc (https://countermitm.readthedocs.io/en/latest/new.html#joining-a-verified-group-secure-join). --- CHANGELOG.md | 4 ++ src/mimefactory.rs | 4 +- src/securejoin.rs | 102 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 89 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bddc660f..91263fafd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,13 @@ ## Unreleased +### Fixes +- Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 + ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object + ## 1.106.0 ### Changes diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 04c3543ab..b0918d484 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -689,7 +689,9 @@ impl<'a> MimeFactory<'a> { .fold(message, |message, header| message.header(header)); // Add gossip headers in chats with multiple recipients - if peerstates.len() > 1 && self.should_do_gossip(context).await? { + if (peerstates.len() > 1 || context.get_config_bool(Config::BccSelf).await?) + && self.should_do_gossip(context).await? + { for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) { if peerstate.peek_key(min_verified).is_some() { if let Some(header) = peerstate.render_gossip_header(min_verified) { diff --git a/src/securejoin.rs b/src/securejoin.rs index 420966ceb..92ddb0733 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -579,36 +579,98 @@ pub(crate) async fn observe_securejoin_on_other_device( .await?; return Ok(HandshakeMessage::Ignore); } - let fingerprint: Fingerprint = - match mime_message.get_header(HeaderDef::SecureJoinFingerprint) { - Some(fp) => fp.parse()?, + let addr = Contact::load_from_db(context, contact_id) + .await? + .get_addr() + .to_string(); + if mime_message.gossiped_addr.contains(&addr) { + let mut peerstate = match Peerstate::from_addr(context, &addr).await? { + Some(p) => p, None => { could_not_establish_secure_connection( - context, - contact_id, - info_chat_id(context, contact_id).await?, - "Fingerprint not provided, please update Delta Chat on all your devices.", - ) - .await?; + context, + contact_id, + info_chat_id(context, contact_id).await?, + &format!("No peerstate in db for '{}' at step {}", &addr, step), + ) + .await?; return Ok(HandshakeMessage::Ignore); } }; - if mark_peer_as_verified( - context, - &fingerprint, - Contact::load_from_db(context, contact_id) - .await? - .get_addr() - .to_owned(), - ) - .await - .is_err() + let fingerprint = match peerstate.gossip_key_fingerprint.clone() { + Some(fp) => fp, + None => { + could_not_establish_secure_connection( + context, + contact_id, + info_chat_id(context, contact_id).await?, + &format!( + "No gossip key fingerprint in db for '{}' at step {}", + &addr, step, + ), + ) + .await?; + return Ok(HandshakeMessage::Ignore); + } + }; + if peerstate.set_verified( + PeerstateKeyType::GossipKey, + &fingerprint, + PeerstateVerifiedStatus::BidirectVerified, + addr, + ) { + peerstate.prefer_encrypt = EncryptPreference::Mutual; + peerstate.save_to_db(&context.sql).await.unwrap_or_default(); + } else { + could_not_establish_secure_connection( + context, + contact_id, + info_chat_id(context, contact_id).await?, + &format!( + "Could not mark peer as verified for fingerprint {} at step {}", + fingerprint.hex(), + step, + ), + ) + .await?; + return Ok(HandshakeMessage::Ignore); + } + } else if let Some(fingerprint) = + mime_message.get_header(HeaderDef::SecureJoinFingerprint) { + // FIXME: Old versions of DC send this header instead of gossips. Remove this + // eventually. + let fingerprint = fingerprint.parse()?; + if mark_peer_as_verified( + context, + &fingerprint, + Contact::load_from_db(context, contact_id) + .await? + .get_addr() + .to_owned(), + ) + .await + .is_err() + { + could_not_establish_secure_connection( + context, + contact_id, + info_chat_id(context, contact_id).await?, + format!("Fingerprint mismatch on observing {}.", step).as_ref(), + ) + .await?; + return Ok(HandshakeMessage::Ignore); + } + } else { could_not_establish_secure_connection( context, contact_id, info_chat_id(context, contact_id).await?, - format!("Fingerprint mismatch on observing {}.", step).as_ref(), + &format!( + "No gossip header for '{}' at step {}, please update Delta Chat on all \ + your devices.", + &addr, step, + ), ) .await?; return Ok(HandshakeMessage::Ignore); From 13b2fe8d30e161a754d89b67aa6884096e674e94 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 3 Jan 2023 21:12:27 -0300 Subject: [PATCH 47/77] import_self_keys(): Log set_self_key() error as DC_EVENT_INFO It isn't an error actually since we just skip the file. --- src/imex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imex.rs b/src/imex.rs index ea4033959..e265e2f6b 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -656,7 +656,7 @@ async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> { Ok(buf) => { let armored = std::string::String::from_utf8_lossy(&buf); if let Err(err) = set_self_key(context, &armored, set_default, false).await { - error!(context, "set_self_key: {}", err); + info!(context, "set_self_key: {}", err); continue; } } From 57d7df530b8a592ccd86bc3ba4fc13400f4b3f30 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Thu, 29 Dec 2022 18:55:13 -0300 Subject: [PATCH 48/77] Add a test that a new verified member is seen on the second device going online (#3836) - Alice has two devices, the second is offline. - Alice creates a verified group and sends a QR invitation to Bob. - Bob joins the group and sends a message there. Alice sees it. - Alice's second devices goes online, but doesn't see Bob in the group. --- python/tests/test_0_complex_or_slow.py | 42 ++++++++++++++++++++++++++ python/tests/test_1_online.py | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/python/tests/test_0_complex_or_slow.py b/python/tests/test_0_complex_or_slow.py index 034839ee5..48e4467fc 100644 --- a/python/tests/test_0_complex_or_slow.py +++ b/python/tests/test_0_complex_or_slow.py @@ -492,3 +492,45 @@ def test_multidevice_sync_seen(acfactory, lp): assert ac1_clone_message.is_in_seen # Test that the timer is started on the second device after synchronizing the seen status. assert "Expires: " in ac1_clone_message.get_message_info() + + +def test_see_new_verified_member_after_going_online(acfactory, tmpdir, lp): + """The test for the bug #3836: + - Alice has two devices, the second is offline. + - Alice creates a verified group and sends a QR invitation to Bob. + - Bob joins the group and sends a message there. Alice sees it. + - Alice's second devices goes online, but doesn't see Bob in the group. + """ + ac1, ac2 = acfactory.get_online_accounts(2) + ac2_addr = ac2.get_config("addr") + ac1_offl = acfactory.new_online_configuring_account(cloned_from=ac1) + for ac in [ac1, ac1_offl]: + ac.set_config("bcc_self", "1") + acfactory.bring_accounts_online() + dir = tmpdir.mkdir("exportdir") + ac1.export_self_keys(dir.strpath) + ac1_offl.import_self_keys(dir.strpath) + ac1_offl.stop_io() + + lp.sec("ac1: create verified-group QR, ac2 scans and joins") + chat = ac1.create_group_chat("hello", verified=True) + assert chat.is_protected() + qr = chat.get_join_qr() + lp.sec("ac2: start QR-code based join-group protocol") + chat2 = ac2.qr_join_chat(qr) + ac1._evtracker.wait_securejoin_inviter_progress(1000) + + lp.sec("ac2: sending message") + msg_out = chat2.send_text("hello") + + lp.sec("ac1: receiving message") + msg_in = ac1._evtracker.wait_next_incoming_message() + assert msg_in.text == msg_out.text + assert msg_in.get_sender_contact().addr == ac2_addr + + lp.sec("ac1_offl: going online, receiving message") + ac1_offl.start_io() + ac1_offl._evtracker.wait_securejoin_inviter_progress(1000) + msg_in = ac1_offl._evtracker.wait_next_incoming_message() + assert msg_in.text == msg_out.text + assert msg_in.get_sender_contact().addr == ac2_addr diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 954812d3b..9634a470a 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -88,7 +88,7 @@ def test_export_import_self_keys(acfactory, tmpdir, lp): lp.indent(dir.strpath + os.sep + name) lp.sec("importing into existing account") ac2.import_self_keys(dir.strpath) - (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*", check_error=False) + (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*") assert key_id2 == key_id From 8dc6ff268d75f2da9847af8b37bc8e8157dc1e16 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Wed, 4 Jan 2023 22:39:29 -0300 Subject: [PATCH 49/77] check_verified_properties(): Don't ignore fails of Peerstate::set_verified() - Return Result from set_verified() so that it can't be missed. - Pass Fingerprint to set_verified() by value to avoid cloning it inside. This optimises out an extra clone() if we already have a value that can be moved at the caller side. However, this may add an extra clone() if set_verified() fails, but let's not optimise the fail scenario. --- src/peerstate.rs | 31 +++++++++++++++++------------- src/receive_imf.rs | 4 ++-- src/securejoin.rs | 39 +++++++++++++++++--------------------- src/securejoin/bobstate.rs | 2 +- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/src/peerstate.rs b/src/peerstate.rs index 761fec50f..10c561287 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -17,7 +17,7 @@ use crate::message::Message; use crate::mimeparser::SystemMessage; use crate::sql::Sql; use crate::stock_str; -use anyhow::{Context as _, Result}; +use anyhow::{Context as _, Error, Result}; use num_traits::FromPrimitive; #[derive(Debug)] @@ -369,43 +369,48 @@ impl Peerstate { /// verifier: /// The address which verifies the given contact /// If we are verifying the contact, use that contacts address - /// Returns whether the value of the key has changed pub fn set_verified( &mut self, which_key: PeerstateKeyType, - fingerprint: &Fingerprint, + fingerprint: Fingerprint, verified: PeerstateVerifiedStatus, verifier: String, - ) -> bool { + ) -> Result<()> { if verified == PeerstateVerifiedStatus::BidirectVerified { match which_key { PeerstateKeyType::PublicKey => { if self.public_key_fingerprint.is_some() - && self.public_key_fingerprint.as_ref().unwrap() == fingerprint + && self.public_key_fingerprint.as_ref().unwrap() == &fingerprint { self.verified_key = self.public_key.clone(); - self.verified_key_fingerprint = self.public_key_fingerprint.clone(); + self.verified_key_fingerprint = Some(fingerprint); self.verifier = Some(verifier); - true + Ok(()) } else { - false + Err(Error::msg(format!( + "{} is not peer's public key fingerprint", + fingerprint, + ))) } } PeerstateKeyType::GossipKey => { if self.gossip_key_fingerprint.is_some() - && self.gossip_key_fingerprint.as_ref().unwrap() == fingerprint + && self.gossip_key_fingerprint.as_ref().unwrap() == &fingerprint { self.verified_key = self.gossip_key.clone(); - self.verified_key_fingerprint = self.gossip_key_fingerprint.clone(); + self.verified_key_fingerprint = Some(fingerprint); self.verifier = Some(verifier); - true + Ok(()) } else { - false + Err(Error::msg(format!( + "{} is not peer's gossip key fingerprint", + fingerprint, + ))) } } } } else { - false + Err(Error::msg("BidirectVerified required")) } } diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 971ce7e41..990eebcef 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2178,10 +2178,10 @@ async fn check_verified_properties( if let Some(fp) = fp { peerstate.set_verified( PeerstateKeyType::GossipKey, - &fp, + fp, PeerstateVerifiedStatus::BidirectVerified, contact.get_addr().to_owned(), - ); + )?; peerstate.save_to_db(&context.sql).await?; is_verified = true; } diff --git a/src/securejoin.rs b/src/securejoin.rs index 92ddb0733..eff6abd56 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -415,7 +415,7 @@ pub(crate) async fn handle_securejoin_handshake( .await? .get_addr() .to_owned(); - if mark_peer_as_verified(context, &fingerprint, contact_addr) + if mark_peer_as_verified(context, fingerprint.clone(), contact_addr) .await .is_err() { @@ -613,28 +613,23 @@ pub(crate) async fn observe_securejoin_on_other_device( return Ok(HandshakeMessage::Ignore); } }; - if peerstate.set_verified( + if let Err(err) = peerstate.set_verified( PeerstateKeyType::GossipKey, - &fingerprint, + fingerprint, PeerstateVerifiedStatus::BidirectVerified, addr, ) { - peerstate.prefer_encrypt = EncryptPreference::Mutual; - peerstate.save_to_db(&context.sql).await.unwrap_or_default(); - } else { could_not_establish_secure_connection( context, contact_id, info_chat_id(context, contact_id).await?, - &format!( - "Could not mark peer as verified for fingerprint {} at step {}", - fingerprint.hex(), - step, - ), + &format!("Could not mark peer as verified at step {}: {}", step, err), ) .await?; return Ok(HandshakeMessage::Ignore); } + peerstate.prefer_encrypt = EncryptPreference::Mutual; + peerstate.save_to_db(&context.sql).await.unwrap_or_default(); } else if let Some(fingerprint) = mime_message.get_header(HeaderDef::SecureJoinFingerprint) { @@ -643,7 +638,7 @@ pub(crate) async fn observe_securejoin_on_other_device( let fingerprint = fingerprint.parse()?; if mark_peer_as_verified( context, - &fingerprint, + fingerprint, Contact::load_from_db(context, contact_id) .await? .get_addr() @@ -715,25 +710,25 @@ async fn could_not_establish_secure_connection( async fn mark_peer_as_verified( context: &Context, - fingerprint: &Fingerprint, + fingerprint: Fingerprint, verifier: String, ) -> Result<(), Error> { - if let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, fingerprint).await? { - if peerstate.set_verified( + if let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, &fingerprint).await? { + if let Err(err) = peerstate.set_verified( PeerstateKeyType::PublicKey, fingerprint, PeerstateVerifiedStatus::BidirectVerified, verifier, ) { - peerstate.prefer_encrypt = EncryptPreference::Mutual; - peerstate.save_to_db(&context.sql).await.unwrap_or_default(); - return Ok(()); + error!(context, "Could not mark peer as verified: {}", err); + return Err(err); } + peerstate.prefer_encrypt = EncryptPreference::Mutual; + peerstate.save_to_db(&context.sql).await.unwrap_or_default(); + Ok(()) + } else { + bail!("no peerstate in db for fingerprint {}", fingerprint.hex()); } - bail!( - "could not mark peer as verified for fingerprint {}", - fingerprint.hex() - ); } /* ****************************************************************************** diff --git a/src/securejoin/bobstate.rs b/src/securejoin/bobstate.rs index e1145a006..36a43e02d 100644 --- a/src/securejoin/bobstate.rs +++ b/src/securejoin/bobstate.rs @@ -368,7 +368,7 @@ impl BobState { } mark_peer_as_verified( context, - self.invite.fingerprint(), + self.invite.fingerprint().clone(), mime_message.from.addr.to_string(), ) .await?; From 5f883a44458cbc76d0ae49ef86bb3ae51bc2ab84 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Fri, 6 Jan 2023 15:54:56 -0300 Subject: [PATCH 50/77] Prepare to remove "vc-contact-confirm-received", "vg-member-added-received" messages from Securejoin protocol --- deltachat-ffi/deltachat.h | 2 +- python/tests/test_0_complex_or_slow.py | 3 +++ src/events.rs | 2 +- src/mimefactory.rs | 11 +++++++++++ src/securejoin.rs | 12 +++++++++--- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 47452f6ac..507fbe76c 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -5811,7 +5811,7 @@ void dc_event_unref(dc_event_t* event); * @param data2 (int) The progress as: * 300=vg-/vc-request received, typically shown as "bob@addr joins". * 600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified". - * 800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol. + * 800=contact added to chat, shown as "bob@addr securely joined GROUP". Only for the verified-group-protocol. * 1000=Protocol finished for this contact. */ #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS 2060 diff --git a/python/tests/test_0_complex_or_slow.py b/python/tests/test_0_complex_or_slow.py index 48e4467fc..43e2b16c7 100644 --- a/python/tests/test_0_complex_or_slow.py +++ b/python/tests/test_0_complex_or_slow.py @@ -521,6 +521,9 @@ def test_see_new_verified_member_after_going_online(acfactory, tmpdir, lp): ac1._evtracker.wait_securejoin_inviter_progress(1000) lp.sec("ac2: sending message") + # Message can be sent only after a receipt of "vg-member-added" message. Just wait for + # "Member Me () added by ." message. + ac2._evtracker.wait_next_incoming_message() msg_out = chat2.send_text("hello") lp.sec("ac1: receiving message") diff --git a/src/events.rs b/src/events.rs index b2744f4d5..6d2962341 100644 --- a/src/events.rs +++ b/src/events.rs @@ -283,7 +283,7 @@ pub enum EventType { /// @param data2 (int) Progress as: /// 300=vg-/vc-request received, typically shown as "bob@addr joins". /// 600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified". - /// 800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol. + /// 800=contact added to chat, shown as "bob@addr securely joined GROUP". Only for the verified-group-protocol. /// 1000=Protocol finished for this contact. SecurejoinInviterProgress { contact_id: ContactId, diff --git a/src/mimefactory.rs b/src/mimefactory.rs index b0918d484..7f1f7b5d2 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -917,6 +917,17 @@ impl<'a> MimeFactory<'a> { "Secure-Join".to_string(), "vg-member-added".to_string(), )); + // FIXME: Old clients require Secure-Join-Fingerprint header. Remove this + // eventually. + let fingerprint = Peerstate::from_addr(context, email_to_add) + .await? + .context("No peerstate found in db")? + .public_key_fingerprint + .context("No public key fingerprint in db for the member to add")?; + headers.protected.push(Header::new( + "Secure-Join-Fingerprint".into(), + fingerprint.hex(), + )); } } SystemMessage::GroupNameChanged => { diff --git a/src/securejoin.rs b/src/securejoin.rs index eff6abd56..4da710ac4 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -455,6 +455,8 @@ pub(crate) async fn handle_securejoin_handshake( } None => bail!("Chat {} not found", &field_grpid), } + inviter_progress!(context, contact_id, 800); + inviter_progress!(context, contact_id, 1000); } else { // Alice -> Bob secure_connection_established( @@ -503,9 +505,6 @@ pub(crate) async fn handle_securejoin_handshake( return Ok(HandshakeMessage::Ignore); } if join_vg { - // Responsible for showing "$Bob securely joined $group" message - inviter_progress!(context, contact_id, 800); - inviter_progress!(context, contact_id, 1000); let field_grpid = mime_message .get_header(HeaderDef::SecureJoinGroup) .map(|s| s.as_str()) @@ -670,6 +669,12 @@ pub(crate) async fn observe_securejoin_on_other_device( .await?; return Ok(HandshakeMessage::Ignore); } + if step.as_str() == "vg-member-added" { + inviter_progress!(context, contact_id, 800); + } + if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" { + inviter_progress!(context, contact_id, 1000); + } Ok(if step.as_str() == "vg-member-added" { HandshakeMessage::Propagate } else { @@ -768,6 +773,7 @@ mod tests { use crate::chatlist::Chatlist; use crate::constants::{Chattype, DC_GCM_ADDDAYMARKER}; use crate::contact::ContactAddress; + use crate::contact::VerifiedStatus; use crate::peerstate::Peerstate; use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; From 1c44135b412da0bae26835ca10e4793f402ae008 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 12 Jan 2023 14:41:41 +0000 Subject: [PATCH 51/77] Remove deprecated `attach_selfavatar` config According to the comment it was added in Dec 2019 with an intention to remove it "after some time". --- CHANGELOG.md | 1 + src/chat.rs | 17 ++++++++--------- src/config.rs | 3 --- src/mimefactory.rs | 1 + 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91263fafd..33d183b9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object +- Remove `attach_selfavatar` config #3951 ## 1.106.0 diff --git a/src/chat.rs b/src/chat.rs index cef50d3d1..0b54efc50 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2902,14 +2902,12 @@ pub(crate) async fn add_contact_to_chat_ex( Ok(true) } +/// Returns true if an avatar should be attached in the given chat. +/// +/// This function does not check if the avatar is set. +/// If avatar is not set and this function returns `true`, +/// a `Chat-User-Avatar: 0` header should be sent to reset the avatar. pub(crate) async fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Result { - // versions before 12/2019 already allowed to set selfavatar, however, it was never sent to others. - // to avoid sending out previously set selfavatars unexpectedly we added this additional check. - // it can be removed after some time. - if !context.sql.get_raw_config_bool("attach_selfavatar").await? { - return Ok(false); - } - let timestamp_some_days_ago = time() - DC_RESEND_USER_AVATAR_DAYS * 24 * 60 * 60; let needs_attach = context .sql @@ -4700,12 +4698,13 @@ mod tests { ) .await?; add_contact_to_chat(&t, chat_id, contact_id).await?; - assert!(!shall_attach_selfavatar(&t, chat_id).await?); - t.set_config(Config::Selfavatar, None).await?; // setting to None also forces re-sending assert!(shall_attach_selfavatar(&t, chat_id).await?); chat_id.set_selfavatar_timestamp(&t, time()).await?; assert!(!shall_attach_selfavatar(&t, chat_id).await?); + + t.set_config(Config::Selfavatar, None).await?; // setting to None also forces re-sending + assert!(shall_attach_selfavatar(&t, chat_id).await?); Ok(()) } diff --git a/src/config.rs b/src/config.rs index 40cddcc8d..41b884fab 100644 --- a/src/config.rs +++ b/src/config.rs @@ -292,9 +292,6 @@ impl Context { self.sql .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![]) .await?; - self.sql - .set_raw_config_bool("attach_selfavatar", true) - .await?; match value { Some(value) => { let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?; diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 7f1f7b5d2..bd198329d 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -76,6 +76,7 @@ pub struct MimeFactory<'a> { /// and must be deleted if the message is actually queued for sending. sync_ids_to_delete: Option, + /// True if the avatar should be attached. attach_selfavatar: bool, } From 0053e143e77a45eeffb5c260051d349712ab158c Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 12 Jan 2023 20:51:07 +0000 Subject: [PATCH 52/77] Do not emit ChatModified event when user avatar is updated --- src/receive_imf.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 990eebcef..a28601cd4 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -290,7 +290,7 @@ pub(crate) async fn receive_imf_inner( .update_contacts_timestamp(from_id, Param::AvatarTimestamp, sent_timestamp) .await? { - match contact::set_profile_image( + if let Err(err) = contact::set_profile_image( context, from_id, avatar_action, @@ -298,15 +298,10 @@ pub(crate) async fn receive_imf_inner( ) .await { - Ok(()) => { - context.emit_event(EventType::ChatModified(chat_id)); - } - Err(err) => { - warn!( - context, - "receive_imf cannot update profile image: {:#}", err - ); - } + warn!( + context, + "receive_imf cannot update profile image: {:#}", err + ); }; } } From 5b265dbc1c2e497643b0c0af6247a89334ea7bf7 Mon Sep 17 00:00:00 2001 From: bjoern Date: Fri, 13 Jan 2023 18:25:20 +0100 Subject: [PATCH 53/77] remove comma from unit in message-details (#3954) it shoud read "filename.ext, 123 bytes" and not "filename.ext, 123, bytes" --- src/message.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index 63bf1450f..48819a218 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1180,7 +1180,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { if let Some(path) = msg.get_file(context) { let bytes = get_filebytes(context, &path).await?; - ret += &format!("\nFile: {}, {}, bytes\n", path.display(), bytes); + ret += &format!("\nFile: {}, {} bytes\n", path.display(), bytes); } if msg.viewtype != Viewtype::Text { From badbf766bb3b0e82cfbfb778900c377fe1aaf875 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Fri, 13 Jan 2023 15:47:26 -0300 Subject: [PATCH 54/77] Move event emitting for a new message to a separate function --- src/chat.rs | 16 +++++++++++----- src/receive_imf.rs | 6 +----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 0b54efc50..042f10b2d 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -545,6 +545,16 @@ impl ChatId { Ok(()) } + /// Emits an appropriate event for a message. `important` is whether a notification should be + /// shown. + pub(crate) fn emit_msg_event(self, context: &Context, msg_id: MsgId, important: bool) { + if important { + context.emit_incoming_msg(self, msg_id); + } else { + context.emit_msgs_changed(self, msg_id); + } + } + /// Deletes a chat. pub async fn delete(self, context: &Context) -> Result<()> { ensure!( @@ -3469,11 +3479,7 @@ pub async fn add_device_msg_with_importance( } if !msg_id.is_unset() { - if important { - context.emit_incoming_msg(chat_id, msg_id); - } else { - context.emit_msgs_changed(chat_id, msg_id); - } + chat_id.emit_msg_event(context, msg_id, important); } Ok(msg_id) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index a28601cd4..55f099726 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -356,11 +356,7 @@ pub(crate) async fn receive_imf_inner( } else if !chat_id.is_trash() { let fresh = received_msg.state == MessageState::InFresh; for msg_id in &received_msg.msg_ids { - if incoming && fresh { - context.emit_incoming_msg(chat_id, *msg_id); - } else { - context.emit_msgs_changed(chat_id, *msg_id); - }; + chat_id.emit_msg_event(context, *msg_id, incoming && fresh); } } From 3cf78749df36126d35d8c6a88e2f2b598e7edb77 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Fri, 13 Jan 2023 15:58:52 -0300 Subject: [PATCH 55/77] Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with unread messages increases (#3940) --- CHANGELOG.md | 2 ++ python/tests/test_1_online.py | 33 ++++++++++++++++++ src/chat.rs | 65 ++++++++++++++++++++++++++++------- src/receive_imf.rs | 2 +- 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33d183b9c..6d20774bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Fixes - Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 +- Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with + unread messages increases #3959 ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 9634a470a..32a355fad 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -2261,6 +2261,39 @@ def test_aeap_flow_verified(acfactory, lp): assert ac1new.get_config("addr") in [contact.addr for contact in msg_in_2.chat.get_contacts()] +def test_archived_muted_chat(acfactory, lp): + """If an archived and muted chat receives a new message, DC_EVENT_MSGS_CHANGED for + DC_CHAT_ID_ARCHIVED_LINK must be generated if the chat had only seen messages previously. + """ + ac1, ac2 = acfactory.get_online_accounts(2) + chat = acfactory.get_accepted_chat(ac1, ac2) + + lp.sec("ac1: send message to ac2") + chat.send_text("message0") + + lp.sec("wait for ac2 to receive message") + msg2 = ac2._evtracker.wait_next_incoming_message() + assert msg2.text == "message0" + msg2.mark_seen() + + chat2 = msg2.chat + chat2.archive() + chat2.mute() + + lp.sec("ac1: send another message to ac2") + chat.send_text("message1") + + lp.sec("wait for ac2 to receive DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK") + while 1: + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") + if ev.data1 == const.DC_CHAT_ID_ARCHIVED_LINK: + assert ev.data2 == 0 + archive = ac2.get_chat_by_id(const.DC_CHAT_ID_ARCHIVED_LINK) + assert archive.count_fresh_messages() == 1 + assert chat2.count_fresh_messages() == 1 + break + + class TestOnlineConfigureFails: def test_invalid_password(self, acfactory): configdict = acfactory.get_next_liveconfig() diff --git a/src/chat.rs b/src/chat.rs index 042f10b2d..d80364b9f 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -531,16 +531,50 @@ impl ChatId { Ok(()) } - // Unarchives a chat that is archived and not muted. - // Needed when a message is added to a chat so that the chat gets a normal visibility again. - // Sending an appropriate event is up to the caller. - pub async fn unarchive_if_not_muted(self, context: &Context) -> Result<()> { + /// Unarchives a chat that is archived and not muted. + /// Needed after a message is added to a chat so that the chat gets a normal visibility again. + /// `msg_state` is the state of the message. Matters only for incoming messages currently. For + /// multiple outgoing messages the function may be called once with MessageState::Undefined. + /// Sending an appropriate event is up to the caller. + /// Also emits DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived + /// chats with unread messages increases (which is possible if the chat is muted). + pub async fn unarchive_if_not_muted( + self, + context: &Context, + msg_state: MessageState, + ) -> Result<()> { + if msg_state != MessageState::InFresh { + context.sql.execute( + "UPDATE chats SET archived=0 WHERE id=? AND NOT(muted_until=-1 OR muted_until>?)", + paramsv![self, time()], + ).await?; + return Ok(()); + } + let chat = Chat::load_from_db(context, self).await?; + if chat.visibility != ChatVisibility::Archived { + return Ok(()); + } + if chat.is_muted() { + let unread_cnt = context + .sql + .count( + "SELECT COUNT(*) + FROM msgs + WHERE state=? + AND hidden=0 + AND chat_id=?", + paramsv![MessageState::InFresh, self], + ) + .await?; + if unread_cnt == 1 { + // Added the first unread message in the chat. + context.emit_msgs_changed(DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)); + } + return Ok(()); + } context .sql - .execute( - "UPDATE chats SET archived=0 WHERE id=? AND archived=1 AND NOT(muted_until=-1 OR muted_until>?)", - paramsv![self, time()], - ) + .execute("UPDATE chats SET archived=0 WHERE id=?", paramsv![self]) .await?; Ok(()) } @@ -2019,7 +2053,9 @@ async fn prepare_msg_common( msg.state = change_state_to; prepare_msg_blob(context, msg).await?; - chat_id.unarchive_if_not_muted(context).await?; + if !msg.hidden { + chat_id.unarchive_if_not_muted(context, msg.state).await?; + } msg.id = chat .prepare_msg_raw( context, @@ -3211,7 +3247,9 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) let mut created_msgs: Vec = Vec::new(); let mut curr_timestamp: i64; - chat_id.unarchive_if_not_muted(context).await?; + chat_id + .unarchive_if_not_muted(context, MessageState::Undefined) + .await?; if let Ok(mut chat) = Chat::load_from_db(context, chat_id).await { if let Some(reason) = chat.why_cant_send(context).await? { bail!("cannot send to {}: {}", chat_id, reason); @@ -3414,7 +3452,6 @@ pub async fn add_device_msg_with_importance( let rfc724_mid = create_outgoing_rfc724_mid(None, "@device"); msg.try_calc_and_set_dimensions(context).await.ok(); prepare_msg_blob(context, msg).await?; - chat_id.unarchive_if_not_muted(context).await?; let timestamp_sent = create_smeared_timestamp(context).await; @@ -3434,6 +3471,7 @@ pub async fn add_device_msg_with_importance( } } + let state = MessageState::InFresh; let row_id = context .sql .insert( @@ -3457,7 +3495,7 @@ pub async fn add_device_msg_with_importance( timestamp_sent, timestamp_sent, // timestamp_sent equals timestamp_rcvd msg.viewtype, - MessageState::InFresh, + state, msg.text.as_ref().cloned().unwrap_or_default(), msg.param.to_string(), rfc724_mid, @@ -3466,6 +3504,9 @@ pub async fn add_device_msg_with_importance( .await?; msg_id = MsgId::new(u32::try_from(row_id)?); + if !msg.hidden { + chat_id.unarchive_if_not_muted(context, state).await?; + } } if let Some(label) = label { diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 55f099726..5ede0f424 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1235,7 +1235,7 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, replace_msg_id.delete_from_db(context).await?; } - chat_id.unarchive_if_not_muted(context).await?; + chat_id.unarchive_if_not_muted(context, state).await?; info!( context, From ed2486730990a066b5a367311c261778b364149d Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Tue, 17 Jan 2023 14:06:57 +0100 Subject: [PATCH 56/77] fix verifier-by addr was empty string instead of None (#3961) fix verifier-by addr was empty string intead of None --- CHANGELOG.md | 1 + src/peerstate.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d20774bc..b1ab928cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixes - Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 +- fix verifier-by addr was empty string intead of None #3961 - Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with unread messages increases #3959 diff --git a/src/peerstate.rs b/src/peerstate.rs index 10c561287..d0b9f335d 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -223,7 +223,10 @@ impl Peerstate { .transpose() .unwrap_or_default(), fingerprint_changed: false, - verifier: row.get("verifier")?, + verifier: { + let verifier: Option = row.get("verifier")?; + verifier.filter(|verifier| !verifier.is_empty()) + }, }; Ok(res) From e43b36b61f6b54ca9a6691b9f83b6e99d8fb2727 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 17 Jan 2023 13:57:27 +0000 Subject: [PATCH 57/77] Peerstate.verifier fixes Derive Debug, PartialEq and Eq for Peerstate, so `verifier` is included in Debug output and compared. Store verifier as empty string instead of NULL in the database. --- CHANGELOG.md | 1 + src/contact.rs | 23 +++++++++++++++++++++++ src/peerstate.rs | 42 ++---------------------------------------- 3 files changed, 26 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1ab928cd..49bce56dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - fix verifier-by addr was empty string intead of None #3961 - Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with unread messages increases #3959 +- Fix Peerstate comparison #3962 ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object diff --git a/src/contact.rs b/src/contact.rs index 95a713b5b..33b51ab8e 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -2628,4 +2628,27 @@ Hi."#; Ok(()) } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_verified_by_none() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + let bob = tcm.bob().await; + + let contact_id = Contact::create(&alice, "Bob", "bob@example.net").await?; + let contact = Contact::get_by_id(&alice, contact_id).await?; + assert!(contact.get_verifier_addr(&alice).await?.is_none()); + assert!(contact.get_verifier_id(&alice).await?.is_none()); + + // Receive a message from Bob to create a peerstate. + let chat = bob.create_chat(&alice).await; + let sent_msg = bob.send_text(chat.id, "moin").await; + alice.recv_msg(&sent_msg).await; + + let contact = Contact::get_by_id(&alice, contact_id).await?; + assert!(contact.get_verifier_addr(&alice).await?.is_none()); + assert!(contact.get_verifier_id(&alice).await?.is_none()); + + Ok(()) + } } diff --git a/src/peerstate.rs b/src/peerstate.rs index d0b9f335d..0c54d80c5 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -3,7 +3,6 @@ #![allow(missing_docs)] use std::collections::HashSet; -use std::fmt; use crate::aheader::{Aheader, EncryptPreference}; use crate::chat::{self, Chat}; @@ -35,6 +34,7 @@ pub enum PeerstateVerifiedStatus { } /// Peerstate represents the state of an Autocrypt peer. +#[derive(Debug, PartialEq, Eq)] pub struct Peerstate { pub addr: String, pub last_seen: i64, @@ -52,44 +52,6 @@ pub struct Peerstate { pub verifier: Option, } -impl PartialEq for Peerstate { - fn eq(&self, other: &Peerstate) -> bool { - self.addr == other.addr - && self.last_seen == other.last_seen - && self.last_seen_autocrypt == other.last_seen_autocrypt - && self.prefer_encrypt == other.prefer_encrypt - && self.public_key == other.public_key - && self.public_key_fingerprint == other.public_key_fingerprint - && self.gossip_key == other.gossip_key - && self.gossip_timestamp == other.gossip_timestamp - && self.gossip_key_fingerprint == other.gossip_key_fingerprint - && self.verified_key == other.verified_key - && self.verified_key_fingerprint == other.verified_key_fingerprint - && self.fingerprint_changed == other.fingerprint_changed - } -} - -impl Eq for Peerstate {} - -impl fmt::Debug for Peerstate { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("Peerstate") - .field("addr", &self.addr) - .field("last_seen", &self.last_seen) - .field("last_seen_autocrypt", &self.last_seen_autocrypt) - .field("prefer_encrypt", &self.prefer_encrypt) - .field("public_key", &self.public_key) - .field("public_key_fingerprint", &self.public_key_fingerprint) - .field("gossip_key", &self.gossip_key) - .field("gossip_timestamp", &self.gossip_timestamp) - .field("gossip_key_fingerprint", &self.gossip_key_fingerprint) - .field("verified_key", &self.verified_key) - .field("verified_key_fingerprint", &self.verified_key_fingerprint) - .field("fingerprint_changed", &self.fingerprint_changed) - .finish() - } -} - impl Peerstate { pub fn from_header(header: &Aheader, message_time: i64) -> Self { Peerstate { @@ -458,7 +420,7 @@ impl Peerstate { self.verified_key.as_ref().map(|k| k.to_bytes()), self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()), self.addr, - self.verifier, + self.verifier.as_deref().unwrap_or(""), ], ) .await?; From cf0349acc80ffdafbe22ef77767edd323d4e3a2f Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 18 Jan 2023 09:30:14 +0000 Subject: [PATCH 58/77] configure: log SOCKS5 configuration for IMAP like we do for SMTP --- CHANGELOG.md | 1 + src/configure.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49bce56dd..c53a141f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with unread messages increases #3959 - Fix Peerstate comparison #3962 +- Log SOCKS5 configuration for IMAP like already done for SMTP #3964 ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object diff --git a/src/configure.rs b/src/configure.rs index 82403a76a..493980233 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -565,13 +565,18 @@ async fn try_imap_one_param( provider_strict_tls: bool, ) -> Result { let inf = format!( - "imap: {}@{}:{} security={} certificate_checks={} oauth2={}", + "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}", param.user, param.server, param.port, param.security, param.certificate_checks, - param.oauth2 + param.oauth2, + if let Some(socks5_config) = socks5_config { + socks5_config.to_string() + } else { + "None".to_string() + } ); info!(context, "Trying: {}", inf); From 42c709e7b1647ec240e1b1356be58f99b86d8073 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 18 Jan 2023 10:12:18 +0000 Subject: [PATCH 59/77] Fix SOCKS5 usage for IMAP Connect to SOCKS5 server rather than target server and send TCP connect command. --- CHANGELOG.md | 1 + src/imap.rs | 2 +- src/imap/client.rs | 17 +++++++---------- src/socks.rs | 15 +++++++++++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c53a141f9..850a05104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ unread messages increases #3959 - Fix Peerstate comparison #3962 - Log SOCKS5 configuration for IMAP like already done for SMTP #3964 +- Fix SOCKS5 usage for IMAP #3965 ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object diff --git a/src/imap.rs b/src/imap.rs index f8b75574f..0d91da8f7 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -315,7 +315,7 @@ impl Imap { ) .await } else { - Client::connect_insecure_socks5((imap_server, imap_port), socks5_config.clone()) + Client::connect_insecure_socks5(imap_server, imap_port, socks5_config.clone()) .await } } else if config.lp.security == Socket::Starttls { diff --git a/src/imap/client.rs b/src/imap/client.rs index 74624a119..f4eacd6dc 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -124,8 +124,7 @@ impl Client { let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?; // Run STARTTLS command and convert the client back into a stream. - let session_stream: Box = Box::new(tcp_stream); - let mut client = ImapClient::new(session_stream); + let mut client = ImapClient::new(tcp_stream); let _greeting = client .read_response() .await @@ -155,7 +154,7 @@ impl Client { strict_tls: bool, socks5_config: Socks5Config, ) -> Result { - let socks5_stream = socks5_config.connect((domain, port), IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config.connect(domain, port, IMAP_TIMEOUT).await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(domain, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); @@ -170,10 +169,11 @@ impl Client { } pub async fn connect_insecure_socks5( - target_addr: impl ToSocketAddrs, + domain: &str, + port: u16, socks5_config: Socks5Config, ) -> Result { - let socks5_stream = socks5_config.connect(target_addr, IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config.connect(domain, port, IMAP_TIMEOUT).await?; let buffered_stream = BufWriter::new(socks5_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -191,13 +191,10 @@ impl Client { socks5_config: Socks5Config, strict_tls: bool, ) -> Result { - let socks5_stream = socks5_config - .connect((hostname, port), IMAP_TIMEOUT) - .await?; + let socks5_stream = socks5_config.connect(hostname, port, IMAP_TIMEOUT).await?; // Run STARTTLS command and convert the client back into a stream. - let session_stream: Box = Box::new(socks5_stream); - let mut client = ImapClient::new(session_stream); + let mut client = ImapClient::new(socks5_stream); let _greeting = client .read_response() .await diff --git a/src/socks.rs b/src/socks.rs index e7ea20730..4dd1ffcc5 100644 --- a/src/socks.rs +++ b/src/socks.rs @@ -7,12 +7,14 @@ use std::time::Duration; use crate::net::connect_tcp; use anyhow::Result; pub use async_smtp::ServerAddress; -use tokio::net::{self, TcpStream}; +use tokio::net::TcpStream; use tokio_io_timeout::TimeoutStream; use crate::context::Context; use fast_socks5::client::{Config, Socks5Stream}; +use fast_socks5::util::target_addr::ToTargetAddr; use fast_socks5::AuthenticationMethod; +use fast_socks5::Socks5Command; #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct Socks5Config { @@ -56,10 +58,11 @@ impl Socks5Config { pub async fn connect( &self, - target_addr: impl net::ToSocketAddrs, + target_host: &str, + target_port: u16, timeout_val: Duration, ) -> Result>>>> { - let tcp_stream = connect_tcp(target_addr, timeout_val).await?; + let tcp_stream = connect_tcp((self.host.clone(), self.port), timeout_val).await?; let authentication_method = if let Some((username, password)) = self.user_password.as_ref() { @@ -70,8 +73,12 @@ impl Socks5Config { } else { None }; - let socks_stream = + let mut socks_stream = Socks5Stream::use_stream(tcp_stream, authentication_method, Config::default()).await?; + let target_addr = (target_host, target_port).to_target_addr()?; + socks_stream + .request(Socks5Command::TCPConnect, target_addr) + .await?; Ok(socks_stream) } From de268b8225ce1e58a0a9c26db7ead035c75ec5b0 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 18 Jan 2023 10:48:22 +0000 Subject: [PATCH 60/77] Terminate recently seen loop if cannot receive interrupts It seems .abort() does not work on the recently seen loop in some cases, e.g. if it is busy looping in a separate thread. In my case after account reconfiguration recently seen loop kept running and issuing warnings about closed interrupt channel. Exit from recently seen loop on errors to avoid using 100% CPU in such cases. --- CHANGELOG.md | 1 + src/contact.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 850a05104..1b66a8f9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - Fix Peerstate comparison #3962 - Log SOCKS5 configuration for IMAP like already done for SMTP #3964 - Fix SOCKS5 usage for IMAP #3965 +- Exit from recently seen loop on interrupt channel errors to avoid busy looping #3966 ### API-Changes - jsonrpc: add verified-by information to `Contact`-Object diff --git a/src/contact.rs b/src/contact.rs index 33b51ab8e..056797de0 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -1612,6 +1612,9 @@ impl RecentlySeenLoop { context, "Error receiving an interruption in recently seen loop: {}", err ); + // Maybe the sender side is closed. + // Terminate the loop to avoid looping indefinitely. + return; } Ok(Ok(RecentlySeenInterrupt { contact_id, From 3c3710420b5156f212e98517908a5d27fef95366 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Wed, 18 Jan 2023 13:47:32 -0300 Subject: [PATCH 61/77] Don't unpin chat on sending / receiving not fresh messages (#3967) This bug was introduced by 3cf78749df36126d35d8c6a88e2f2b598e7edb77 - there are three visibility states: Archived, Pinned and Normal - in the database, this "visibility" is named historically "archived" ... the original code has an AND archived=1 therefore. --- src/chat.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index d80364b9f..5a9bfde64 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -544,10 +544,14 @@ impl ChatId { msg_state: MessageState, ) -> Result<()> { if msg_state != MessageState::InFresh { - context.sql.execute( - "UPDATE chats SET archived=0 WHERE id=? AND NOT(muted_until=-1 OR muted_until>?)", - paramsv![self, time()], - ).await?; + context + .sql + .execute( + "UPDATE chats SET archived=0 WHERE id=? AND archived=1 \ + AND NOT(muted_until=-1 OR muted_until>?)", + paramsv![self, time()], + ) + .await?; return Ok(()); } let chat = Chat::load_from_db(context, self).await?; @@ -4690,6 +4694,46 @@ mod tests { assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_pinned_after_new_msgs() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let alice_chat_id = alice.create_chat(&bob).await.id; + let bob_chat_id = bob.create_chat(&alice).await.id; + + assert!(alice_chat_id + .set_visibility(&alice, ChatVisibility::Pinned) + .await + .is_ok()); + assert_eq!( + Chat::load_from_db(&alice, alice_chat_id) + .await? + .get_visibility(), + ChatVisibility::Pinned, + ); + + send_text_msg(&alice, alice_chat_id, "hi!".into()).await?; + assert_eq!( + Chat::load_from_db(&alice, alice_chat_id) + .await? + .get_visibility(), + ChatVisibility::Pinned, + ); + + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("hi!".into())); + let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.chat_id, alice_chat_id); + assert_eq!( + Chat::load_from_db(&alice, alice_chat_id) + .await? + .get_visibility(), + ChatVisibility::Pinned, + ); + Ok(()) + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_set_chat_name() { let t = TestContext::new().await; From 677136f4abf5203b6458aece8f8d190a23b16d87 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 6 Jan 2023 13:50:51 +0000 Subject: [PATCH 62/77] Use SMTP pipelining --- CHANGELOG.md | 3 +++ Cargo.lock | 3 +-- Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b66a8f9f..9ea9cc51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Changes +- Pipeline SMTP commands #3924 + ### Fixes - Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 - fix verifier-by addr was empty string intead of None #3961 diff --git a/Cargo.lock b/Cargo.lock index 8bc55496c..90f7593ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,8 +166,7 @@ dependencies = [ [[package]] name = "async-smtp" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da21e1dd19fbad3e095ad519fb1558ab77fd82e5c4778dca8f9be0464589e1e" +source = "git+https://github.com/async-email/async-smtp?branch=master#983597125fbb1b0eefd3eb5a21d917af71b2e1c3" dependencies = [ "async-native-tls", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 2ba38945e..01a31e28a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ ansi_term = { version = "0.12.1", optional = true } anyhow = "1" async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] } async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] } -async-smtp = { version = "0.5", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } +async-smtp = { git = "https://github.com/async-email/async-smtp", branch = "master", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } trust-dns-resolver = "0.22" tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar From 4615c84f310ef53d2919e3de18cb8cf909e6d959 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 13:13:25 +0000 Subject: [PATCH 63/77] Automatically group imports using nightly rustfmt --- benches/create_account.rs | 3 ++- benches/get_chat_msgs.rs | 1 - benches/get_chatlist.rs | 1 - benches/search_msgs.rs | 3 ++- deltachat-ffi/src/lib.rs | 19 ++++++++++--------- deltachat-ffi/src/lot.rs | 6 ++++-- deltachat-ffi/src/string.rs | 3 ++- deltachat-jsonrpc/src/api/mod.rs | 18 ++++++++---------- deltachat-jsonrpc/src/lib.rs | 3 ++- deltachat-jsonrpc/src/webserver.rs | 3 ++- deltachat_derive/src/lib.rs | 3 ++- examples/simple.rs | 3 +-- src/accounts.rs | 1 - src/aheader.rs | 3 ++- src/authres.rs | 1 - src/blob.rs | 6 ++---- src/chat.rs | 2 -- src/chatlist.rs | 1 - src/config.rs | 7 +++---- src/configure.rs | 7 +++---- src/configure/auto_mozilla.rs | 9 ++++----- src/configure/auto_outlook.rs | 7 +++---- src/contact.rs | 1 - src/context.rs | 11 ++++++----- src/decrypt.rs | 3 +-- src/download.rs | 8 ++++---- src/e2ee.rs | 3 +-- src/ephemeral.rs | 2 +- src/html.rs | 6 +++--- src/http.rs | 3 ++- src/imap/client.rs | 5 +---- src/imap/idle.rs | 4 ++-- src/imap/scan_folders.rs | 3 +-- src/imap/select_folder.rs | 6 +++--- src/imex.rs | 5 ++--- src/job.rs | 1 - src/key.rs | 12 ++++++------ src/log.rs | 3 ++- src/login_param.rs | 1 - src/message.rs | 3 +-- src/mimefactory.rs | 3 +-- src/mimeparser.rs | 3 ++- src/oauth2.rs | 1 - src/param.rs | 3 +-- src/peerstate.rs | 5 +++-- src/pgp.rs | 5 +++-- src/plaintext.rs | 3 ++- src/provider.rs | 10 ++++++---- src/provider/data.rs | 7 ++++--- src/qr.rs | 11 +++++------ src/qr/dclogin_scheme.rs | 11 ++++++----- src/quota.rs | 3 ++- src/reaction.rs | 1 - src/receive_imf/tests.rs | 1 - src/scheduler.rs | 3 +-- src/scheduler/connectivity.rs | 4 ++-- src/securejoin.rs | 4 ++-- src/securejoin/bob.rs | 7 +++---- src/securejoin/bobstate.rs | 5 ++--- src/simplify.rs | 3 ++- src/smtp/send.rs | 5 +++-- src/socks.rs | 10 +++++----- src/sql.rs | 3 +-- src/stock_str.rs | 5 ++--- src/summary.rs | 5 +++-- src/sync.rs | 12 +++++++----- src/tools.rs | 7 +++---- src/update_helper.rs | 3 ++- src/webxdc.rs | 3 +-- 69 files changed, 162 insertions(+), 174 deletions(-) diff --git a/benches/create_account.rs b/benches/create_account.rs index 56f8877d6..5e1ae8561 100644 --- a/benches/create_account.rs +++ b/benches/create_account.rs @@ -1,6 +1,7 @@ +use std::path::PathBuf; + use criterion::{black_box, criterion_group, criterion_main, Criterion}; use deltachat::accounts::Accounts; -use std::path::PathBuf; use tempfile::tempdir; async fn create_accounts(n: u32) { diff --git a/benches/get_chat_msgs.rs b/benches/get_chat_msgs.rs index 61d39e0e7..fac37dd87 100644 --- a/benches/get_chat_msgs.rs +++ b/benches/get_chat_msgs.rs @@ -1,7 +1,6 @@ use std::path::Path; use criterion::{black_box, criterion_group, criterion_main, Criterion}; - use deltachat::chat::{self, ChatId}; use deltachat::chatlist::Chatlist; use deltachat::context::Context; diff --git a/benches/get_chatlist.rs b/benches/get_chatlist.rs index 3b7a0ca1d..7b0f5556e 100644 --- a/benches/get_chatlist.rs +++ b/benches/get_chatlist.rs @@ -1,7 +1,6 @@ use std::path::Path; use criterion::{black_box, criterion_group, criterion_main, Criterion}; - use deltachat::chatlist::Chatlist; use deltachat::context::Context; use deltachat::stock_str::StockStrings; diff --git a/benches/search_msgs.rs b/benches/search_msgs.rs index 48333696d..5605eb3cf 100644 --- a/benches/search_msgs.rs +++ b/benches/search_msgs.rs @@ -1,8 +1,9 @@ +use std::path::Path; + use criterion::{black_box, criterion_group, criterion_main, Criterion}; use deltachat::context::Context; use deltachat::stock_str::StockStrings; use deltachat::Events; -use std::path::Path; async fn search_benchmark(dbfile: impl AsRef) { let id = 100; diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 5ad070e83..45504d041 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -23,13 +23,6 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use anyhow::Context as _; -use deltachat::qr_code_generator::get_securejoin_qr_svg; -use num_traits::{FromPrimitive, ToPrimitive}; -use once_cell::sync::Lazy; -use rand::Rng; -use tokio::runtime::Runtime; -use tokio::sync::RwLock; - use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use deltachat::constants::DC_MSG_ID_LAST_SPECIAL; use deltachat::contact::{Contact, ContactId, Origin}; @@ -37,21 +30,28 @@ use deltachat::context::Context; use deltachat::ephemeral::Timer as EphemeralTimer; use deltachat::key::DcKey; use deltachat::message::MsgId; +use deltachat::qr_code_generator::get_securejoin_qr_svg; use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions}; use deltachat::stock_str::StockMessage; use deltachat::stock_str::StockStrings; use deltachat::webxdc::StatusUpdateSerial; use deltachat::*; use deltachat::{accounts::Accounts, log::LogExt}; +use num_traits::{FromPrimitive, ToPrimitive}; +use once_cell::sync::Lazy; +use rand::Rng; +use tokio::runtime::Runtime; +use tokio::sync::RwLock; use tokio::task::JoinHandle; mod dc_array; mod lot; mod string; -use self::string::*; use deltachat::chatlist::Chatlist; +use self::string::*; + // as C lacks a good and portable error handling, // in general, the C Interface is forgiving wrt to bad parameters. // - objects returned by some functions @@ -4577,11 +4577,12 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter( #[cfg(feature = "jsonrpc")] mod jsonrpc { - use super::*; use deltachat_jsonrpc::api::CommandApi; use deltachat_jsonrpc::events::event_to_json_rpc_notification; use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession}; + use super::*; + pub struct dc_jsonrpc_instance_t { receiver: OutReceiver, handle: RpcSession, diff --git a/deltachat-ffi/src/lot.rs b/deltachat-ffi/src/lot.rs index 0f3dadc3b..1e6a7831a 100644 --- a/deltachat-ffi/src/lot.rs +++ b/deltachat-ffi/src/lot.rs @@ -1,10 +1,12 @@ //! # Legacy generic return values for C API. +use std::borrow::Cow; + +use anyhow::Error; + use crate::message::MessageState; use crate::qr::Qr; use crate::summary::{Summary, SummaryPrefix}; -use anyhow::Error; -use std::borrow::Cow; /// An object containing a set of values. /// The meaning of the values is defined by the function returning the object. diff --git a/deltachat-ffi/src/string.rs b/deltachat-ffi/src/string.rs index 65c091108..296be1421 100644 --- a/deltachat-ffi/src/string.rs +++ b/deltachat-ffi/src/string.rs @@ -287,9 +287,10 @@ fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path { #[cfg(test)] mod tests { - use super::*; use libc::{free, strcmp}; + use super::*; + #[test] fn test_os_str_to_c_string_cwd() { let some_dir = std::env::current_dir().unwrap(); diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index a311f8f07..d40a08a19 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1,4 +1,9 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::{collections::HashMap, str::FromStr}; + use anyhow::{anyhow, bail, ensure, Context, Result}; +pub use deltachat::accounts::Accounts; use deltachat::{ chat::{ self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, marknoticed_chat, @@ -23,21 +28,14 @@ use deltachat::{ webxdc::StatusUpdateSerial, }; use sanitize_filename::is_sanitized; -use std::collections::BTreeMap; -use std::sync::Arc; -use std::{collections::HashMap, str::FromStr}; use tokio::{fs, sync::RwLock}; use walkdir::WalkDir; use yerpc::rpc; -pub use deltachat::accounts::Accounts; - pub mod events; pub mod types; -use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult}; -use crate::api::types::qr::QrObject; - +use num_traits::FromPrimitive; use types::account::Account; use types::chat::FullChat; use types::chat_list::ChatListEntry; @@ -53,8 +51,8 @@ use self::types::{ JSONRPCMessageListItem, MessageNotificationInfo, MessageSearchResult, MessageViewtype, }, }; - -use num_traits::FromPrimitive; +use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult}; +use crate::api::types::qr::QrObject; #[derive(Clone, Debug)] pub struct CommandApi { diff --git a/deltachat-jsonrpc/src/lib.rs b/deltachat-jsonrpc/src/lib.rs index 344e91afb..17429d8de 100644 --- a/deltachat-jsonrpc/src/lib.rs +++ b/deltachat-jsonrpc/src/lib.rs @@ -4,12 +4,13 @@ pub use yerpc; #[cfg(test)] mod tests { - use super::api::{Accounts, CommandApi}; use async_channel::unbounded; use futures::StreamExt; use tempfile::TempDir; use yerpc::{RpcClient, RpcSession}; + use super::api::{Accounts, CommandApi}; + #[tokio::test(flavor = "multi_thread")] async fn basic_json_rpc_functionality() -> anyhow::Result<()> { let tmp_dir = TempDir::new().unwrap().path().into(); diff --git a/deltachat-jsonrpc/src/webserver.rs b/deltachat-jsonrpc/src/webserver.rs index c2f770be7..9231069c5 100644 --- a/deltachat-jsonrpc/src/webserver.rs +++ b/deltachat-jsonrpc/src/webserver.rs @@ -1,6 +1,7 @@ -use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router}; use std::net::SocketAddr; use std::path::PathBuf; + +use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router}; use yerpc::axum::handle_ws_rpc; use yerpc::{RpcClient, RpcSession}; diff --git a/deltachat_derive/src/lib.rs b/deltachat_derive/src/lib.rs index e24c5cf39..4fed46d5d 100644 --- a/deltachat_derive/src/lib.rs +++ b/deltachat_derive/src/lib.rs @@ -1,9 +1,10 @@ #![recursion_limit = "128"] extern crate proc_macro; -use crate::proc_macro::TokenStream; use quote::quote; +use crate::proc_macro::TokenStream; + // For now, assume (not check) that these macroses are applied to enum without // data. If this assumption is violated, compiler error will point to // generated code, which is not very user-friendly. diff --git a/examples/simple.rs b/examples/simple.rs index d14b1ac48..0dd754448 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -1,5 +1,3 @@ -use tempfile::tempdir; - use deltachat::chat::{self, ChatId}; use deltachat::chatlist::*; use deltachat::config; @@ -8,6 +6,7 @@ use deltachat::context::*; use deltachat::message::Message; use deltachat::stock_str::StockStrings; use deltachat::{EventType, Events}; +use tempfile::tempdir; fn cb(event: EventType) { match event { diff --git a/src/accounts.rs b/src/accounts.rs index 8de0a7da8..7b01de94e 100644 --- a/src/accounts.rs +++ b/src/accounts.rs @@ -509,7 +509,6 @@ impl AccountConfig { #[cfg(test)] mod tests { use super::*; - use crate::stock_str::{self, StockMessage}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/aheader.rs b/src/aheader.rs index ac9a54f60..5e4268e6f 100644 --- a/src/aheader.rs +++ b/src/aheader.rs @@ -2,11 +2,12 @@ //! //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header). -use anyhow::{bail, Context as _, Error, Result}; use std::collections::BTreeMap; use std::fmt; use std::str::FromStr; +use anyhow::{bail, Context as _, Error, Result}; + use crate::key::{DcKey, SignedPublicKey}; /// Possible values for encryption preference diff --git a/src/authres.rs b/src/authres.rs index d2b67f46a..eb267ddc9 100644 --- a/src/authres.rs +++ b/src/authres.rs @@ -355,7 +355,6 @@ mod tests { use tokio::io::AsyncReadExt; use super::*; - use crate::aheader::EncryptPreference; use crate::e2ee; use crate::message; diff --git a/src/blob.rs b/src/blob.rs index 7b6109f2e..5de6a44b2 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -499,17 +499,15 @@ fn encoded_img_exceeds_bytes( #[cfg(test)] mod tests { - use fs::File; - use anyhow::Result; + use fs::File; use image::{GenericImageView, Pixel}; + use super::*; use crate::chat::{self, create_group_chat, ProtectionStatus}; use crate::message::Message; use crate::test_utils::{self, TestContext}; - use super::*; - fn check_image_size(path: impl AsRef, width: u32, height: u32) -> image::DynamicImage { tokio::task::block_in_place(move || { let img = image::open(path).expect("failed to open image"); diff --git a/src/chat.rs b/src/chat.rs index 5a9bfde64..384884075 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -3675,12 +3675,10 @@ pub(crate) async fn update_msg_text_and_timestamp( #[cfg(test)] mod tests { use super::*; - use crate::chatlist::{get_archived_cnt, Chatlist}; use crate::constants::{DC_GCL_ARCHIVED_ONLY, DC_GCL_NO_SPECIALS}; use crate::contact::{Contact, ContactAddress}; use crate::receive_imf::receive_imf; - use crate::test_utils::TestContext; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/chatlist.rs b/src/chatlist.rs index df80b15e2..e20025de9 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -365,7 +365,6 @@ pub async fn get_archived_cnt(context: &Context) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus}; use crate::message::Viewtype; use crate::receive_imf::receive_imf; diff --git a/src/config.rs b/src/config.rs index 41b884fab..0bca875f1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -440,16 +440,15 @@ fn get_config_keys_string() -> String { #[cfg(test)] mod tests { - use super::*; - use std::str::FromStr; use std::string::ToString; + use num_traits::FromPrimitive; + + use super::*; use crate::constants; use crate::test_utils::TestContext; - use num_traits::FromPrimitive; - #[test] fn test_to_string() { assert_eq!(Config::MailServer.to_string(), "mail_server"); diff --git a/src/configure.rs b/src/configure.rs index 493980233..064b9c8e0 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -6,9 +6,12 @@ mod read_url; mod server_params; use anyhow::{bail, ensure, Context as _, Result}; +use auto_mozilla::moz_autoconfigure; +use auto_outlook::outlk_autodiscover; use futures::FutureExt; use futures_lite::FutureExt as _; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; +use server_params::{expand_param_vector, ServerParams}; use tokio::task; use crate::config::Config; @@ -28,10 +31,6 @@ use crate::stock_str; use crate::tools::{time, EmailAddress}; use crate::{chat, e2ee, provider}; -use auto_mozilla::moz_autoconfigure; -use auto_outlook::outlk_autodiscover; -use server_params::{expand_param_vector, ServerParams}; - macro_rules! progress { ($context:tt, $progress:expr, $comment:expr) => { assert!( diff --git a/src/configure/auto_mozilla.rs b/src/configure/auto_mozilla.rs index 2f5ca45b2..d53b5a8c5 100644 --- a/src/configure/auto_mozilla.rs +++ b/src/configure/auto_mozilla.rs @@ -1,17 +1,16 @@ //! # Thunderbird's Autoconfiguration implementation //! //! Documentation: -use quick_xml::events::{BytesStart, Event}; - use std::io::BufRead; use std::str::FromStr; -use crate::context::Context; -use crate::login_param::LoginParam; -use crate::provider::{Protocol, Socket}; +use quick_xml::events::{BytesStart, Event}; use super::read_url::read_url; use super::{Error, ServerParams}; +use crate::context::Context; +use crate::login_param::LoginParam; +use crate::provider::{Protocol, Socket}; #[derive(Debug)] struct Server { diff --git a/src/configure/auto_outlook.rs b/src/configure/auto_outlook.rs index d374f7016..8d42fd353 100644 --- a/src/configure/auto_outlook.rs +++ b/src/configure/auto_outlook.rs @@ -3,15 +3,14 @@ //! This module implements autoconfiguration via POX (Plain Old XML) interface to Autodiscover //! Service. Newer SOAP interface, introduced in Exchange 2010, is not used. -use quick_xml::events::Event; - use std::io::BufRead; -use crate::context::Context; -use crate::provider::{Protocol, Socket}; +use quick_xml::events::Event; use super::read_url::read_url; use super::{Error, ServerParams}; +use crate::context::Context; +use crate::provider::{Protocol, Socket}; /// Result of parsing a single `Protocol` tag. /// diff --git a/src/contact.rs b/src/contact.rs index 056797de0..a1e546c67 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -1656,7 +1656,6 @@ impl RecentlySeenLoop { #[cfg(test)] mod tests { use super::*; - use crate::chat::{get_chat_contacts, send_text_msg, Chat}; use crate::chatlist::Chatlist; use crate::receive_imf::receive_imf; diff --git a/src/context.rs b/src/context.rs index c297ac409..329667ddf 100644 --- a/src/context.rs +++ b/src/context.rs @@ -861,8 +861,13 @@ pub fn get_version_str() -> &'static str { #[cfg(test)] mod tests { - use super::*; + use std::time::Duration; + use anyhow::Context as _; + use strum::IntoEnumIterator; + use tempfile::tempdir; + + use super::*; use crate::chat::{ get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration, }; @@ -873,10 +878,6 @@ mod tests { use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; use crate::tools::create_outgoing_rfc724_mid; - use anyhow::Context as _; - use std::time::Duration; - use strum::IntoEnumIterator; - use tempfile::tempdir; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_wrong_db() -> Result<()> { diff --git a/src/decrypt.rs b/src/decrypt.rs index b7f831ab0..1165e3dd7 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -344,11 +344,10 @@ pub(crate) async fn get_autocrypt_peerstate( #[cfg(test)] mod tests { + use super::*; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; - use super::*; - #[test] fn test_has_decrypted_pgp_armor() { let data = b" -----BEGIN PGP MESSAGE-----"; diff --git a/src/download.rs b/src/download.rs index 015eb56f1..cd08c01dd 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,9 +1,11 @@ //! # Download large messages manually. +use std::cmp::max; +use std::collections::BTreeMap; + use anyhow::{anyhow, Result}; use deltachat_derive::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; use crate::config::Config; use crate::context::Context; @@ -14,7 +16,6 @@ use crate::mimeparser::{MimeMessage, Part}; use crate::param::Params; use crate::tools::time; use crate::{job_try, stock_str, EventType}; -use std::cmp::max; /// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`. /// @@ -264,14 +265,13 @@ impl MimeMessage { mod tests { use num_traits::FromPrimitive; + use super::*; use crate::chat::{get_chat_msgs, send_msg}; use crate::ephemeral::Timer; use crate::message::Viewtype; use crate::receive_imf::receive_imf_inner; use crate::test_utils::TestContext; - use super::*; - #[test] fn test_downloadstate_values() { // values may be written to disk and must not change diff --git a/src/e2ee.rs b/src/e2ee.rs index 62f699fdf..f145373e4 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -144,13 +144,12 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result { #[cfg(test)] mod tests { + use super::*; use crate::chat; use crate::message::{Message, Viewtype}; use crate::param::Param; use crate::test_utils::{bob_keypair, TestContext}; - use super::*; - mod ensure_secret_key_exists { use super::*; diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 1118ef077..0e8dfdf7a 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -64,6 +64,7 @@ #![allow(missing_docs)] +use std::cmp::max; use std::convert::{TryFrom, TryInto}; use std::num::ParseIntError; use std::str::FromStr; @@ -86,7 +87,6 @@ use crate::mimeparser::SystemMessage; use crate::sql::{self, params_iter}; use crate::stock_str; use crate::tools::{duration_to_str, time}; -use std::cmp::max; #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)] pub enum Timer { diff --git a/src/html.rs b/src/html.rs index 279995292..7e963e16b 100644 --- a/src/html.rs +++ b/src/html.rs @@ -7,12 +7,14 @@ //! `MsgId.get_html()` will return HTML - //! this allows nice quoting, handling linebreaks properly etc. -use futures::future::FutureExt; use std::future::Future; use std::pin::Pin; use anyhow::{Context as _, Result}; +use futures::future::FutureExt; use lettre_email::mime::{self, Mime}; +use lettre_email::PartBuilder; +use mailparse::ParsedContentType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::message::{Message, MsgId}; @@ -20,8 +22,6 @@ use crate::mimeparser::parse_message_id; use crate::param::Param::SendHtml; use crate::plaintext::PlainText; use crate::{context::Context, message}; -use lettre_email::PartBuilder; -use mailparse::ParsedContentType; impl Message { /// Check if the message can be retrieved as HTML. diff --git a/src/http.rs b/src/http.rs index 4f0c3fb35..17e73a10d 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,8 +1,9 @@ //! # HTTP module. -use anyhow::Result; use std::time::Duration; +use anyhow::Result; + const HTTP_TIMEOUT: Duration = Duration::from_secs(30); pub(crate) fn get_client() -> Result { diff --git a/src/imap/client.rs b/src/imap/client.rs index f4eacd6dc..aaada0f90 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -4,21 +4,18 @@ use std::{ }; use anyhow::{Context as _, Result}; - use async_imap::Client as ImapClient; use async_imap::Session as ImapSession; - use tokio::io::BufWriter; use tokio::net::ToSocketAddrs; use super::capabilities::Capabilities; use super::session::Session; +use super::session::SessionStream; use crate::login_param::build_tls; use crate::net::connect_tcp; use crate::socks::Socks5Config; -use super::session::SessionStream; - /// IMAP write and read timeout in seconds. pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(30); diff --git a/src/imap/idle.rs b/src/imap/idle.rs index 78232982e..bcc605c63 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -1,12 +1,12 @@ -use super::Imap; +use std::time::{Duration, SystemTime}; use anyhow::{bail, Context as _, Result}; use async_channel::Receiver; use async_imap::extensions::idle::IdleResponse; use futures_lite::FutureExt; -use std::time::{Duration, SystemTime}; use super::session::Session; +use super::Imap; use crate::imap::client::IMAP_TIMEOUT; use crate::{context::Context, scheduler::InterruptInfo}; diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs index d7c732244..991b6b38f 100644 --- a/src/imap/scan_folders.rs +++ b/src/imap/scan_folders.rs @@ -3,13 +3,12 @@ use std::{collections::BTreeMap, time::Instant}; use anyhow::{Context as _, Result}; use futures::stream::StreamExt; +use super::{get_folder_meaning, get_folder_meaning_by_name}; use crate::config::Config; use crate::imap::Imap; use crate::log::LogExt; use crate::{context::Context, imap::FolderMeaning}; -use super::{get_folder_meaning, get_folder_meaning_by_name}; - impl Imap { /// Returns true if folders were scanned, false if scanning was postponed. pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result { diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index facf04ffc..0d12f0238 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -1,8 +1,8 @@ -use super::session::Session as ImapSession; - -use crate::context::Context; use anyhow::Context as _; +use super::session::Session as ImapSession; +use crate::context::Context; + type Result = std::result::Result; #[derive(Debug, thiserror::Error)] diff --git a/src/imex.rs b/src/imex.rs index e265e2f6b..e77f7baa3 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -769,14 +769,13 @@ where #[cfg(test)] mod tests { - use super::*; + use ::pgp::armor::BlockType; + use super::*; use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE}; use crate::stock_str::StockMessage; use crate::test_utils::{alice_keypair, TestContext}; - use ::pgp::armor::BlockType; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_render_setup_file() { let t = TestContext::new_alice().await; diff --git a/src/job.rs b/src/job.rs index 689a7cade..7f5528ca9 100644 --- a/src/job.rs +++ b/src/job.rs @@ -424,7 +424,6 @@ LIMIT 1; #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestContext; async fn insert_job(context: &Context, foreign_id: i64, valid: bool) { diff --git a/src/key.rs b/src/key.rs index 82ea25667..73967286c 100644 --- a/src/key.rs +++ b/src/key.rs @@ -11,6 +11,7 @@ use anyhow::{ensure, Context as _, Result}; use futures::Future; use num_traits::FromPrimitive; use pgp::composed::Deserializable; +pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; use pgp::ser::Serialize; use pgp::types::{KeyTrait, SecretKeyTrait}; use tokio::runtime::Handle; @@ -18,11 +19,9 @@ use tokio::runtime::Handle; use crate::config::Config; use crate::constants::KeyGenType; use crate::context::Context; -use crate::tools::{time, EmailAddress}; - // Re-export key types pub use crate::pgp::KeyPair; -pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; +use crate::tools::{time, EmailAddress}; /// Convenience trait for working with keys. /// @@ -390,11 +389,12 @@ impl std::str::FromStr for Fingerprint { #[cfg(test)] mod tests { - use super::*; - use crate::test_utils::{alice_keypair, TestContext}; + use std::sync::Arc; use once_cell::sync::Lazy; - use std::sync::Arc; + + use super::*; + use crate::test_utils::{alice_keypair, TestContext}; static KEYPAIR: Lazy = Lazy::new(alice_keypair); diff --git a/src/log.rs b/src/log.rs index 51ada7480..c755484e6 100644 --- a/src/log.rs +++ b/src/log.rs @@ -155,9 +155,10 @@ impl LogExt for Result { #[cfg(test)] mod tests { - use crate::test_utils::TestContext; use anyhow::Result; + use crate::test_utils::TestContext; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_last_error() -> Result<()> { let t = TestContext::new().await; diff --git a/src/login_param.rs b/src/login_param.rs index feb2e4e73..bd5541047 100644 --- a/src/login_param.rs +++ b/src/login_param.rs @@ -332,7 +332,6 @@ pub fn build_tls(strict_tls: bool) -> async_native_tls::TlsConnector { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestContext; #[test] diff --git a/src/message.rs b/src/message.rs index 48819a218..0bde53961 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1982,14 +1982,13 @@ impl Viewtype { mod tests { use num_traits::FromPrimitive; + use super::*; use crate::chat::{marknoticed_chat, ChatItem}; use crate::chatlist::Chatlist; use crate::receive_imf::receive_imf; use crate::test_utils as test; use crate::test_utils::{TestContext, TestContextManager}; - use super::*; - #[test] fn test_guess_msgtype_from_suffix() { assert_eq!( diff --git a/src/mimefactory.rs b/src/mimefactory.rs index bd198329d..14e6563d2 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1508,6 +1508,7 @@ fn maybe_encode_words(words: &str) -> String { mod tests { use mailparse::{addrparse_header, MailHeaderMap}; + use super::*; use crate::chat::ChatId; use crate::chat::{ self, add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg, @@ -1518,8 +1519,6 @@ mod tests { use crate::mimeparser::MimeMessage; use crate::receive_imf::receive_imf; use crate::test_utils::{get_chat_msg, TestContext}; - - use super::*; #[test] fn test_render_email_address() { let display_name = "ä space"; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 34318a143..6a97e8c91 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -1953,6 +1953,8 @@ where mod tests { #![allow(clippy::indexing_slicing)] + use mailparse::ParsedMail; + use super::*; use crate::{ chatlist::Chatlist, @@ -1962,7 +1964,6 @@ mod tests { receive_imf::receive_imf, test_utils::TestContext, }; - use mailparse::ParsedMail; impl AvatarAction { pub fn is_change(&self) -> bool { diff --git a/src/oauth2.rs b/src/oauth2.rs index ab0b4eef5..bde9f2240 100644 --- a/src/oauth2.rs +++ b/src/oauth2.rs @@ -352,7 +352,6 @@ fn normalize_addr(addr: &str) -> &str { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestContext; #[test] diff --git a/src/param.rs b/src/param.rs index 69e587e18..55bf5f930 100644 --- a/src/param.rs +++ b/src/param.rs @@ -435,14 +435,13 @@ impl<'a> ParamsFile<'a> { #[cfg(test)] mod tests { - use super::*; - use std::path::Path; use std::str::FromStr; use anyhow::Result; use tokio::fs; + use super::*; use crate::test_utils::TestContext; #[test] diff --git a/src/peerstate.rs b/src/peerstate.rs index 0c54d80c5..c3f3b1a68 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -4,6 +4,9 @@ use std::collections::HashSet; +use anyhow::{Context as _, Error, Result}; +use num_traits::FromPrimitive; + use crate::aheader::{Aheader, EncryptPreference}; use crate::chat::{self, Chat}; use crate::chatlist::Chatlist; @@ -16,8 +19,6 @@ use crate::message::Message; use crate::mimeparser::SystemMessage; use crate::sql::Sql; use crate::stock_str; -use anyhow::{Context as _, Error, Result}; -use num_traits::FromPrimitive; #[derive(Debug)] pub enum PeerstateKeyType { diff --git a/src/pgp.rs b/src/pgp.rs index 93762e68a..cacdecae0 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -382,11 +382,12 @@ pub async fn symm_decrypt( #[cfg(test)] mod tests { - use super::*; - use crate::test_utils::{alice_keypair, bob_keypair}; use once_cell::sync::Lazy; use tokio::sync::OnceCell; + use super::*; + use crate::test_utils::{alice_keypair, bob_keypair}; + #[test] fn test_split_armored_data_1() { let (typ, _headers, base64) = split_armored_data( diff --git a/src/plaintext.rs b/src/plaintext.rs index 5bdeffed4..13dd1ac7c 100644 --- a/src/plaintext.rs +++ b/src/plaintext.rs @@ -2,9 +2,10 @@ #![allow(missing_docs)] -use crate::simplify::split_lines; use once_cell::sync::Lazy; +use crate::simplify::split_lines; + #[derive(Debug)] pub struct PlainText { pub text: String, diff --git a/src/provider.rs b/src/provider.rs index 4d8ee74d8..5d81b642b 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -4,13 +4,14 @@ mod data; -use crate::config::Config; -use crate::context::Context; -use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED}; use anyhow::Result; use chrono::{NaiveDateTime, NaiveTime}; use trust_dns_resolver::{config, AsyncResolver, TokioAsyncResolver}; +use crate::config::Config; +use crate::context::Context; +use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED}; + #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)] #[repr(u8)] pub enum Status { @@ -195,10 +196,11 @@ pub fn get_provider_update_timestamp() -> i64 { mod tests { #![allow(clippy::indexing_slicing)] + use chrono::NaiveDate; + use super::*; use crate::test_utils::TestContext; use crate::tools::time; - use chrono::NaiveDate; #[test] fn test_get_provider_by_domain_unexistant() { diff --git a/src/provider/data.rs b/src/provider/data.rs index 8c9516272..21084f821 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -1,12 +1,13 @@ // file generated by src/provider/update.py +use std::collections::HashMap; + +use once_cell::sync::Lazy; + use crate::provider::Protocol::*; use crate::provider::Socket::*; use crate::provider::UsernamePattern::*; use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status}; -use std::collections::HashMap; - -use once_cell::sync::Lazy; // 163.md: 163.com static P_163: Lazy = Lazy::new(|| Provider { diff --git a/src/qr.rs b/src/qr.rs index c6b5e18b7..7c14c4de0 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -3,14 +3,15 @@ #![allow(missing_docs)] mod dclogin_scheme; -pub use dclogin_scheme::LoginOptions; +use std::collections::BTreeMap; use anyhow::{anyhow, bail, ensure, Context as _, Error, Result}; +pub use dclogin_scheme::LoginOptions; use once_cell::sync::Lazy; use percent_encoding::percent_decode_str; use serde::Deserialize; -use std::collections::BTreeMap; +use self::dclogin_scheme::configure_from_login_qr; use crate::chat::{self, get_chat_id_by_grpid, ChatIdBlocked}; use crate::config::Config; use crate::constants::Blocked; @@ -24,8 +25,6 @@ use crate::peerstate::Peerstate; use crate::tools::time; use crate::{token, EventType}; -use self::dclogin_scheme::configure_from_login_qr; - const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase const DCACCOUNT_SCHEME: &str = "DCACCOUNT:"; pub(super) const DCLOGIN_SCHEME: &str = "DCLOGIN:"; @@ -642,14 +641,14 @@ fn normalize_address(addr: &str) -> Result { #[cfg(test)] mod tests { - use super::*; + use anyhow::Result; + use super::*; use crate::aheader::EncryptPreference; use crate::chat::{create_group_chat, ProtectionStatus}; use crate::key::DcKey; use crate::securejoin::get_securejoin_qr; use crate::test_utils::{alice_keypair, TestContext}; - use anyhow::Result; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_decode_http() -> Result<()> { diff --git a/src/qr/dclogin_scheme.rs b/src/qr/dclogin_scheme.rs index c7e9e8a76..c7b7613ee 100644 --- a/src/qr/dclogin_scheme.rs +++ b/src/qr/dclogin_scheme.rs @@ -1,13 +1,13 @@ use std::collections::HashMap; -use crate::config::Config; -use crate::context::Context; -use crate::provider::Socket; -use crate::{contact, login_param::CertificateChecks}; use anyhow::{bail, Context as _, Result}; use num_traits::cast::ToPrimitive; use super::{Qr, DCLOGIN_SCHEME}; +use crate::config::Config; +use crate::context::Context; +use crate::provider::Socket; +use crate::{contact, login_param::CertificateChecks}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum LoginOptions { @@ -221,9 +221,10 @@ pub(crate) async fn configure_from_login_qr( #[cfg(test)] mod test { + use anyhow::{self, bail}; + use super::{decode_login, LoginOptions}; use crate::{login_param::CertificateChecks, provider::Socket, qr::Qr}; - use anyhow::{self, bail}; macro_rules! login_options_just_pw { ($pw: expr) => { diff --git a/src/quota.rs b/src/quota.rs index 9b233a349..2d71ca13f 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -2,9 +2,10 @@ #![allow(missing_docs)] +use std::collections::BTreeMap; + use anyhow::{anyhow, Context as _, Result}; use async_imap::types::{Quota, QuotaResource}; -use std::collections::BTreeMap; use crate::chat::add_device_msg_with_importance; use crate::config::Config; diff --git a/src/reaction.rs b/src/reaction.rs index f858f6172..fdd193ccd 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -286,7 +286,6 @@ pub async fn get_msg_reactions(context: &Context, msg_id: MsgId) -> Result bool { #[cfg(test)] mod tests { - use super::*; use proptest::prelude::*; + use super::*; + proptest! { #[test] // proptest does not support [[:graphical:][:space:]] regex. diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 6f09d1346..6a4c48f9e 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -1,12 +1,13 @@ //! # SMTP message sending -use super::Smtp; +use std::time::Duration; + use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport}; +use super::Smtp; use crate::config::Config; use crate::context::Context; use crate::events::EventType; -use std::time::Duration; pub type Result = std::result::Result; diff --git a/src/socks.rs b/src/socks.rs index 4dd1ffcc5..b05c265dc 100644 --- a/src/socks.rs +++ b/src/socks.rs @@ -4,17 +4,17 @@ use std::fmt; use std::pin::Pin; use std::time::Duration; -use crate::net::connect_tcp; use anyhow::Result; pub use async_smtp::ServerAddress; -use tokio::net::TcpStream; -use tokio_io_timeout::TimeoutStream; - -use crate::context::Context; use fast_socks5::client::{Config, Socks5Stream}; use fast_socks5::util::target_addr::ToTargetAddr; use fast_socks5::AuthenticationMethod; use fast_socks5::Socks5Command; +use tokio::net::TcpStream; +use tokio_io_timeout::TimeoutStream; + +use crate::context::Context; +use crate::net::connect_tcp; #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct Socks5Config { diff --git a/src/sql.rs b/src/sql.rs index 1203a276e..778a79107 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -874,11 +874,10 @@ pub fn repeat_vars(count: usize) -> String { mod tests { use async_channel as channel; + use super::*; use crate::config::Config; use crate::{test_utils::TestContext, EventType}; - use super::*; - #[test] fn test_maybe_add_file() { let mut files = Default::default(); diff --git a/src/stock_str.rs b/src/stock_str.rs index 2e93962d6..aa7ac2903 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::sync::Arc; use anyhow::{bail, Result}; +use humansize::{format_size, BINARY}; use strum::EnumProperty as EnumPropertyTrait; use strum_macros::EnumProperty; use tokio::sync::RwLock; @@ -19,7 +20,6 @@ use crate::context::Context; use crate::message::{Message, Viewtype}; use crate::param::Param; use crate::tools::timestamp_to_str; -use humansize::{format_size, BINARY}; #[derive(Debug, Clone)] pub struct StockStrings { @@ -1308,13 +1308,12 @@ impl Accounts { mod tests { use num_traits::ToPrimitive; + use super::*; use crate::chat::delete_and_reset_all_device_msgs; use crate::chat::Chat; use crate::chatlist::Chatlist; use crate::test_utils::TestContext; - use super::*; - #[test] fn test_enum_mapping() { assert_eq!(StockMessage::NoMessages.to_usize().unwrap(), 1); diff --git a/src/summary.rs b/src/summary.rs index 5b4d3f0bc..5fc91c61e 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -1,5 +1,8 @@ //! # Message summary for chatlist. +use std::borrow::Cow; +use std::fmt; + use crate::chat::Chat; use crate::constants::Chattype; use crate::contact::{Contact, ContactId}; @@ -9,8 +12,6 @@ use crate::mimeparser::SystemMessage; use crate::param::Param; use crate::stock_str; use crate::tools::truncate; -use std::borrow::Cow; -use std::fmt; /// Prefix displayed before message and separated by ":" in the chatlist. #[derive(Debug)] diff --git a/src/sync.rs b/src/sync.rs index 041980ab0..b3835e499 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,10 @@ //! # Synchronize items between devices. +use anyhow::Result; +use lettre_email::mime::{self}; +use lettre_email::PartBuilder; +use serde::{Deserialize, Serialize}; + use crate::chat::{Chat, ChatId}; use crate::config::Config; use crate::constants::Blocked; @@ -12,10 +17,6 @@ use crate::sync::SyncData::{AddQrToken, DeleteQrToken}; use crate::token::Namespace; use crate::tools::time; use crate::{chat, stock_str, token}; -use anyhow::Result; -use lettre_email::mime::{self}; -use lettre_email::PartBuilder; -use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] pub(crate) struct QrTokenData { @@ -260,12 +261,13 @@ impl Context { #[cfg(test)] mod tests { + use anyhow::bail; + use super::*; use crate::chat::Chat; use crate::chatlist::Chatlist; use crate::test_utils::TestContext; use crate::token::Namespace; - use anyhow::bail; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_is_sync_sending_enabled() -> Result<()> { diff --git a/src/tools.rs b/src/tools.rs index e9c34f5e1..ca935cf1e 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -9,7 +9,6 @@ use std::fmt; use std::io::Cursor; use std::path::{Path, PathBuf}; use std::str::from_utf8; - use std::time::{Duration, SystemTime}; use anyhow::{bail, Error, Result}; @@ -703,7 +702,6 @@ mod tests { #![allow(clippy::indexing_slicing)] use super::*; - use crate::{ config::Config, message::get_msg_info, receive_imf::receive_imf, test_utils::TestContext, }; @@ -1008,11 +1006,12 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; assert_eq!(EmailAddress::new("@d.tt").is_ok(), false); } - use crate::chatlist::Chatlist; - use crate::{chat, test_utils}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use proptest::prelude::*; + use crate::chatlist::Chatlist; + use crate::{chat, test_utils}; + proptest! { #[test] fn test_truncate( diff --git a/src/update_helper.rs b/src/update_helper.rs index bfe46d500..9136b23df 100644 --- a/src/update_helper.rs +++ b/src/update_helper.rs @@ -1,10 +1,11 @@ //! # Functions to update timestamps. +use anyhow::Result; + use crate::chat::{Chat, ChatId}; use crate::contact::{Contact, ContactId}; use crate::context::Context; use crate::param::{Param, Params}; -use anyhow::Result; impl Context { /// Updates a contact's timestamp, if reasonable. diff --git a/src/webxdc.rs b/src/webxdc.rs index 000da4fb6..992597736 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -770,6 +770,7 @@ impl Message { #[cfg(test)] mod tests { + use super::*; use crate::chat::{ add_contact_to_chat, create_broadcast_list, create_group_chat, forward_msgs, remove_contact_from_chat, resend_msgs, send_msg, send_text_msg, ChatId, ProtectionStatus, @@ -781,8 +782,6 @@ mod tests { use crate::receive_imf::{receive_imf, receive_imf_inner}; use crate::test_utils::TestContext; - use super::*; - #[allow(clippy::assertions_on_constants)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_webxdc_file_limits() -> Result<()> { From ed20a23297a86624cc72b22ccbb1db32a2433c43 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 14:57:25 +0000 Subject: [PATCH 64/77] Resolve IP addresses explicitly --- src/imap.rs | 17 ++++++++++++----- src/imap/client.rs | 39 +++++++++++++++++++++++++++++---------- src/net.rs | 42 ++++++++++++++++++++++++++++++++++++------ src/socks.rs | 3 ++- 4 files changed, 79 insertions(+), 22 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index 0d91da8f7..5eaf0fd6a 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -308,6 +308,7 @@ impl Imap { if let Some(socks5_config) = &config.socks5_config { if config.lp.security == Socket::Starttls { Client::connect_starttls_socks5( + context, imap_server, imap_port, socks5_config.clone(), @@ -315,13 +316,18 @@ impl Imap { ) .await } else { - Client::connect_insecure_socks5(imap_server, imap_port, socks5_config.clone()) - .await + Client::connect_insecure_socks5( + context, + imap_server, + imap_port, + socks5_config.clone(), + ) + .await } } else if config.lp.security == Socket::Starttls { - Client::connect_starttls(imap_server, imap_port, config.strict_tls).await + Client::connect_starttls(context, imap_server, imap_port, config.strict_tls).await } else { - Client::connect_insecure((imap_server, imap_port)).await + Client::connect_insecure(context, imap_server, imap_port).await } } else { let config = &self.config; @@ -330,6 +336,7 @@ impl Imap { if let Some(socks5_config) = &config.socks5_config { Client::connect_secure_socks5( + context, imap_server, imap_port, config.strict_tls, @@ -337,7 +344,7 @@ impl Imap { ) .await } else { - Client::connect_secure(imap_server, imap_port, config.strict_tls).await + Client::connect_secure(context, imap_server, imap_port, config.strict_tls).await } }; diff --git a/src/imap/client.rs b/src/imap/client.rs index aaada0f90..b6b106c45 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -7,11 +7,11 @@ use anyhow::{Context as _, Result}; use async_imap::Client as ImapClient; use async_imap::Session as ImapSession; use tokio::io::BufWriter; -use tokio::net::ToSocketAddrs; use super::capabilities::Capabilities; use super::session::Session; use super::session::SessionStream; +use crate::context::Context; use crate::login_param::build_tls; use crate::net::connect_tcp; use crate::socks::Socks5Config; @@ -88,8 +88,13 @@ impl Client { Ok(Session::new(session, capabilities)) } - pub async fn connect_secure(hostname: &str, port: u16, strict_tls: bool) -> Result { - let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?; + pub async fn connect_secure( + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + ) -> Result { + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT).await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); @@ -104,8 +109,8 @@ impl Client { Ok(Client { inner: client }) } - pub async fn connect_insecure(addr: impl ToSocketAddrs) -> Result { - let tcp_stream = connect_tcp(addr, IMAP_TIMEOUT).await?; + pub async fn connect_insecure(context: &Context, hostname: &str, port: u16) -> Result { + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT).await?; let buffered_stream = BufWriter::new(tcp_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -117,8 +122,13 @@ impl Client { Ok(Client { inner: client }) } - pub async fn connect_starttls(hostname: &str, port: u16, strict_tls: bool) -> Result { - let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?; + pub async fn connect_starttls( + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + ) -> Result { + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT).await?; // Run STARTTLS command and convert the client back into a stream. let mut client = ImapClient::new(tcp_stream); @@ -146,12 +156,15 @@ impl Client { } pub async fn connect_secure_socks5( + context: &Context, domain: &str, port: u16, strict_tls: bool, socks5_config: Socks5Config, ) -> Result { - let socks5_stream = socks5_config.connect(domain, port, IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config + .connect(context, domain, port, IMAP_TIMEOUT) + .await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(domain, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); @@ -166,11 +179,14 @@ impl Client { } pub async fn connect_insecure_socks5( + context: &Context, domain: &str, port: u16, socks5_config: Socks5Config, ) -> Result { - let socks5_stream = socks5_config.connect(domain, port, IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config + .connect(context, domain, port, IMAP_TIMEOUT) + .await?; let buffered_stream = BufWriter::new(socks5_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -183,12 +199,15 @@ impl Client { } pub async fn connect_starttls_socks5( + context: &Context, hostname: &str, port: u16, socks5_config: Socks5Config, strict_tls: bool, ) -> Result { - let socks5_stream = socks5_config.connect(hostname, port, IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config + .connect(context, hostname, port, IMAP_TIMEOUT) + .await?; // Run STARTTLS command and convert the client back into a stream. let mut client = ImapClient::new(socks5_stream); diff --git a/src/net.rs b/src/net.rs index 6c0c7dd0b..2885c0523 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,25 +1,55 @@ ///! # Common network utilities. +use std::net::SocketAddr; use std::pin::Pin; use std::time::Duration; use anyhow::{Context as _, Result}; -use tokio::net::{TcpStream, ToSocketAddrs}; +use tokio::net::{lookup_host, TcpStream}; use tokio::time::timeout; use tokio_io_timeout::TimeoutStream; +use crate::context::Context; + +async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result { + let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) + .await + .context("connection timeout")? + .context("connection failure")?; + Ok(tcp_stream) +} + /// Returns a TCP connection stream with read/write timeouts set /// and Nagle's algorithm disabled with `TCP_NODELAY`. /// /// `TCP_NODELAY` ensures writing to the stream always results in immediate sending of the packet /// to the network, which is important to reduce the latency of interactive protocols such as IMAP. pub(crate) async fn connect_tcp( - addr: impl ToSocketAddrs, + context: &Context, + host: &str, + port: u16, timeout_val: Duration, ) -> Result>>> { - let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) - .await - .context("connection timeout")? - .context("connection failure")?; + let mut tcp_stream = None; + for resolved_addr in lookup_host((host, port)).await? { + info!( + context, + "Resolved {}:{} into {}.", host, port, &resolved_addr + ); + match connect_tcp_inner(resolved_addr, timeout_val).await { + Ok(stream) => { + tcp_stream = Some(stream); + break; + } + Err(err) => { + warn!( + context, + "Failed to connect to {}: {:#}.", resolved_addr, err + ); + } + } + } + let tcp_stream = + tcp_stream.with_context(|| format!("failed to connect to {}:{}", host, port))?; // Disable Nagle's algorithm. tcp_stream.set_nodelay(true)?; diff --git a/src/socks.rs b/src/socks.rs index b05c265dc..52eb4fd61 100644 --- a/src/socks.rs +++ b/src/socks.rs @@ -58,11 +58,12 @@ impl Socks5Config { pub async fn connect( &self, + context: &Context, target_host: &str, target_port: u16, timeout_val: Duration, ) -> Result>>>> { - let tcp_stream = connect_tcp((self.host.clone(), self.port), timeout_val).await?; + let tcp_stream = connect_tcp(context, &self.host, self.port, timeout_val).await?; let authentication_method = if let Some((username, password)) = self.user_password.as_ref() { From 773754d74f6981f0e296fd6728889b12bc89cd02 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 16:17:31 +0000 Subject: [PATCH 65/77] Introduce DNS cache table Only used for IMAP connections currently. --- src/imap/client.rs | 12 +++--- src/net.rs | 88 ++++++++++++++++++++++++++++++++++++++++--- src/socks.rs | 6 ++- src/sql/migrations.rs | 13 +++++++ 4 files changed, 107 insertions(+), 12 deletions(-) diff --git a/src/imap/client.rs b/src/imap/client.rs index b6b106c45..e371a49c2 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -94,7 +94,7 @@ impl Client { port: u16, strict_tls: bool, ) -> Result { - let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT).await?; + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); @@ -110,7 +110,7 @@ impl Client { } pub async fn connect_insecure(context: &Context, hostname: &str, port: u16) -> Result { - let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT).await?; + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, false).await?; let buffered_stream = BufWriter::new(tcp_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -128,7 +128,7 @@ impl Client { port: u16, strict_tls: bool, ) -> Result { - let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT).await?; + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?; // Run STARTTLS command and convert the client back into a stream. let mut client = ImapClient::new(tcp_stream); @@ -163,7 +163,7 @@ impl Client { socks5_config: Socks5Config, ) -> Result { let socks5_stream = socks5_config - .connect(context, domain, port, IMAP_TIMEOUT) + .connect(context, domain, port, IMAP_TIMEOUT, strict_tls) .await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(domain, socks5_stream).await?; @@ -185,7 +185,7 @@ impl Client { socks5_config: Socks5Config, ) -> Result { let socks5_stream = socks5_config - .connect(context, domain, port, IMAP_TIMEOUT) + .connect(context, domain, port, IMAP_TIMEOUT, false) .await?; let buffered_stream = BufWriter::new(socks5_stream); let session_stream: Box = Box::new(buffered_stream); @@ -206,7 +206,7 @@ impl Client { strict_tls: bool, ) -> Result { let socks5_stream = socks5_config - .connect(context, hostname, port, IMAP_TIMEOUT) + .connect(context, hostname, port, IMAP_TIMEOUT, strict_tls) .await?; // Run STARTTLS command and convert the client back into a stream. diff --git a/src/net.rs b/src/net.rs index 2885c0523..d5b6a1ec3 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,6 +1,7 @@ ///! # Common network utilities. use std::net::SocketAddr; use std::pin::Pin; +use std::str::FromStr; use std::time::Duration; use anyhow::{Context as _, Result}; @@ -9,6 +10,7 @@ use tokio::time::timeout; use tokio_io_timeout::TimeoutStream; use crate::context::Context; +use crate::tools::time; async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result { let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) @@ -18,23 +20,98 @@ async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result Result> { + let now = time(); + let mut resolved_addrs: Vec = lookup_host((hostname, port)).await?.collect(); + + for (i, addr) in resolved_addrs.iter().enumerate() { + info!(context, "Resolved {}:{} into {}.", hostname, port, &addr); + + let i = i64::try_from(i).unwrap_or_default(); + + // Update the cache. + // + // Add sequence number to the timestamp, so addresses are ordered by timestamp in the same + // order as the resolver returns them. + context + .sql + .execute( + "INSERT INTO dns_cache + (hostname, port, address, timestamp) + VALUES (?, ?, ?, ?) + ON CONFLICT (hostname, port, address) + DO UPDATE SET timestamp=excluded.timestamp", + paramsv![hostname, port, addr.to_string(), now.saturating_add(i)], + ) + .await?; + } + + if load_cache { + for cached_address in context + .sql + .query_map( + "SELECT address + FROM dns_cache + WHERE hostname = ? + AND ? < timestamp + 30 * 24 * 3600 + ORDER BY timestamp DESC", + paramsv![hostname, now], + |row| { + let address: String = row.get(0)?; + Ok(address) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await? + { + match SocketAddr::from_str(&cached_address) { + Ok(addr) => { + if !resolved_addrs.contains(&addr) { + resolved_addrs.push(addr); + } + } + Err(err) => { + warn!( + context, + "Failed to parse cached address {:?}: {:#}.", cached_address, err + ); + } + } + } + } + + Ok(resolved_addrs) +} + /// Returns a TCP connection stream with read/write timeouts set /// and Nagle's algorithm disabled with `TCP_NODELAY`. /// /// `TCP_NODELAY` ensures writing to the stream always results in immediate sending of the packet /// to the network, which is important to reduce the latency of interactive protocols such as IMAP. +/// +/// If `load_cache` is true, may use cached DNS results. +/// Use this only if the connection is going to be protected with TLS. pub(crate) async fn connect_tcp( context: &Context, host: &str, port: u16, timeout_val: Duration, + load_cache: bool, ) -> Result>>> { let mut tcp_stream = None; - for resolved_addr in lookup_host((host, port)).await? { - info!( - context, - "Resolved {}:{} into {}.", host, port, &resolved_addr - ); + + for resolved_addr in lookup_host_with_cache(context, host, port, load_cache).await? { match connect_tcp_inner(resolved_addr, timeout_val).await { Ok(stream) => { tcp_stream = Some(stream); @@ -48,6 +125,7 @@ pub(crate) async fn connect_tcp( } } } + let tcp_stream = tcp_stream.with_context(|| format!("failed to connect to {}:{}", host, port))?; diff --git a/src/socks.rs b/src/socks.rs index 52eb4fd61..58d7aa1db 100644 --- a/src/socks.rs +++ b/src/socks.rs @@ -56,14 +56,18 @@ impl Socks5Config { } } + /// If `load_dns_cache` is true, loads cached DNS resolution results. + /// Use this only if the connection is going to be protected with TLS checks. pub async fn connect( &self, context: &Context, target_host: &str, target_port: u16, timeout_val: Duration, + load_dns_cache: bool, ) -> Result>>>> { - let tcp_stream = connect_tcp(context, &self.host, self.port, timeout_val).await?; + let tcp_stream = + connect_tcp(context, &self.host, self.port, timeout_val, load_dns_cache).await?; let authentication_method = if let Some((username, password)) = self.user_password.as_ref() { diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index fde27051c..549689c6a 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -671,6 +671,19 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid); ) .await?; } + if dbversion < 97 { + sql.execute_migration( + "CREATE TABLE dns_cache ( + hostname TEXT NOT NULL, + port INTEGER NOT NULL, + address TEXT NOT NULL, + timestamp INTEGER NOT NULL, + UNIQUE (hostname, port, address) + )", + 97, + ) + .await?; + } let new_version = sql .get_raw_config_int(VERSION_CFG) From 7d508dcb5240840fa49c0cb2b694125dd78a24ca Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 16:21:47 +0000 Subject: [PATCH 66/77] Log DNS resolution errors instead of failing directly --- src/net.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/net.rs b/src/net.rs index d5b6a1ec3..d162c50c4 100644 --- a/src/net.rs +++ b/src/net.rs @@ -30,7 +30,16 @@ async fn lookup_host_with_cache( load_cache: bool, ) -> Result> { let now = time(); - let mut resolved_addrs: Vec = lookup_host((hostname, port)).await?.collect(); + let mut resolved_addrs: Vec = match lookup_host((hostname, port)).await { + Ok(res) => res.collect(), + Err(err) => { + warn!( + context, + "DNS resolution for {}:{} failed: {:#}.", hostname, port, err + ); + Vec::new() + } + }; for (i, addr) in resolved_addrs.iter().enumerate() { info!(context, "Resolved {}:{} into {}.", hostname, port, &addr); From c4c4c977a683b70130e9eeb3111a6fad6d63f495 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 16:25:27 +0000 Subject: [PATCH 67/77] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea9cc51f..516d85844 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changes - Pipeline SMTP commands #3924 +- Cache DNS results #3970 ### Fixes - Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 From 9adb9ab5f499022d693e8520bf795464b1f1483d Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 16:42:15 +0000 Subject: [PATCH 68/77] Return last connection error from `connect_tcp` --- src/net.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/net.rs b/src/net.rs index d162c50c4..7418d2f1f 100644 --- a/src/net.rs +++ b/src/net.rs @@ -4,7 +4,7 @@ use std::pin::Pin; use std::str::FromStr; use std::time::Duration; -use anyhow::{Context as _, Result}; +use anyhow::{Context as _, Error, Result}; use tokio::net::{lookup_host, TcpStream}; use tokio::time::timeout; use tokio_io_timeout::TimeoutStream; @@ -119,6 +119,7 @@ pub(crate) async fn connect_tcp( load_cache: bool, ) -> Result>>> { let mut tcp_stream = None; + let mut last_error = None; for resolved_addr in lookup_host_with_cache(context, host, port, load_cache).await? { match connect_tcp_inner(resolved_addr, timeout_val).await { @@ -131,12 +132,17 @@ pub(crate) async fn connect_tcp( context, "Failed to connect to {}: {:#}.", resolved_addr, err ); + last_error = Some(err); } } } - let tcp_stream = - tcp_stream.with_context(|| format!("failed to connect to {}:{}", host, port))?; + let tcp_stream = match tcp_stream { + Some(tcp_stream) => tcp_stream, + None => { + return Err(last_error.unwrap_or_else(|| Error::msg("no DNS resolution results"))); + } + }; // Disable Nagle's algorithm. tcp_stream.set_nodelay(true)?; From eaeaa297c72b94ac4d93095b3415e217c07a53ad Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 16:55:43 +0000 Subject: [PATCH 69/77] Maximize priority of the cached address on successful connection --- src/net.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/net.rs b/src/net.rs index 7418d2f1f..437b991f9 100644 --- a/src/net.rs +++ b/src/net.rs @@ -58,7 +58,12 @@ async fn lookup_host_with_cache( VALUES (?, ?, ?, ?) ON CONFLICT (hostname, port, address) DO UPDATE SET timestamp=excluded.timestamp", - paramsv![hostname, port, addr.to_string(), now.saturating_add(i)], + paramsv![ + hostname, + port, + addr.to_string(), + now.saturating_add(i).saturating_add(1) + ], ) .await?; } @@ -125,6 +130,17 @@ pub(crate) async fn connect_tcp( match connect_tcp_inner(resolved_addr, timeout_val).await { Ok(stream) => { tcp_stream = Some(stream); + + // Maximize priority of this cached entry. + context + .sql + .execute( + "UPDATE dns_cache + SET timestamp = ? + WHERE address = ?", + paramsv![time(), resolved_addr.to_string()], + ) + .await?; break; } Err(err) => { From 20124bfca0fcb375cee057fae486462ae4620cef Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 17:33:59 +0000 Subject: [PATCH 70/77] Add DNS lookup timeout --- src/net.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/net.rs b/src/net.rs index 437b991f9..0053109a5 100644 --- a/src/net.rs +++ b/src/net.rs @@ -20,6 +20,18 @@ async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result Result> { + let res = timeout(timeout_val, lookup_host((hostname, port))) + .await + .context("DNS lookup timeout")? + .context("DNS lookup failure")?; + Ok(res.collect()) +} + /// Looks up hostname and port using DNS and updates the address resolution cache. /// /// If `load_cache` is true, appends cached results not older than 30 days to the end. @@ -27,11 +39,12 @@ async fn lookup_host_with_cache( context: &Context, hostname: &str, port: u16, + timeout_val: Duration, load_cache: bool, ) -> Result> { let now = time(); - let mut resolved_addrs: Vec = match lookup_host((hostname, port)).await { - Ok(res) => res.collect(), + let mut resolved_addrs = match lookup_host_with_timeout(hostname, port, timeout_val).await { + Ok(res) => res, Err(err) => { warn!( context, @@ -126,7 +139,9 @@ pub(crate) async fn connect_tcp( let mut tcp_stream = None; let mut last_error = None; - for resolved_addr in lookup_host_with_cache(context, host, port, load_cache).await? { + for resolved_addr in + lookup_host_with_cache(context, host, port, timeout_val, load_cache).await? + { match connect_tcp_inner(resolved_addr, timeout_val).await { Ok(stream) => { tcp_stream = Some(stream); From 7a47c9e38bb8d25601b21765439edd0dba6c5518 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 18:29:18 +0000 Subject: [PATCH 71/77] Adapt `nicer_configuration_error` to new error message --- src/configure.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/configure.rs b/src/configure.rs index 064b9c8e0..ebce79c5d 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -665,6 +665,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec Date: Thu, 19 Jan 2023 20:26:11 +0000 Subject: [PATCH 72/77] Remove port number from DNS cache table --- src/net.rs | 18 +++++++++--------- src/sql/migrations.rs | 5 ++--- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/net.rs b/src/net.rs index 0053109a5..37648297a 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,5 +1,5 @@ ///! # Common network utilities. -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::pin::Pin; use std::str::FromStr; use std::time::Duration; @@ -67,14 +67,13 @@ async fn lookup_host_with_cache( .sql .execute( "INSERT INTO dns_cache - (hostname, port, address, timestamp) - VALUES (?, ?, ?, ?) - ON CONFLICT (hostname, port, address) + (hostname, address, timestamp) + VALUES (?, ?, ?) + ON CONFLICT (hostname, address) DO UPDATE SET timestamp=excluded.timestamp", paramsv![ hostname, - port, - addr.to_string(), + addr.ip().to_string(), now.saturating_add(i).saturating_add(1) ], ) @@ -102,8 +101,9 @@ async fn lookup_host_with_cache( ) .await? { - match SocketAddr::from_str(&cached_address) { - Ok(addr) => { + match IpAddr::from_str(&cached_address) { + Ok(ip_addr) => { + let addr = SocketAddr::new(ip_addr, port); if !resolved_addrs.contains(&addr) { resolved_addrs.push(addr); } @@ -153,7 +153,7 @@ pub(crate) async fn connect_tcp( "UPDATE dns_cache SET timestamp = ? WHERE address = ?", - paramsv![time(), resolved_addr.to_string()], + paramsv![time(), resolved_addr.ip().to_string()], ) .await?; break; diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 549689c6a..4611e9f7f 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -675,10 +675,9 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid); sql.execute_migration( "CREATE TABLE dns_cache ( hostname TEXT NOT NULL, - port INTEGER NOT NULL, - address TEXT NOT NULL, + address TEXT NOT NULL, -- IPv4 or IPv6 address timestamp INTEGER NOT NULL, - UNIQUE (hostname, port, address) + UNIQUE (hostname, address) )", 97, ) From 0978357c5fcc30f24b40a3fa3025042151e52ff0 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 20:43:53 +0000 Subject: [PATCH 73/77] Do not cache IP addresses which resolve into themselves --- src/net.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/net.rs b/src/net.rs index 37648297a..d983a6b5c 100644 --- a/src/net.rs +++ b/src/net.rs @@ -55,8 +55,13 @@ async fn lookup_host_with_cache( }; for (i, addr) in resolved_addrs.iter().enumerate() { - info!(context, "Resolved {}:{} into {}.", hostname, port, &addr); + let ip_string = addr.ip().to_string(); + if ip_string == hostname { + // IP address resolved into itself, not interesting to cache. + continue; + } + info!(context, "Resolved {}:{} into {}.", hostname, port, &addr); let i = i64::try_from(i).unwrap_or_default(); // Update the cache. @@ -71,11 +76,7 @@ async fn lookup_host_with_cache( VALUES (?, ?, ?) ON CONFLICT (hostname, address) DO UPDATE SET timestamp=excluded.timestamp", - paramsv![ - hostname, - addr.ip().to_string(), - now.saturating_add(i).saturating_add(1) - ], + paramsv![hostname, ip_string, now.saturating_add(i).saturating_add(1)], ) .await?; } From 41ccc133941f67ba1a40a0b826c2da9bc779cd2e Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 21:06:31 +0000 Subject: [PATCH 74/77] Extend `lookup_host_with_cache` comment --- src/net.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/net.rs b/src/net.rs index d983a6b5c..e94e1484e 100644 --- a/src/net.rs +++ b/src/net.rs @@ -129,7 +129,11 @@ async fn lookup_host_with_cache( /// to the network, which is important to reduce the latency of interactive protocols such as IMAP. /// /// If `load_cache` is true, may use cached DNS results. -/// Use this only if the connection is going to be protected with TLS. +/// Because the cache may be poisoned with incorrect results by networks hijacking DNS requests, +/// this option should only be used when connection is authenticated, +/// for example using TLS. +/// If TLS is not used or invalid TLS certificates are allowed, +/// this option should be disabled. pub(crate) async fn connect_tcp( context: &Context, host: &str, From a483df8b20667540546a5307b299094387175fb7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 21:29:17 +0000 Subject: [PATCH 75/77] Add all resolver results with the same timestamp --- src/net.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/net.rs b/src/net.rs index e94e1484e..c1065ea9e 100644 --- a/src/net.rs +++ b/src/net.rs @@ -54,7 +54,7 @@ async fn lookup_host_with_cache( } }; - for (i, addr) in resolved_addrs.iter().enumerate() { + for addr in resolved_addrs.iter() { let ip_string = addr.ip().to_string(); if ip_string == hostname { // IP address resolved into itself, not interesting to cache. @@ -62,12 +62,8 @@ async fn lookup_host_with_cache( } info!(context, "Resolved {}:{} into {}.", hostname, port, &addr); - let i = i64::try_from(i).unwrap_or_default(); // Update the cache. - // - // Add sequence number to the timestamp, so addresses are ordered by timestamp in the same - // order as the resolver returns them. context .sql .execute( @@ -76,7 +72,7 @@ async fn lookup_host_with_cache( VALUES (?, ?, ?) ON CONFLICT (hostname, address) DO UPDATE SET timestamp=excluded.timestamp", - paramsv![hostname, ip_string, now.saturating_add(i).saturating_add(1)], + paramsv![hostname, ip_string, now], ) .await?; } From b8da19e49f2b7d7faf428580c4ab02882a47252c Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 22:10:40 +0000 Subject: [PATCH 76/77] Upgrade async-smtp to v0.6 --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90f7593ce..489a1a754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,8 +165,9 @@ dependencies = [ [[package]] name = "async-smtp" -version = "0.5.0" -source = "git+https://github.com/async-email/async-smtp?branch=master#983597125fbb1b0eefd3eb5a21d917af71b2e1c3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ade89127f9e0d44f9e83cf574d499060005cd45b7dc76be89c0167487fe8edd" dependencies = [ "async-native-tls", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 01a31e28a..31e01a91e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ ansi_term = { version = "0.12.1", optional = true } anyhow = "1" async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] } async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] } -async-smtp = { git = "https://github.com/async-email/async-smtp", branch = "master", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } +async-smtp = { version = "0.6", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } trust-dns-resolver = "0.22" tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar From fac7b064b4912b0989e918b745cc5cafafe66061 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 19 Jan 2023 00:31:39 +0000 Subject: [PATCH 77/77] Refine Python CI Add lint environment to `deltachat-rpc-client/` and set line length to 120, same as in `python/`. Switch from flake8 to ruff. Fix ruff warnings. --- .github/workflows/ci.yml | 2 +- .../examples/echobot_advanced.py | 4 +- deltachat-rpc-client/pyproject.toml | 10 ++++ .../src/deltachat_rpc_client/__init__.py | 15 +++++ .../src/deltachat_rpc_client/_utils.py | 22 +++---- .../src/deltachat_rpc_client/account.py | 18 ++---- .../src/deltachat_rpc_client/chat.py | 34 +++-------- .../src/deltachat_rpc_client/client.py | 32 +++------- .../src/deltachat_rpc_client/events.py | 8 +-- .../src/deltachat_rpc_client/message.py | 14 +---- .../src/deltachat_rpc_client/pytestplugin.py | 4 +- .../src/deltachat_rpc_client/rpc.py | 5 +- deltachat-rpc-client/tests/test_something.py | 37 +++++------- deltachat-rpc-client/tests/test_webxdc.py | 6 +- deltachat-rpc-client/tox.ini | 11 ++++ python/doc/conf.py | 3 +- python/examples/group_tracking.py | 12 ++-- python/examples/test_examples.py | 18 +++--- python/pyproject.toml | 6 +- python/src/deltachat/__init__.py | 3 +- python/src/deltachat/_build.py | 6 +- python/src/deltachat/account.py | 25 ++++---- python/src/deltachat/chat.py | 53 +++++++--------- python/src/deltachat/contact.py | 6 +- python/src/deltachat/direct_imap.py | 11 ++-- python/src/deltachat/events.py | 37 ++++++------ python/src/deltachat/hookspec.py | 2 +- python/src/deltachat/message.py | 54 +++++++++-------- python/src/deltachat/props.py | 7 ++- python/src/deltachat/provider.py | 3 +- python/src/deltachat/reactions.py | 2 +- python/src/deltachat/testplugin.py | 60 ++++++++++--------- python/src/deltachat/tracker.py | 7 ++- python/tests/auditwheels.py | 2 +- python/tests/test_0_complex_or_slow.py | 2 +- python/tests/test_1_online.py | 32 +++++----- python/tests/test_3_offline.py | 16 ++--- python/tests/test_4_lowlevel.py | 2 +- python/tox.ini | 12 +--- scripts/cleanup_devpi_indices.py | 13 ++-- scripts/set_core_version.py | 41 +++++++------ 41 files changed, 312 insertions(+), 345 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81c0b0077..b18e90963 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,7 +144,7 @@ jobs: env: DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} working-directory: deltachat-rpc-client - run: tox -e py3 + run: tox -e py3,lint - name: install pypy if: ${{ matrix.python }} diff --git a/deltachat-rpc-client/examples/echobot_advanced.py b/deltachat-rpc-client/examples/echobot_advanced.py index 48e3d025b..303f3ee66 100644 --- a/deltachat-rpc-client/examples/echobot_advanced.py +++ b/deltachat-rpc-client/examples/echobot_advanced.py @@ -27,9 +27,7 @@ async def log_error(event): @hooks.on(events.MemberListChanged) async def on_memberlist_changed(event): - logging.info( - "member %s was %s", event.member, "added" if event.member_added else "removed" - ) + logging.info("member %s was %s", event.member, "added" if event.member_added else "removed") @hooks.on(events.GroupImageChanged) diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index 4b3bd3520..7797ac903 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -27,3 +27,13 @@ deltachat_rpc_client = [ [project.entry-points.pytest11] "deltachat_rpc_client.pytestplugin" = "deltachat_rpc_client.pytestplugin" + +[tool.black] +line-length = 120 + +[tool.ruff] +select = ["E", "F", "W", "N", "YTT", "B", "C4", "ISC", "ICN", "PT", "RET", "SIM", "TID", "ARG", "DTZ", "ERA", "PLC", "PLE", "PLW", "PIE", "COM"] +line-length = 120 + +[tool.isort] +profile = "black" diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py b/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py index ff15923b5..94edad50e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py @@ -8,3 +8,18 @@ from .contact import Contact from .deltachat import DeltaChat from .message import Message from .rpc import Rpc + +__all__ = [ + "Account", + "AttrDict", + "Bot", + "Chat", + "Client", + "Contact", + "DeltaChat", + "EventType", + "Message", + "Rpc", + "run_bot_cli", + "run_client_cli", +] diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py b/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py index 562c62de1..f53d19c92 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py @@ -30,12 +30,7 @@ class AttrDict(dict): """Dictionary that allows accessing values usin the "dot notation" as attributes.""" def __init__(self, *args, **kwargs) -> None: - super().__init__( - { - _camel_to_snake(key): _to_attrdict(value) - for key, value in dict(*args, **kwargs).items() - } - ) + super().__init__({_camel_to_snake(key): _to_attrdict(value) for key, value in dict(*args, **kwargs).items()}) def __getattr__(self, attr): if attr in self: @@ -51,7 +46,7 @@ class AttrDict(dict): async def run_client_cli( hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None, argv: Optional[list] = None, - **kwargs + **kwargs, ) -> None: """Run a simple command line app, using the given hooks. @@ -65,7 +60,7 @@ async def run_client_cli( async def run_bot_cli( hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None, argv: Optional[list] = None, - **kwargs + **kwargs, ) -> None: """Run a simple bot command line using the given hooks. @@ -80,7 +75,7 @@ async def _run_cli( client_type: Type["Client"], hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None, argv: Optional[list] = None, - **kwargs + **kwargs, ) -> None: from .deltachat import DeltaChat from .rpc import Rpc @@ -107,12 +102,9 @@ async def _run_cli( client = client_type(account, hooks) client.logger.debug("Running deltachat core %s", core_version) if not await client.is_configured(): - assert ( - args.email and args.password - ), "Account is not configured and email and password must be provided" - asyncio.create_task( - client.configure(email=args.email, password=args.password) - ) + assert args.email, "Account is not configured and email must be provided" + assert args.password, "Account is not configured and password must be provided" + asyncio.create_task(client.configure(email=args.email, password=args.password)) await client.run_forever() diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/account.py b/deltachat-rpc-client/src/deltachat_rpc_client/account.py index eab67d10d..44535da08 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/account.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/account.py @@ -89,9 +89,7 @@ class Account: """Configure an account.""" await self._rpc.configure(self.id) - async def create_contact( - self, obj: Union[int, str, Contact], name: Optional[str] = None - ) -> Contact: + async def create_contact(self, obj: Union[int, str, Contact], name: Optional[str] = None) -> Contact: """Create a new Contact or return an existing one. Calling this method will always result in the same @@ -120,10 +118,7 @@ class Account: async def get_blocked_contacts(self) -> List[AttrDict]: """Return a list with snapshots of all blocked contacts.""" contacts = await self._rpc.get_blocked_contacts(self.id) - return [ - AttrDict(contact=Contact(self, contact["id"]), **contact) - for contact in contacts - ] + return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts] async def get_contacts( self, @@ -148,10 +143,7 @@ class Account: if snapshot: contacts = await self._rpc.get_contacts(self.id, flags, query) - return [ - AttrDict(contact=Contact(self, contact["id"]), **contact) - for contact in contacts - ] + return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts] contacts = await self._rpc.get_contact_ids(self.id, flags, query) return [Contact(self, contact_id) for contact_id in contacts] @@ -192,9 +184,7 @@ class Account: if alldone_hint: flags |= ChatlistFlag.ADD_ALLDONE_HINT - entries = await self._rpc.get_chatlist_entries( - self.id, flags, query, contact and contact.id - ) + entries = await self._rpc.get_chatlist_entries(self.id, flags, query, contact and contact.id) if not snapshot: return [Chat(self, entry[0]) for entry in entries] diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index 4a2f4ae77..cdcefcf95 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -63,7 +63,7 @@ class Chat: """ if duration is not None: assert duration > 0, "Invalid duration" - dur: Union[str, dict] = dict(Until=duration) + dur: Union[str, dict] = {"Until": duration} else: dur = "Forever" await self._rpc.set_chat_mute_duration(self.account.id, self.id, dur) @@ -74,27 +74,19 @@ class Chat: async def pin(self) -> None: """Pin this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.PINNED - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.PINNED) async def unpin(self) -> None: """Unpin this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.NORMAL - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL) async def archive(self) -> None: """Archive this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.ARCHIVED - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.ARCHIVED) async def unarchive(self) -> None: """Unarchive this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.NORMAL - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL) async def set_name(self, name: str) -> None: """Set name of this chat.""" @@ -133,9 +125,7 @@ class Chat: if isinstance(quoted_msg, Message): quoted_msg = quoted_msg.id - msg_id, _ = await self._rpc.misc_send_msg( - self.account.id, self.id, text, file, location, quoted_msg - ) + msg_id, _ = await self._rpc.misc_send_msg(self.account.id, self.id, text, file, location, quoted_msg) return Message(self.account, msg_id) async def send_text(self, text: str) -> Message: @@ -241,23 +231,17 @@ class Chat: timestamp_to: Optional[datetime] = None, ) -> List[AttrDict]: """Get list of location snapshots for the given contact in the given timespan.""" - time_from = ( - calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0 - ) + time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0 time_to = calendar.timegm(timestamp_to.utctimetuple()) if timestamp_to else 0 contact_id = contact.id if contact else 0 - result = await self._rpc.get_locations( - self.account.id, self.id, contact_id, time_from, time_to - ) + result = await self._rpc.get_locations(self.account.id, self.id, contact_id, time_from, time_to) locations = [] contacts: Dict[int, Contact] = {} for loc in result: loc = AttrDict(loc) loc["chat"] = self - loc["contact"] = contacts.setdefault( - loc.contact_id, Contact(self.account, loc.contact_id) - ) + loc["contact"] = contacts.setdefault(loc.contact_id, Contact(self.account, loc.contact_id)) loc["message"] = Message(self.account, loc.msg_id) locations.append(loc) return locations diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/client.py b/deltachat-rpc-client/src/deltachat_rpc_client/client.py index 4c6deafa1..393ac3cc7 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/client.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/client.py @@ -47,15 +47,11 @@ class Client: self._should_process_messages = 0 self.add_hooks(hooks or []) - def add_hooks( - self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]] - ) -> None: + def add_hooks(self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]]) -> None: for hook, event in hooks: self.add_hook(hook, event) - def add_hook( - self, hook: Callable, event: Union[type, EventFilter] = RawEvent - ) -> None: + def add_hook(self, hook: Callable, event: Union[type, EventFilter] = RawEvent) -> None: """Register hook for the given event filter.""" if isinstance(event, type): event = event() @@ -64,7 +60,7 @@ class Client: isinstance( event, (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged), - ) + ), ) self._hooks.setdefault(type(event), set()).add((hook, event)) @@ -76,7 +72,7 @@ class Client: isinstance( event, (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged), - ) + ), ) self._hooks.get(type(event), set()).remove((hook, event)) @@ -95,9 +91,7 @@ class Client: """Process events forever.""" await self.run_until(lambda _: False) - async def run_until( - self, func: Callable[[AttrDict], Union[bool, Coroutine]] - ) -> AttrDict: + async def run_until(self, func: Callable[[AttrDict], Union[bool, Coroutine]]) -> AttrDict: """Process events until the given callable evaluates to True. The callable should accept an AttrDict object representing the @@ -122,9 +116,7 @@ class Client: if stop: return event - async def _on_event( - self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent - ) -> None: + async def _on_event(self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent) -> None: for hook, evfilter in self._hooks.get(filter_type, []): if await evfilter.filter(event): try: @@ -133,11 +125,7 @@ class Client: self.logger.exception(ex) async def _parse_command(self, event: AttrDict) -> None: - cmds = [ - hook[1].command - for hook in self._hooks.get(NewMessage, []) - if hook[1].command - ] + cmds = [hook[1].command for hook in self._hooks.get(NewMessage, []) if hook[1].command] parts = event.message_snapshot.text.split(maxsplit=1) payload = parts[1] if len(parts) > 1 else "" cmd = parts.pop(0) @@ -202,11 +190,7 @@ class Client: for message in await self.account.get_fresh_messages_in_arrival_order(): snapshot = await message.get_snapshot() await self._on_new_msg(snapshot) - if ( - snapshot.is_info - and snapshot.system_message_type - != SystemMessageType.WEBXDC_INFO_MESSAGE - ): + if snapshot.is_info and snapshot.system_message_type != SystemMessageType.WEBXDC_INFO_MESSAGE: await self._handle_info_msg(snapshot) await snapshot.message.mark_seen() diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/events.py b/deltachat-rpc-client/src/deltachat_rpc_client/events.py index 146c89ea5..3801dcc72 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/events.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/events.py @@ -10,7 +10,7 @@ from .const import EventType def _tuple_of(obj, type_: type) -> tuple: if not obj: - return tuple() + return () if isinstance(obj, type_): obj = (obj,) @@ -39,7 +39,7 @@ class EventFilter(ABC): """Return True if two event filters are equal.""" def __ne__(self, other): - return not self.__eq__(other) + return not self == other async def _call_func(self, event) -> bool: if not self.func: @@ -65,9 +65,7 @@ class RawEvent(EventFilter): should be dispatched or not. """ - def __init__( - self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs - ): + def __init__(self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs): super().__init__(**kwargs) try: self.types = _tuple_of(types, EventType) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/message.py b/deltachat-rpc-client/src/deltachat_rpc_client/message.py index 86ddbc95e..d38d2cbfe 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/message.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/message.py @@ -49,22 +49,14 @@ class Message: """Mark the message as seen.""" await self._rpc.markseen_msgs(self.account.id, [self.id]) - async def send_webxdc_status_update( - self, update: Union[dict, str], description: str - ) -> None: + async def send_webxdc_status_update(self, update: Union[dict, str], description: str) -> None: """Send a webxdc status update. This message must be a webxdc.""" if not isinstance(update, str): update = json.dumps(update) - await self._rpc.send_webxdc_status_update( - self.account.id, self.id, update, description - ) + await self._rpc.send_webxdc_status_update(self.account.id, self.id, update, description) async def get_webxdc_status_updates(self, last_known_serial: int = 0) -> list: - return json.loads( - await self._rpc.get_webxdc_status_updates( - self.account.id, self.id, last_known_serial - ) - ) + return json.loads(await self._rpc.get_webxdc_status_updates(self.account.id, self.id, last_known_serial)) async def get_webxdc_info(self) -> dict: return await self._rpc.get_webxdc_info(self.account.id, self.id) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py index 6bfd446d1..36969c9c1 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py @@ -67,9 +67,7 @@ class ACFactory: ) -> Message: if not from_account: from_account = (await self.get_online_accounts(1))[0] - to_contact = await from_account.create_contact( - await to_account.get_config("addr") - ) + to_contact = await from_account.create_contact(await to_account.get_config("addr")) if group: to_chat = await from_account.create_group(group) await to_chat.add_contact(to_contact) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py b/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py index 8a407308d..f15c1a29a 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py @@ -30,7 +30,7 @@ class Rpc: "deltachat-rpc-server", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, - **self._kwargs + **self._kwargs, ) self.id = 0 self.event_queues = {} @@ -46,7 +46,7 @@ class Rpc: await self.start() return self - async def __aexit__(self, exc_type, exc, tb): + async def __aexit__(self, _exc_type, _exc, _tb): await self.close() async def reader_loop(self) -> None: @@ -97,5 +97,6 @@ class Rpc: raise JsonRpcError(response["error"]) if "result" in response: return response["result"] + return None return method diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index d1208042e..1d02e1928 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -6,14 +6,14 @@ from deltachat_rpc_client import EventType, events from deltachat_rpc_client.rpc import JsonRpcError -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_system_info(rpc) -> None: system_info = await rpc.get_system_info() assert "arch" in system_info assert "deltachat_core_version" in system_info -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_email_address_validity(rpc) -> None: valid_addresses = [ "email@example.com", @@ -27,7 +27,7 @@ async def test_email_address_validity(rpc) -> None: assert not await rpc.check_email_validity(addr) -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_acfactory(acfactory) -> None: account = await acfactory.new_configured_account() while True: @@ -41,7 +41,7 @@ async def test_acfactory(acfactory) -> None: print("Successful configuration") -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_configure_starttls(acfactory) -> None: account = await acfactory.new_preconfigured_account() @@ -51,7 +51,7 @@ async def test_configure_starttls(acfactory) -> None: assert await account.is_configured() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_account(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -111,7 +111,7 @@ async def test_account(acfactory) -> None: await alice.stop_io() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_chat(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -177,7 +177,7 @@ async def test_chat(acfactory) -> None: await group.get_locations() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_contact(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -195,7 +195,7 @@ async def test_contact(acfactory) -> None: await alice_contact_bob.create_chat() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_message(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -226,7 +226,7 @@ async def test_message(acfactory) -> None: await message.send_reaction("😎") -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_bot(acfactory) -> None: mock = MagicMock() user = (await acfactory.get_online_accounts(1))[0] @@ -237,25 +237,20 @@ async def test_bot(acfactory) -> None: hook = lambda e: mock.hook(e.msg_id), events.RawEvent(EventType.INCOMING_MSG) bot.add_hook(*hook) - event = await acfactory.process_message( - from_account=user, to_client=bot, text="Hello!" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="Hello!") mock.hook.assert_called_once_with(event.msg_id) bot.remove_hook(*hook) - track = lambda e: mock.hook(e.message_snapshot.id) + def track(e): + mock.hook(e.message_snapshot.id) mock.hook.reset_mock() hook = track, events.NewMessage(r"hello") bot.add_hook(*hook) bot.add_hook(track, events.NewMessage(command="/help")) - event = await acfactory.process_message( - from_account=user, to_client=bot, text="hello" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="hello") mock.hook.assert_called_with(event.msg_id) - event = await acfactory.process_message( - from_account=user, to_client=bot, text="hello!" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="hello!") mock.hook.assert_called_with(event.msg_id) await acfactory.process_message(from_account=user, to_client=bot, text="hey!") assert len(mock.hook.mock_calls) == 2 @@ -263,7 +258,5 @@ async def test_bot(acfactory) -> None: mock.hook.reset_mock() await acfactory.process_message(from_account=user, to_client=bot, text="hello") - event = await acfactory.process_message( - from_account=user, to_client=bot, text="/help" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="/help") mock.hook.assert_called_once_with(event.msg_id) diff --git a/deltachat-rpc-client/tests/test_webxdc.py b/deltachat-rpc-client/tests/test_webxdc.py index 1bc5f02c1..22d9db0b4 100644 --- a/deltachat-rpc-client/tests/test_webxdc.py +++ b/deltachat-rpc-client/tests/test_webxdc.py @@ -3,16 +3,14 @@ import pytest from deltachat_rpc_client import EventType -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_webxdc(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) bob_addr = await bob.get_config("addr") alice_contact_bob = await alice.create_contact(bob_addr, "Bob") alice_chat_bob = await alice_contact_bob.create_chat() - await alice_chat_bob.send_message( - text="Let's play chess!", file="../test-data/webxdc/chess.xdc" - ) + await alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc") while True: event = await bob.wait_for_event() diff --git a/deltachat-rpc-client/tox.ini b/deltachat-rpc-client/tox.ini index bea3b4603..1c222e9a9 100644 --- a/deltachat-rpc-client/tox.ini +++ b/deltachat-rpc-client/tox.ini @@ -2,6 +2,7 @@ isolated_build = true envlist = py3 + lint [testenv] commands = @@ -16,3 +17,13 @@ deps = pytest-asyncio aiohttp aiodns + +[testenv:lint] +skipsdist = True +skip_install = True +deps = + ruff + black +commands = + black --check src/ examples/ tests/ + ruff src/ examples/ tests/ diff --git a/python/doc/conf.py b/python/doc/conf.py index 55fdcd5dd..4647c6ae2 100644 --- a/python/doc/conf.py +++ b/python/doc/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index c5ebaed42..59705c379 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -34,8 +34,10 @@ class GroupTrackingPlugin: def ac_member_added(self, chat, contact, actor, message): print( "ac_member_added {} to chat {} from {}".format( - contact.addr, chat.id, actor or message.get_sender_contact().addr - ) + contact.addr, + chat.id, + actor or message.get_sender_contact().addr, + ), ) for member in chat.get_contacts(): print("chat member: {}".format(member.addr)) @@ -44,8 +46,10 @@ class GroupTrackingPlugin: def ac_member_removed(self, chat, contact, actor, message): print( "ac_member_removed {} from chat {} by {}".format( - contact.addr, chat.id, actor or message.get_sender_contact().addr - ) + contact.addr, + chat.id, + actor or message.get_sender_contact().addr, + ), ) diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index 3978417bf..e6b2d53f4 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -13,8 +13,8 @@ def datadir(): datadir = path.join("test-data") if datadir.isdir(): return datadir - else: - pytest.skip("test-data directory not found") + pytest.skip("test-data directory not found") + return None def test_echo_quit_plugin(acfactory, lp): @@ -47,7 +47,7 @@ def test_group_tracking_plugin(acfactory, lp): botproc.fnmatch_lines( """ *ac_configure_completed* - """ + """, ) ac1.add_account_plugin(FFIEventLogger(ac1)) ac2.add_account_plugin(FFIEventLogger(ac2)) @@ -61,7 +61,7 @@ def test_group_tracking_plugin(acfactory, lp): botproc.fnmatch_lines( """ *ac_chat_modified*bot test group* - """ + """, ) lp.sec("adding third member {}".format(ac2.get_config("addr"))) @@ -76,8 +76,9 @@ def test_group_tracking_plugin(acfactory, lp): """ *ac_member_added {}*from*{}* """.format( - contact3.addr, ac1.get_config("addr") - ) + contact3.addr, + ac1.get_config("addr"), + ), ) lp.sec("contact successfully added, now removing") @@ -86,6 +87,7 @@ def test_group_tracking_plugin(acfactory, lp): """ *ac_member_removed {}*from*{}* """.format( - contact3.addr, ac1.get_config("addr") - ) + contact3.addr, + ac1.get_config("addr"), + ), ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9ff0c63e6..18dfe3e9b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -44,5 +44,9 @@ git_describe_command = "git describe --dirty --tags --long --match py-*.*" [tool.black] line-length = 120 +[tool.ruff] +select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM"] +line-length = 120 + [tool.isort] -profile = "black" \ No newline at end of file +profile = "black" diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 8a15ac942..227f88977 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -36,7 +36,8 @@ register_global_plugin(events) def run_cmdline(argv=None, account_plugins=None): """Run a simple default command line app, registering the specified - account plugins.""" + account plugins. + """ import argparse if argv is None: diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 995dd34b8..9ace6fc0e 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -102,8 +102,8 @@ def find_header(flags): printf("%s", _dc_header_file_location()); return 0; } - """ - ) + """, + ), ) cwd = os.getcwd() try: @@ -198,7 +198,7 @@ def ffibuilder(): typedef int... time_t; void free(void *ptr); extern int dc_event_has_string_data(int); - """ + """, ) function_defs = extract_functions(flags) defines = extract_defines(flags) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 2df4bf65b..76b5bea88 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -1,4 +1,4 @@ -""" Account class implementation. """ +"""Account class implementation.""" from __future__ import print_function @@ -39,7 +39,7 @@ def get_core_info(): ffi.gc( lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL), lib.dc_context_unref, - ) + ), ) @@ -172,10 +172,7 @@ class Account(object): namebytes = name.encode("utf8") if isinstance(value, (int, bool)): value = str(int(value)) - if value is not None: - valuebytes = value.encode("utf8") - else: - valuebytes = ffi.NULL + valuebytes = value.encode("utf8") if value is not None else ffi.NULL lib.dc_set_config(self._dc_context, namebytes, valuebytes) def get_config(self, name: str) -> str: @@ -225,9 +222,10 @@ class Account(object): return bool(lib.dc_is_configured(self._dc_context)) def is_open(self) -> bool: - """Determine if account is open + """Determine if account is open. - :returns True if account is open.""" + :returns True if account is open. + """ return bool(lib.dc_context_is_open(self._dc_context)) def set_avatar(self, img_path: Optional[str]) -> None: @@ -543,7 +541,7 @@ class Account(object): return from_dc_charpointer(res) def check_qr(self, qr): - """check qr code and return :class:`ScannedQRCode` instance representing the result""" + """check qr code and return :class:`ScannedQRCode` instance representing the result.""" res = ffi.gc(lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)), lib.dc_lot_unref) lot = DCLot(res) if lot.state() == const.DC_QR_ERROR: @@ -662,7 +660,7 @@ class Account(object): return lib.dc_all_work_done(self._dc_context) def start_io(self): - """start this account's IO scheduling (Rust-core async scheduler) + """start this account's IO scheduling (Rust-core async scheduler). If this account is not configured an Exception is raised. You need to call account.configure() and account.wait_configure_finish() @@ -705,12 +703,10 @@ class Account(object): """ lib.dc_maybe_network(self._dc_context) - def configure(self, reconfigure: bool = False) -> ConfigureTracker: + def configure(self) -> ConfigureTracker: """Start configuration process and return a Configtracker instance on which you can block with wait_finish() to get a True/False success value for the configuration process. - - :param reconfigure: deprecated, doesn't need to be checked anymore. """ if not self.get_config("addr") or not self.get_config("mail_pw"): raise MissingCredentials("addr or mail_pwd not set in config") @@ -733,7 +729,8 @@ class Account(object): def shutdown(self) -> None: """shutdown and destroy account (stop callback thread, close and remove - underlying dc_context).""" + underlying dc_context). + """ if self._dc_context is None: return diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index d546d4775..783f25304 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -1,4 +1,4 @@ -""" Chat and Location related API. """ +"""Chat and Location related API.""" import calendar import json @@ -37,7 +37,7 @@ class Chat(object): return self.id == getattr(other, "id", None) and self.account._dc_context == other.account._dc_context def __ne__(self, other) -> bool: - return not (self == other) + return not self == other def __repr__(self) -> str: return "".format(self.id, self.get_name()) @@ -74,19 +74,19 @@ class Chat(object): return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP def is_single(self) -> bool: - """Return True if this chat is a single/direct chat, False otherwise""" + """Return True if this chat is a single/direct chat, False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_SINGLE def is_mailinglist(self) -> bool: - """Return True if this chat is a mailing list, False otherwise""" + """Return True if this chat is a mailing list, False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_MAILINGLIST def is_broadcast(self) -> bool: - """Return True if this chat is a broadcast list, False otherwise""" + """Return True if this chat is a broadcast list, False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_BROADCAST def is_multiuser(self) -> bool: - """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise""" + """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) in ( const.DC_CHAT_TYPE_GROUP, const.DC_CHAT_TYPE_MAILINGLIST, @@ -94,11 +94,11 @@ class Chat(object): ) def is_self_talk(self) -> bool: - """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise""" + """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise.""" return bool(lib.dc_chat_is_self_talk(self._dc_chat)) def is_device_talk(self) -> bool: - """Returns True if this chat is the "Device Messages" chat, False otherwise""" + """Returns True if this chat is the "Device Messages" chat, False otherwise.""" return bool(lib.dc_chat_is_device_talk(self._dc_chat)) def is_muted(self) -> bool: @@ -109,12 +109,12 @@ class Chat(object): return bool(lib.dc_chat_is_muted(self._dc_chat)) def is_pinned(self) -> bool: - """Return True if this chat is pinned, False otherwise""" + """Return True if this chat is pinned, False otherwise.""" return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_PINNED def is_archived(self) -> bool: """Return True if this chat is archived, False otherwise. - :returns: True if archived, False otherwise + :returns: True if archived, False otherwise. """ return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED @@ -136,7 +136,7 @@ class Chat(object): def can_send(self) -> bool: """Check if messages can be sent to a give chat. - This is not true eg. for the contact requests or for the device-talk + This is not true eg. for the contact requests or for the device-talk. :returns: True if the chat is writable, False otherwise """ @@ -167,7 +167,7 @@ class Chat(object): def get_color(self): """return the color of the chat. - :returns: color as 0x00rrggbb + :returns: color as 0x00rrggbb. """ return lib.dc_chat_get_color(self._dc_chat) @@ -178,21 +178,18 @@ class Chat(object): return json.loads(s) def mute(self, duration: Optional[int] = None) -> None: - """mutes the chat + """mutes the chat. :param duration: Number of seconds to mute the chat for. None to mute until unmuted again. :returns: None """ - if duration is None: - mute_duration = -1 - else: - mute_duration = duration + mute_duration = -1 if duration is None else duration ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration) if not bool(ret): raise ValueError("Call to dc_set_chat_mute_duration failed") def unmute(self) -> None: - """unmutes the chat + """unmutes the chat. :returns: None """ @@ -252,7 +249,8 @@ class Chat(object): def get_encryption_info(self) -> Optional[str]: """Return encryption info for this chat. - :returns: a string with encryption preferences of all chat members""" + :returns: a string with encryption preferences of all chat members + """ res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id) return from_dc_charpointer(res) @@ -463,7 +461,7 @@ class Chat(object): def get_contacts(self): """get all contacts for this chat. - :returns: list of :class:`deltachat.contact.Contact` objects for this chat + :returns: list of :class:`deltachat.contact.Contact` objects for this chat. """ from .contact import Contact @@ -547,19 +545,10 @@ class Chat(object): :param timespan_to: a datetime object or None (indicating up till now) :returns: list of :class:`deltachat.chat.Location` objects. """ - if timestamp_from is None: - time_from = 0 - else: - time_from = calendar.timegm(timestamp_from.utctimetuple()) - if timestamp_to is None: - time_to = 0 - else: - time_to = calendar.timegm(timestamp_to.utctimetuple()) + time_from = 0 if timestamp_from is None else calendar.timegm(timestamp_from.utctimetuple()) + time_to = 0 if timestamp_to is None else calendar.timegm(timestamp_to.utctimetuple()) - if contact is None: - contact_id = 0 - else: - contact_id = contact.id + contact_id = 0 if contact is None else contact.id dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to) return [ diff --git a/python/src/deltachat/contact.py b/python/src/deltachat/contact.py index 75c44b8e5..adf4cdbad 100644 --- a/python/src/deltachat/contact.py +++ b/python/src/deltachat/contact.py @@ -1,4 +1,4 @@ -""" Contact object. """ +"""Contact object.""" from datetime import date, datetime, timezone from typing import Optional @@ -28,7 +28,7 @@ class Contact(object): return self.account._dc_context == other.account._dc_context and self.id == other.id def __ne__(self, other): - return not (self == other) + return not self == other def __repr__(self): return "".format(self.id, self.addr, self.account._dc_context) @@ -76,7 +76,7 @@ class Contact(object): return lib.dc_contact_is_verified(self._dc_contact) def get_verifier(self, contact): - """Return the address of the contact that verified the contact""" + """Return the address of the contact that verified the contact.""" return from_dc_charpointer(lib.dc_contact_get_verifier_addr(contact._dc_contact)) def get_profile_image(self) -> Optional[str]: diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index 5c36d47f1..f2f4417c5 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -79,15 +79,17 @@ class DirectImap: def select_config_folder(self, config_name: str): """Return info about selected folder if it is - configured, otherwise None.""" + configured, otherwise None. + """ if "_" not in config_name: config_name = "configured_{}_folder".format(config_name) foldername = self.account.get_config(config_name) if foldername: return self.select_folder(foldername) + return None def list_folders(self) -> List[str]: - """return list of all existing folder names""" + """return list of all existing folder names.""" assert not self._idling return [folder.name for folder in self.conn.folder.list()] @@ -103,7 +105,7 @@ class DirectImap: def get_all_messages(self) -> List[MailMessage]: assert not self._idling - return [mail for mail in self.conn.fetch()] + return list(self.conn.fetch()) def get_unread_messages(self) -> List[str]: assert not self._idling @@ -221,5 +223,4 @@ class IdleManager: def done(self): """send idle-done to server if we are currently in idle mode.""" - res = self.direct_imap.conn.idle.stop() - return res + return self.direct_imap.conn.idle.stop() diff --git a/python/src/deltachat/events.py b/python/src/deltachat/events.py index 49ef23f96..9cd1caefe 100644 --- a/python/src/deltachat/events.py +++ b/python/src/deltachat/events.py @@ -32,12 +32,11 @@ class FFIEvent: def __str__(self): if self.name == "DC_EVENT_INFO": return "INFO {data2}".format(data2=self.data2) - elif self.name == "DC_EVENT_WARNING": + if self.name == "DC_EVENT_WARNING": return "WARNING {data2}".format(data2=self.data2) - elif self.name == "DC_EVENT_ERROR": + if self.name == "DC_EVENT_ERROR": return "ERROR {data2}".format(data2=self.data2) - else: - return "{name} data1={data1} data2={data2}".format(**self.__dict__) + return "{name} data1={data1} data2={data2}".format(**self.__dict__) class FFIEventLogger: @@ -135,7 +134,8 @@ class FFIEventTracker: def wait_for_connectivity(self, connectivity): """Wait for the specified connectivity. This only works reliably if the connectivity doesn't change - again too quickly, otherwise we might miss it.""" + again too quickly, otherwise we might miss it. + """ while 1: if self.account.get_connectivity() == connectivity: return @@ -143,12 +143,13 @@ class FFIEventTracker: def wait_for_connectivity_change(self, previous, expected_next): """Wait until the connectivity changes to `expected_next`. - Fails the test if it changes to something else.""" + Fails the test if it changes to something else. + """ while 1: current = self.account.get_connectivity() if current == expected_next: return - elif current != previous: + if current != previous: raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current)) self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED") @@ -183,7 +184,8 @@ class FFIEventTracker: - ac1 and ac2 are created - ac1 sends a message to ac2 - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message - - therefore no DC_EVENT_INCOMING_MSG is sent""" + - therefore no DC_EVENT_INCOMING_MSG is sent + """ self.get_info_contains("INBOX: Idle entering") def wait_next_incoming_message(self): @@ -193,14 +195,15 @@ class FFIEventTracker: def wait_next_messages_changed(self): """wait for and return next message-changed message or None - if the event contains no msgid""" + if the event contains no msgid + """ ev = self.get_matching("DC_EVENT_MSGS_CHANGED") if ev.data2 > 0: return self.account.get_message_by_id(ev.data2) return None def wait_next_reactions_changed(self): - """wait for and return next reactions-changed message""" + """wait for and return next reactions-changed message.""" ev = self.get_matching("DC_EVENT_REACTIONS_CHANGED") assert ev.data1 > 0 return self.account.get_message_by_id(ev.data2) @@ -292,10 +295,10 @@ class EventThread(threading.Thread): if data1 == 0 or data1 == 1000: success = data1 == 1000 comment = ffi_event.data2 - yield "ac_configure_completed", dict(success=success, comment=comment) + yield "ac_configure_completed", {"success": success, "comment": comment} elif name == "DC_EVENT_INCOMING_MSG": msg = account.get_message_by_id(ffi_event.data2) - yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg)) + yield map_system_message(msg) or ("ac_incoming_message", {"message": msg}) elif name == "DC_EVENT_MSGS_CHANGED": if ffi_event.data2 != 0: msg = account.get_message_by_id(ffi_event.data2) @@ -303,19 +306,19 @@ class EventThread(threading.Thread): res = map_system_message(msg) if res and res[0].startswith("ac_member"): yield res - yield "ac_outgoing_message", dict(message=msg) + yield "ac_outgoing_message", {"message": msg} elif msg.is_in_fresh(): yield map_system_message(msg) or ( "ac_incoming_message", - dict(message=msg), + {"message": msg}, ) elif name == "DC_EVENT_REACTIONS_CHANGED": assert ffi_event.data1 > 0 msg = account.get_message_by_id(ffi_event.data2) - yield "ac_reactions_changed", dict(message=msg) + yield "ac_reactions_changed", {"message": msg} elif name == "DC_EVENT_MSG_DELIVERED": msg = account.get_message_by_id(ffi_event.data2) - yield "ac_message_delivered", dict(message=msg) + yield "ac_message_delivered", {"message": msg} elif name == "DC_EVENT_CHAT_MODIFIED": chat = account.get_chat_by_id(ffi_event.data1) - yield "ac_chat_modified", dict(chat=chat) + yield "ac_chat_modified", {"chat": chat} diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 4d2fe6960..51d4a9c83 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -1,4 +1,4 @@ -""" Hooks for Python bindings to Delta Chat Core Rust CFFI""" +"""Hooks for Python bindings to Delta Chat Core Rust CFFI.""" import pluggy diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 0891b46cf..0e083ad86 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -1,4 +1,4 @@ -""" The Message object. """ +"""The Message object.""" import json import os @@ -59,10 +59,7 @@ class Message(object): :param view_type: the message type code or one of the strings: "text", "audio", "video", "file", "sticker", "videochat", "webxdc" """ - if isinstance(view_type, int): - view_type_code = view_type - else: - view_type_code = get_viewtype_code_from_name(view_type) + view_type_code = view_type if isinstance(view_type, int) else get_viewtype_code_from_name(view_type) return Message( account, ffi.gc(lib.dc_msg_new(account._dc_context, view_type_code), lib.dc_msg_unref), @@ -129,7 +126,7 @@ class Message(object): @props.with_doc def filemime(self) -> str: - """mime type of the file (if it exists)""" + """mime type of the file (if it exists).""" return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg)) def get_status_updates(self, serial: int = 0) -> list: @@ -141,7 +138,7 @@ class Message(object): :param serial: The last known serial. Pass 0 if there are no known serials to receive all updates. """ return json.loads( - from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial)) + from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial)), ) def send_status_update(self, json_data: Union[str, dict], description: str) -> bool: @@ -158,8 +155,11 @@ class Message(object): json_data = json.dumps(json_data, default=str) return bool( lib.dc_send_webxdc_status_update( - self.account._dc_context, self.id, as_dc_charpointer(json_data), as_dc_charpointer(description) - ) + self.account._dc_context, + self.id, + as_dc_charpointer(json_data), + as_dc_charpointer(description), + ), ) def send_reaction(self, reaction: str): @@ -232,16 +232,18 @@ class Message(object): ts = lib.dc_msg_get_received_timestamp(self._dc_msg) if ts: return datetime.fromtimestamp(ts, timezone.utc) + return None @props.with_doc def ephemeral_timer(self): - """Ephemeral timer in seconds + """Ephemeral timer in seconds. :returns: timer in seconds or None if there is no timer """ timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg) if timer: return timer + return None @props.with_doc def ephemeral_timestamp(self): @@ -255,23 +257,25 @@ class Message(object): @property def quoted_text(self) -> Optional[str]: - """Text inside the quote + """Text inside the quote. - :returns: Quoted text""" + :returns: Quoted text + """ return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg)) @property def quote(self): - """Quote getter + """Quote getter. - :returns: Quoted message, if found in the database""" + :returns: Quoted message, if found in the database + """ msg = lib.dc_msg_get_quoted_msg(self._dc_msg) if msg: return Message(self.account, ffi.gc(msg, lib.dc_msg_unref)) @quote.setter def quote(self, quoted_message): - """Quote setter""" + """Quote setter.""" lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg) def force_plaintext(self) -> None: @@ -286,7 +290,7 @@ class Message(object): :returns: email-mime message object (with headers only, no body). """ - import email.parser + import email mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id) if mime_headers: @@ -297,7 +301,7 @@ class Message(object): @property def error(self) -> Optional[str]: - """Error message""" + """Error message.""" return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg)) @property @@ -493,7 +497,8 @@ def get_viewtype_code_from_name(view_type_name): if code is not None: return code raise ValueError( - "message typecode not found for {!r}, " "available {!r}".format(view_type_name, list(_view_type_mapping.keys())) + "message typecode not found for {!r}, " + "available {!r}".format(view_type_name, list(_view_type_mapping.keys())), ) @@ -506,14 +511,11 @@ def map_system_message(msg): if msg.is_system_message(): res = parse_system_add_remove(msg.text) if not res: - return + return None action, affected, actor = res affected = msg.account.get_contact_by_addr(affected) - if actor == "me": - actor = None - else: - actor = msg.account.get_contact_by_addr(actor) - d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg) + actor = None if actor == "me" else msg.account.get_contact_by_addr(actor) + d = {"chat": msg.chat, "contact": affected, "actor": actor, "message": msg} return "ac_member_" + res[0], d @@ -528,8 +530,8 @@ def extract_addr(text): def parse_system_add_remove(text): """return add/remove info from parsing the given system message text. - returns a (action, affected, actor) triple""" - + returns a (action, affected, actor) triple + """ # You removed member a@b. # You added member a@b. # Member Me (x@y) removed by a@b. diff --git a/python/src/deltachat/props.py b/python/src/deltachat/props.py index ab21794ae..7dcf4a1aa 100644 --- a/python/src/deltachat/props.py +++ b/python/src/deltachat/props.py @@ -8,7 +8,7 @@ def with_doc(f): # copied over unmodified from # https://github.com/devpi/devpi/blob/master/common/devpi_common/types.py def cached(f): - """returns a cached property that is calculated by function f""" + """returns a cached property that is calculated by function f.""" def get(self): try: @@ -17,8 +17,9 @@ def cached(f): self._property_cache = {} except KeyError: pass - x = self._property_cache[f] = f(self) - return x + res = f(self) + self._property_cache[f] = res + return res def set(self, val): propcache = self.__dict__.setdefault("_property_cache", {}) diff --git a/python/src/deltachat/provider.py b/python/src/deltachat/provider.py index 76d5cf780..d760dbabb 100644 --- a/python/src/deltachat/provider.py +++ b/python/src/deltachat/provider.py @@ -9,7 +9,8 @@ class ProviderNotFoundError(Exception): class Provider(object): - """Provider information. + """ + Provider information. :param domain: The email to get the provider info for. """ diff --git a/python/src/deltachat/reactions.py b/python/src/deltachat/reactions.py index 9e9ed9555..92cdf1905 100644 --- a/python/src/deltachat/reactions.py +++ b/python/src/deltachat/reactions.py @@ -1,4 +1,4 @@ -""" The Reactions object. """ +"""The Reactions object.""" from .capi import ffi, lib from .cutil import from_dc_charpointer, iter_array diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index ed92917b6..b61c1a0e4 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -29,7 +29,7 @@ def pytest_addoption(parser): "--liveconfig", action="store", default=None, - help="a file with >=2 lines where each line " "contains NAME=VALUE config settings for one account", + help="a file with >=2 lines where each line contains NAME=VALUE config settings for one account", ) group.addoption( "--ignored", @@ -124,7 +124,7 @@ def pytest_report_header(config, startdir): info["deltachat_core_version"], info["sqlite_version"], info["journal_mode"], - ) + ), ] cfg = config.option.liveconfig @@ -180,7 +180,7 @@ class TestProcess: if res.status_code != 200: pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text)) d = res.json() - config = dict(addr=d["email"], mail_pw=d["password"]) + config = {"addr": d["email"], "mail_pw": d["password"]} print("newtmpuser {}: addr={}".format(index, config["addr"])) self._configlist.append(config) yield config @@ -229,7 +229,7 @@ def write_dict_to_dir(dic, target_dir): path.write_bytes(content) -@pytest.fixture +@pytest.fixture() def data(request): class Data: def __init__(self) -> None: @@ -253,6 +253,7 @@ def data(request): if os.path.exists(fn): return fn print("WARNING: path does not exist: {!r}".format(fn)) + return None def read_path(self, bn, mode="r"): fn = self.get_path(bn) @@ -264,8 +265,11 @@ def data(request): class ACSetup: - """accounts setup helper to deal with multiple configure-process - and io & imap initialization phases. From tests, use the higher level + """ + Accounts setup helper to deal with multiple configure-process + and io & imap initialization phases. + + From tests, use the higher level public ACFactory methods instead of its private helper class. """ @@ -289,7 +293,7 @@ class ACSetup: self._account2state[account] = self.CONFIGURED self.log("added already configured account", account, account.get_config("addr")) - def start_configure(self, account, reconfigure=False): + def start_configure(self, account): """add an account and start its configure process.""" class PendingTracker: @@ -299,7 +303,7 @@ class ACSetup: account.add_account_plugin(PendingTracker(), name="pending_tracker") self._account2state[account] = self.CONFIGURING - account.configure(reconfigure=reconfigure) + account.configure() self.log("started configure on", account) def wait_one_configured(self, account): @@ -411,7 +415,8 @@ class ACFactory: acc.disable_logging() def get_next_liveconfig(self): - """Base function to get functional online configurations + """ + Base function to get functional online configurations where we can make valid SMTP and IMAP connections with. """ configdict = next(self._liveconfig_producer).copy() @@ -465,8 +470,7 @@ class ACFactory: if fname_pub and fname_sec: account._preconfigure_keypair(addr, fname_pub, fname_sec) return True - else: - print("WARN: could not use preconfigured keys for {!r}".format(addr)) + print("WARN: could not use preconfigured keys for {!r}".format(addr)) def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account: # do a pseudo-configured account @@ -476,14 +480,14 @@ class ACFactory: acname = ac._logid addr = "{}@offline.org".format(acname) ac.update_config( - dict( - addr=addr, - displayname=acname, - mail_pw="123", - configured_addr=addr, - configured_mail_pw="123", - configured="1", - ) + { + "addr": addr, + "displayname": acname, + "mail_pw": "123", + "configured_addr": addr, + "configured_mail_pw": "123", + "configured": "1", + }, ) self._preconfigure_key(ac, addr) self._acsetup.init_logging(ac) @@ -494,12 +498,12 @@ class ACFactory: configdict = self.get_next_liveconfig() else: # XXX we might want to transfer the key to the new account - configdict = dict( - addr=cloned_from.get_config("addr"), - mail_pw=cloned_from.get_config("mail_pw"), - imap_certificate_checks=cloned_from.get_config("imap_certificate_checks"), - smtp_certificate_checks=cloned_from.get_config("smtp_certificate_checks"), - ) + configdict = { + "addr": cloned_from.get_config("addr"), + "mail_pw": cloned_from.get_config("mail_pw"), + "imap_certificate_checks": cloned_from.get_config("imap_certificate_checks"), + "smtp_certificate_checks": cloned_from.get_config("smtp_certificate_checks"), + } configdict.update(kwargs) ac = self._get_cached_account(addr=configdict["addr"]) if cache else None if ac is not None: @@ -600,7 +604,7 @@ class ACFactory: acc._evtracker.wait_next_incoming_message() -@pytest.fixture +@pytest.fixture() def acfactory(request, tmpdir, testprocess, data): am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testprocess, data=data) yield am @@ -665,12 +669,12 @@ class BotProcess: ignored.append(line) -@pytest.fixture +@pytest.fixture() def tmp_db_path(tmpdir): return tmpdir.join("test.db").strpath -@pytest.fixture +@pytest.fixture() def lp(): class Printer: def sec(self, msg: str) -> None: diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 9f9cb4c53..199e5d5c9 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -77,11 +77,11 @@ class ConfigureTracker: self.account.remove_account_plugin(self) def wait_smtp_connected(self): - """wait until smtp is configured.""" + """Wait until SMTP is configured.""" self._smtp_finished.wait() def wait_imap_connected(self): - """wait until smtp is configured.""" + """Wait until IMAP is configured.""" self._imap_finished.wait() def wait_progress(self, data1=None): @@ -91,7 +91,8 @@ class ConfigureTracker: break def wait_finish(self, timeout=None): - """wait until configure is completed. + """ + Wait until configure is completed. Raise Exception if Configure failed """ diff --git a/python/tests/auditwheels.py b/python/tests/auditwheels.py index 1c34ab692..a56ca5821 100644 --- a/python/tests/auditwheels.py +++ b/python/tests/auditwheels.py @@ -15,5 +15,5 @@ if __name__ == "__main__": p, "-w", workspacedir, - ] + ], ) diff --git a/python/tests/test_0_complex_or_slow.py b/python/tests/test_0_complex_or_slow.py index 43e2b16c7..49e0ddf38 100644 --- a/python/tests/test_0_complex_or_slow.py +++ b/python/tests/test_0_complex_or_slow.py @@ -216,7 +216,7 @@ def test_fetch_existing(acfactory, lp, mvbox_move): # would also find the "Sent" folder, but it would be too late: # The sentbox thread, started by `start_io()`, would have seen that there is no # ConfiguredSentboxFolder and do nothing. - acfactory._acsetup.start_configure(ac1, reconfigure=True) + acfactory._acsetup.start_configure(ac1) acfactory.bring_accounts_online() assert_folders_configured(ac1) diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 32a355fad..42cb0f298 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -32,7 +32,7 @@ def test_basic_imap_api(acfactory, tmpdir): imap2.shutdown() -@pytest.mark.ignored +@pytest.mark.ignored() def test_configure_generate_key(acfactory, lp): # A slow test which will generate new keys. acfactory.remove_preconfigured_keys() @@ -510,7 +510,7 @@ def test_send_and_receive_message_markseen(acfactory, lp): idle2.wait_for_seen() lp.step("1") - for i in range(2): + for _i in range(2): ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ") assert ev.data1 > const.DC_CHAT_ID_LAST_SPECIAL assert ev.data2 > const.DC_MSG_ID_LAST_SPECIAL @@ -529,7 +529,7 @@ def test_send_and_receive_message_markseen(acfactory, lp): pass # mark_seen_messages() has generated events before it returns -def test_moved_markseen(acfactory, lp): +def test_moved_markseen(acfactory): """Test that message already moved to DeltaChat folder is marked as seen.""" ac1 = acfactory.new_online_configuring_account() ac2 = acfactory.new_online_configuring_account(mvbox_move=True) @@ -553,7 +553,7 @@ def test_moved_markseen(acfactory, lp): ac2.mark_seen_messages([msg]) uid = idle2.wait_for_seen() - assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1 + assert len(list(ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*"))))) == 1 def test_message_override_sender_name(acfactory, lp): @@ -832,7 +832,7 @@ def test_send_first_message_as_long_unicode_with_cr(acfactory, lp): lp.sec("sending multi-line non-unicode message from ac1 to ac2") text1 = ( "hello\nworld\nthis is a very long message that should be" - + " wrapped using format=flowed and unwrapped on the receiver" + " wrapped using format=flowed and unwrapped on the receiver" ) msg_out = chat.send_text(text1) assert not msg_out.is_encrypted() @@ -894,7 +894,7 @@ def test_dont_show_emails(acfactory, lp): message in Drafts that is moved to Sent later """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -908,7 +908,7 @@ def test_dont_show_emails(acfactory, lp): message in Sent """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -922,7 +922,7 @@ def test_dont_show_emails(acfactory, lp): Unknown message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -936,7 +936,7 @@ def test_dont_show_emails(acfactory, lp): Unknown & malformed message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -950,7 +950,7 @@ def test_dont_show_emails(acfactory, lp): Unknown & malformed message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -964,7 +964,7 @@ def test_dont_show_emails(acfactory, lp): Actually interesting message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -978,7 +978,7 @@ def test_dont_show_emails(acfactory, lp): Unknown message in Junk """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) @@ -1710,7 +1710,7 @@ def test_system_group_msg_from_blocked_user(acfactory, lp): assert contact.is_blocked() chat_on_ac2.remove_contact(ac1) ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED") - assert not ac1.get_self_contact() in chat_on_ac1.get_contacts() + assert ac1.get_self_contact() not in chat_on_ac1.get_contacts() def test_set_get_group_image(acfactory, data, lp): @@ -1784,7 +1784,7 @@ def test_connectivity(acfactory, lp): lp.sec( "Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " - + "all messages are fetched" + "all messages are fetched", ) ac1.direct_imap.select_config_folder("inbox") @@ -2146,7 +2146,7 @@ def test_group_quote(acfactory, lp): @pytest.mark.parametrize( - "folder,move,expected_destination,", + ("folder", "move", "expected_destination"), [ ( "xyz", @@ -2298,7 +2298,7 @@ class TestOnlineConfigureFails: def test_invalid_password(self, acfactory): configdict = acfactory.get_next_liveconfig() ac1 = acfactory.get_unconfigured_account() - ac1.update_config(dict(addr=configdict["addr"], mail_pw="123")) + ac1.update_config({"addr": configdict["addr"], "mail_pw": "123"}) configtracker = ac1.configure() configtracker.wait_progress(500) configtracker.wait_progress(0) diff --git a/python/tests/test_3_offline.py b/python/tests/test_3_offline.py index 98890287a..97dedfe37 100644 --- a/python/tests/test_3_offline.py +++ b/python/tests/test_3_offline.py @@ -15,7 +15,7 @@ from deltachat.tracker import ImexFailed @pytest.mark.parametrize( - "msgtext,res", + ("msgtext", "res"), [ ( "Member Me (tmp1@x.org) removed by tmp2@x.org.", @@ -108,7 +108,7 @@ class TestOfflineAccountBasic: def test_update_config(self, acfactory): ac1 = acfactory.get_unconfigured_account() - ac1.update_config(dict(mvbox_move=False)) + ac1.update_config({"mvbox_move": False}) assert ac1.get_config("mvbox_move") == "0" def test_has_savemime(self, acfactory): @@ -229,11 +229,11 @@ class TestOfflineContact: class TestOfflineChat: - @pytest.fixture + @pytest.fixture() def ac1(self, acfactory): return acfactory.get_pseudo_configured_account() - @pytest.fixture + @pytest.fixture() def chat1(self, ac1): return ac1.create_contact("some1@example.org", name="some1").create_chat() @@ -257,7 +257,7 @@ class TestOfflineChat: assert chat2.id == chat1.id assert chat2.get_name() == chat1.get_name() assert chat1 == chat2 - assert not (chat1 != chat2) + assert not chat1.__ne__(chat2) assert chat1 != chat3 for ichat in ac1.get_chats(): @@ -450,7 +450,7 @@ class TestOfflineChat: assert msg.filemime == "image/png" @pytest.mark.parametrize( - "fn,typein,typeout", + ("fn", "typein", "typeout"), [ ("r", None, "application/octet-stream"), ("r.txt", None, "text/plain"), @@ -458,7 +458,7 @@ class TestOfflineChat: ("r.txt", "image/png", "image/png"), ], ) - def test_message_file(self, ac1, chat1, data, lp, fn, typein, typeout): + def test_message_file(self, chat1, data, lp, fn, typein, typeout): lp.sec("sending file") fp = data.get_path(fn) msg = chat1.send_file(fp, typein) @@ -694,7 +694,7 @@ class TestOfflineChat: chat1.set_draft(None) assert chat1.get_draft() is None - def test_qr_setup_contact(self, acfactory, lp): + def test_qr_setup_contact(self, acfactory): ac1 = acfactory.get_pseudo_configured_account() ac2 = acfactory.get_pseudo_configured_account() qr = ac1.get_setup_contact_qr() diff --git a/python/tests/test_4_lowlevel.py b/python/tests/test_4_lowlevel.py index 7b0acd822..cb517670c 100644 --- a/python/tests/test_4_lowlevel.py +++ b/python/tests/test_4_lowlevel.py @@ -93,7 +93,7 @@ def test_empty_context(): capi.lib.dc_context_unref(ctx) -def test_dc_close_events(tmpdir, acfactory): +def test_dc_close_events(acfactory): ac1 = acfactory.get_unconfigured_account() # register after_shutdown function diff --git a/python/tox.ini b/python/tox.ini index b182c41a1..d01dd82e4 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -50,18 +50,14 @@ commands = skipsdist = True skip_install = True deps = - flake8 -# isort 5.11.0 is broken: https://github.com/PyCQA/isort/issues/2031 - isort<5.11.0 + ruff black # pygments required by rst-lint pygments restructuredtext_lint commands = - isort --check setup.py install_python_bindings.py src/deltachat examples/ tests/ black --check setup.py install_python_bindings.py src/deltachat examples/ tests/ - flake8 src/deltachat - flake8 tests/ examples/ + ruff src/deltachat tests/ examples/ rst-lint --encoding 'utf-8' README.rst [testenv:mypy] @@ -102,7 +98,3 @@ timeout = 150 timeout_func_only = True markers = ignored: ignore this test in default test runs, use --ignored to run. - -[flake8] -max-line-length = 120 -ignore = E203, E266, E501, W503 diff --git a/scripts/cleanup_devpi_indices.py b/scripts/cleanup_devpi_indices.py index 8e1eaa9ee..b32a50474 100644 --- a/scripts/cleanup_devpi_indices.py +++ b/scripts/cleanup_devpi_indices.py @@ -2,12 +2,13 @@ Remove old "dc" indices except for master which always stays. """ -from requests import Session import datetime -import sys import subprocess +import sys -MAXDAYS=7 +from requests import Session + +MAXDAYS = 7 session = Session() session.headers["Accept"] = "application/json" @@ -54,7 +55,8 @@ def run(): if not dates: print( "%s has no releases" % (baseurl + username + "/" + indexname), - file=sys.stderr) + file=sys.stderr, + ) date = datetime.datetime.now() else: date = datetime.datetime(*max(dates)) @@ -67,6 +69,5 @@ def run(): subprocess.check_call(["devpi", "index", "-y", "--delete", url]) - -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/scripts/set_core_version.py b/scripts/set_core_version.py index 9a46a8278..8046081fa 100755 --- a/scripts/set_core_version.py +++ b/scripts/set_core_version.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import os import json -import re +import os import pathlib +import re import subprocess from argparse import ArgumentParser @@ -23,7 +23,7 @@ def read_toml_version(relpath): res = regex_matches(relpath, rex) if res is not None: return res.group(1) - raise ValueError("no version found in {}".format(relpath)) + raise ValueError(f"no version found in {relpath}") def replace_toml_version(relpath, newversion): @@ -34,8 +34,8 @@ def replace_toml_version(relpath, newversion): for line in open(str(p)): m = rex.match(line) if m is not None: - print("{}: set version={}".format(relpath, newversion)) - f.write('version = "{}"\n'.format(newversion)) + print(f"{relpath}: set version={newversion}") + f.write(f'version = "{newversion}"\n') else: f.write(line) os.rename(tmp_path, str(p)) @@ -44,7 +44,7 @@ def replace_toml_version(relpath, newversion): def read_json_version(relpath): p = pathlib.Path(relpath) assert p.exists() - with open(p, "r") as f: + with open(p) as f: json_data = json.loads(f.read()) return json_data["version"] @@ -52,7 +52,7 @@ def read_json_version(relpath): def update_package_json(relpath, newversion): p = pathlib.Path(relpath) assert p.exists() - with open(p, "r") as f: + with open(p) as f: json_data = json.loads(f.read()) json_data["version"] = newversion with open(p, "w") as f: @@ -63,7 +63,7 @@ def main(): parser = ArgumentParser(prog="set_core_version") parser.add_argument("newversion") - json_list = ["package.json", "deltachat-jsonrpc/typescript/package.json"] + json_list = ["package.json", "deltachat-jsonrpc/typescript/package.json"] toml_list = [ "Cargo.toml", "deltachat-ffi/Cargo.toml", @@ -75,9 +75,9 @@ def main(): except SystemExit: print() for x in toml_list: - print("{}: {}".format(x, read_toml_version(x))) + print(f"{x}: {read_toml_version(x)}") for x in json_list: - print("{}: {}".format(x, read_json_version(x))) + print(f"{x}: {read_json_version(x)}") print() raise SystemExit("need argument: new version, example: 1.25.0") @@ -92,19 +92,19 @@ def main(): if "alpha" not in newversion: for line in open("CHANGELOG.md"): ## 1.25.0 - if line.startswith("## "): - if line[2:].strip().startswith(newversion): - break + if line.startswith("## ") and line[2:].strip().startswith(newversion): + break else: - raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion)) + raise SystemExit( + f"CHANGELOG.md contains no entry for version: {newversion}" + ) for toml_filename in toml_list: replace_toml_version(toml_filename, newversion) - + for json_filename in json_list: update_package_json(json_filename, newversion) - print("running cargo check") subprocess.call(["cargo", "check"]) @@ -114,13 +114,12 @@ def main(): print("after commit, on master make sure to: ") print("") - print(" git tag -a {}".format(newversion)) - print(" git push origin {}".format(newversion)) - print(" git tag -a py-{}".format(newversion)) - print(" git push origin py-{}".format(newversion)) + print(f" git tag -a {newversion}") + print(f" git push origin {newversion}") + print(f" git tag -a py-{newversion}") + print(f" git push origin py-{newversion}") print("") if __name__ == "__main__": main() -