diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee4abe96c..b18e90963 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,7 @@ jobs: build_and_test: name: Build and test strategy: + fail-fast: false matrix: include: # Currently used Rust version. @@ -143,7 +144,7 @@ jobs: env: DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} working-directory: deltachat-rpc-client - run: tox -e py3 + run: tox -e py3,lint - name: install pypy if: ${{ matrix.python }} diff --git a/.github/workflows/jsonrpc.yml b/.github/workflows/jsonrpc.yml index e84985229..de05f50d6 100644 --- a/.github/workflows/jsonrpc.yml +++ b/.github/workflows/jsonrpc.yml @@ -19,11 +19,6 @@ jobs: uses: actions/setup-node@v3 with: node-version: 16.x - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - name: Add Rust cache uses: Swatinem/rust-cache@v2 - name: npm install diff --git a/CHANGELOG.md b/CHANGELOG.md index 48effd837..15e1e06de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,39 @@ ## Unreleased +### Changes +- Pipeline SMTP commands #3924 +- Cache DNS results #3970 + +### Fixes +- Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914 +- fix verifier-by addr was empty string intead of None #3961 +- Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with + unread messages increases #3959 +- Fix Peerstate comparison #3962 +- Log SOCKS5 configuration for IMAP like already done for SMTP #3964 +- Fix SOCKS5 usage for IMAP #3965 +- Exit from recently seen loop on interrupt channel errors to avoid busy looping #3966 + +### API-Changes +- jsonrpc: add verified-by information to `Contact`-Object +- Remove `attach_selfavatar` config #3951 + + +## 1.106.0 + +### Changes +- Only send IncomingMsgBunch if there are more than 0 new messages #3941 + +### Fixes +- fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938 +- Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943 +- Do not treat invalid email addresses as an exception #3942 +- Add timeouts to HTTP requests #3948 + + +## 1.105.0 + ### Changes - Validate signatures in try_decrypt() even if the message isn't encrypted #3859 - Don't parse the message again after detached signatures validation #3862 @@ -12,10 +45,14 @@ - Buffer IMAP client writes #3888 - move `DC_CHAT_ID_ARCHIVED_LINK` to the top of chat lists and make `dc_get_fresh_msg_cnt()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3918 +- make `dc_marknoticed_chat()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3919 +- Update provider database ### API-Changes - jsonrpc: add python API for webxdc updates #3872 +- jsonrpc: add fresh message count to ChatListItemFetchResult::ArchiveLink - Add ffi functions to retrieve `verified by` information #3786 +- resultify `Message::get_filebytes()` #3925 ### Fixes - Do not add an error if the message is encrypted but not signed #3860 @@ -25,6 +62,7 @@ - Fix STARTTLS connection and add a test for it #3907 - Trigger reconnection when failing to fetch existing messages #3911 - Do not retry fetching existing messages after failure, prevents infinite reconnection loop #3913 +- Ensure format=flowed formatting is always reversible on the receiver side #3880 ## 1.104.0 diff --git a/Cargo.lock b/Cargo.lock index 373223d0b..489a1a754 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,9 +165,9 @@ dependencies = [ [[package]] name = "async-smtp" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da21e1dd19fbad3e095ad519fb1558ab77fd82e5c4778dca8f9be0464589e1e" +checksum = "2ade89127f9e0d44f9e83cf574d499060005cd45b7dc76be89c0167487fe8edd" dependencies = [ "async-native-tls", "async-trait", @@ -371,6 +371,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "base64ct" version = "1.5.1" @@ -867,7 +873,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deltachat" -version = "1.104.0" +version = "1.106.0" dependencies = [ "ansi_term", "anyhow", @@ -877,7 +883,7 @@ dependencies = [ "async-smtp", "async_zip", "backtrace", - "base64 0.13.1", + "base64 0.20.0", "bitflags", "chrono", "criterion", @@ -941,7 +947,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.104.0" +version = "1.106.0" dependencies = [ "anyhow", "async-channel", @@ -963,7 +969,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.104.0" +version = "1.106.0" dependencies = [ "anyhow", "deltachat-jsonrpc", @@ -986,7 +992,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.104.0" +version = "1.106.0" dependencies = [ "anyhow", "deltachat", @@ -2739,9 +2745,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" dependencies = [ "memchr", ] @@ -3184,18 +3190,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.148" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.148" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -3204,9 +3210,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ "itoa", "ryu", @@ -3538,9 +3544,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 4cfd9d809..31e01a91e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.104.0" +version = "1.106.0" edition = "2021" license = "MPL-2.0" rust-version = "1.63" @@ -25,12 +25,12 @@ ansi_term = { version = "0.12.1", optional = true } anyhow = "1" async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] } async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] } -async-smtp = { version = "0.5", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } +async-smtp = { version = "0.6", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } trust-dns-resolver = "0.22" tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar backtrace = "0.3" -base64 = "0.13" +base64 = "0.20" bitflags = "1.3" chrono = { version = "0.4", default-features=false, features = ["clock", "std"] } dirs = { version = "4", optional=true } @@ -53,7 +53,7 @@ once_cell = "1.17.0" percent-encoding = "2.2" pgp = { version = "0.9", default-features = false } pretty_env_logger = { version = "0.4", optional = true } -quick-xml = "0.26" +quick-xml = "0.27" r2d2 = "0.8" r2d2_sqlite = "0.20" rand = "0.8" diff --git a/benches/create_account.rs b/benches/create_account.rs index 56f8877d6..5e1ae8561 100644 --- a/benches/create_account.rs +++ b/benches/create_account.rs @@ -1,6 +1,7 @@ +use std::path::PathBuf; + use criterion::{black_box, criterion_group, criterion_main, Criterion}; use deltachat::accounts::Accounts; -use std::path::PathBuf; use tempfile::tempdir; async fn create_accounts(n: u32) { diff --git a/benches/get_chat_msgs.rs b/benches/get_chat_msgs.rs index 61d39e0e7..fac37dd87 100644 --- a/benches/get_chat_msgs.rs +++ b/benches/get_chat_msgs.rs @@ -1,7 +1,6 @@ use std::path::Path; use criterion::{black_box, criterion_group, criterion_main, Criterion}; - use deltachat::chat::{self, ChatId}; use deltachat::chatlist::Chatlist; use deltachat::context::Context; diff --git a/benches/get_chatlist.rs b/benches/get_chatlist.rs index 3b7a0ca1d..7b0f5556e 100644 --- a/benches/get_chatlist.rs +++ b/benches/get_chatlist.rs @@ -1,7 +1,6 @@ use std::path::Path; use criterion::{black_box, criterion_group, criterion_main, Criterion}; - use deltachat::chatlist::Chatlist; use deltachat::context::Context; use deltachat::stock_str::StockStrings; diff --git a/benches/search_msgs.rs b/benches/search_msgs.rs index 48333696d..5605eb3cf 100644 --- a/benches/search_msgs.rs +++ b/benches/search_msgs.rs @@ -1,8 +1,9 @@ +use std::path::Path; + use criterion::{black_box, criterion_group, criterion_main, Criterion}; use deltachat::context::Context; use deltachat::stock_str::StockStrings; use deltachat::Events; -use std::path::Path; async fn search_benchmark(dbfile: impl AsRef) { let id = 100; diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index b6e18901b..3488ba82b 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.104.0" +version = "1.106.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 47452f6ac..507fbe76c 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -5811,7 +5811,7 @@ void dc_event_unref(dc_event_t* event); * @param data2 (int) The progress as: * 300=vg-/vc-request received, typically shown as "bob@addr joins". * 600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified". - * 800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol. + * 800=contact added to chat, shown as "bob@addr securely joined GROUP". Only for the verified-group-protocol. * 1000=Protocol finished for this contact. */ #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS 2060 diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 764f1acad..45504d041 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -23,13 +23,6 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use anyhow::Context as _; -use deltachat::qr_code_generator::get_securejoin_qr_svg; -use num_traits::{FromPrimitive, ToPrimitive}; -use once_cell::sync::Lazy; -use rand::Rng; -use tokio::runtime::Runtime; -use tokio::sync::RwLock; - use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use deltachat::constants::DC_MSG_ID_LAST_SPECIAL; use deltachat::contact::{Contact, ContactId, Origin}; @@ -37,21 +30,28 @@ use deltachat::context::Context; use deltachat::ephemeral::Timer as EphemeralTimer; use deltachat::key::DcKey; use deltachat::message::MsgId; +use deltachat::qr_code_generator::get_securejoin_qr_svg; use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions}; use deltachat::stock_str::StockMessage; use deltachat::stock_str::StockStrings; use deltachat::webxdc::StatusUpdateSerial; use deltachat::*; use deltachat::{accounts::Accounts, log::LogExt}; +use num_traits::{FromPrimitive, ToPrimitive}; +use once_cell::sync::Lazy; +use rand::Rng; +use tokio::runtime::Runtime; +use tokio::sync::RwLock; use tokio::task::JoinHandle; mod dc_array; mod lot; mod string; -use self::string::*; use deltachat::chatlist::Chatlist; +use self::string::*; + // as C lacks a good and portable error handling, // in general, the C Interface is forgiving wrt to bad parameters. // - objects returned by some functions @@ -3309,6 +3309,8 @@ pub unsafe extern "C" fn dc_msg_get_filebytes(msg: *mut dc_msg_t) -> u64 { let ctx = &*ffi_msg.context; block_on(ffi_msg.message.get_filebytes(ctx)) + .unwrap_or_log_default(ctx, "Cannot get file size") + .unwrap_or_default() } #[no_mangle] @@ -3973,13 +3975,10 @@ pub unsafe extern "C" fn dc_contact_get_verifier_addr( } let ffi_contact = &*contact; let ctx = &*ffi_contact.context; - block_on(Contact::get_verifier_addr( - ctx, - &ffi_contact.contact.get_id(), - )) - .log_err(ctx, "failed to get verifier for contact") - .unwrap_or_default() - .strdup() + block_on(ffi_contact.contact.get_verifier_addr(ctx)) + .log_err(ctx, "failed to get verifier for contact") + .unwrap_or_default() + .strdup() } #[no_mangle] @@ -3990,12 +3989,12 @@ pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) } let ffi_contact = &*contact; let ctx = &*ffi_contact.context; - let contact_id = block_on(Contact::get_verifier_id(ctx, &ffi_contact.contact.get_id())) + let verifier_contact_id = block_on(ffi_contact.contact.get_verifier_id(ctx)) .log_err(ctx, "failed to get verifier") .unwrap_or_default() .unwrap_or_default(); - contact_id.to_u32() + verifier_contact_id.to_u32() } // dc_lot_t @@ -4578,11 +4577,12 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter( #[cfg(feature = "jsonrpc")] mod jsonrpc { - use super::*; use deltachat_jsonrpc::api::CommandApi; use deltachat_jsonrpc::events::event_to_json_rpc_notification; use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession}; + use super::*; + pub struct dc_jsonrpc_instance_t { receiver: OutReceiver, handle: RpcSession, diff --git a/deltachat-ffi/src/lot.rs b/deltachat-ffi/src/lot.rs index 0f3dadc3b..1e6a7831a 100644 --- a/deltachat-ffi/src/lot.rs +++ b/deltachat-ffi/src/lot.rs @@ -1,10 +1,12 @@ //! # Legacy generic return values for C API. +use std::borrow::Cow; + +use anyhow::Error; + use crate::message::MessageState; use crate::qr::Qr; use crate::summary::{Summary, SummaryPrefix}; -use anyhow::Error; -use std::borrow::Cow; /// An object containing a set of values. /// The meaning of the values is defined by the function returning the object. diff --git a/deltachat-ffi/src/string.rs b/deltachat-ffi/src/string.rs index 65c091108..296be1421 100644 --- a/deltachat-ffi/src/string.rs +++ b/deltachat-ffi/src/string.rs @@ -287,9 +287,10 @@ fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path { #[cfg(test)] mod tests { - use super::*; use libc::{free, strcmp}; + use super::*; + #[test] fn test_os_str_to_c_string_cwd() { let some_dir = std::env::current_dir().unwrap(); diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index dbe8b2d17..763548287 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.104.0" +version = "1.106.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" @@ -20,10 +20,10 @@ tempfile = "3.3.0" log = "0.4" async-channel = { version = "1.8.0" } futures = { version = "0.3.25" } -serde_json = "1.0.89" +serde_json = "1.0.91" yerpc = { version = "^0.3.1", features = ["anyhow_expose"] } typescript-type-def = { version = "0.5.5", features = ["json_value"] } -tokio = { version = "1.23.0" } +tokio = { version = "1.23.1" } sanitize-filename = "0.4" walkdir = "2.3.2" @@ -32,7 +32,7 @@ axum = { version = "0.6.1", optional = true, features = ["ws"] } env_logger = { version = "0.10.0", optional = true } [dev-dependencies] -tokio = { version = "1.23.0", features = ["full", "rt-multi-thread"] } +tokio = { version = "1.23.1", features = ["full", "rt-multi-thread"] } [features] diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index a311f8f07..d40a08a19 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1,4 +1,9 @@ +use std::collections::BTreeMap; +use std::sync::Arc; +use std::{collections::HashMap, str::FromStr}; + use anyhow::{anyhow, bail, ensure, Context, Result}; +pub use deltachat::accounts::Accounts; use deltachat::{ chat::{ self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, marknoticed_chat, @@ -23,21 +28,14 @@ use deltachat::{ webxdc::StatusUpdateSerial, }; use sanitize_filename::is_sanitized; -use std::collections::BTreeMap; -use std::sync::Arc; -use std::{collections::HashMap, str::FromStr}; use tokio::{fs, sync::RwLock}; use walkdir::WalkDir; use yerpc::rpc; -pub use deltachat::accounts::Accounts; - pub mod events; pub mod types; -use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult}; -use crate::api::types::qr::QrObject; - +use num_traits::FromPrimitive; use types::account::Account; use types::chat::FullChat; use types::chat_list::ChatListEntry; @@ -53,8 +51,8 @@ use self::types::{ JSONRPCMessageListItem, MessageNotificationInfo, MessageSearchResult, MessageViewtype, }, }; - -use num_traits::FromPrimitive; +use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult}; +use crate::api::types::qr::QrObject; #[derive(Clone, Debug)] pub struct CommandApi { diff --git a/deltachat-jsonrpc/src/api/types/chat_list.rs b/deltachat-jsonrpc/src/api/types/chat_list.rs index 45933770f..83b84dd3d 100644 --- a/deltachat-jsonrpc/src/api/types/chat_list.rs +++ b/deltachat-jsonrpc/src/api/types/chat_list.rs @@ -48,12 +48,10 @@ pub enum ChatListItemFetchResult { dm_chat_contact: Option, was_seen_recently: bool, }, - ArchiveLink, #[serde(rename_all = "camelCase")] - Error { - id: u32, - error: String, - }, + ArchiveLink { fresh_message_counter: usize }, + #[serde(rename_all = "camelCase")] + Error { id: u32, error: String }, } pub(crate) async fn get_chat_list_item_by_id( @@ -66,8 +64,12 @@ pub(crate) async fn get_chat_list_item_by_id( _ => Some(MsgId::new(entry.1)), }; + let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?; + if chat_id.is_archived_link() { - return Ok(ChatListItemFetchResult::ArchiveLink); + return Ok(ChatListItemFetchResult::ArchiveLink { + fresh_message_counter, + }); } let chat = Chat::load_from_db(ctx, chat_id).await?; @@ -111,7 +113,6 @@ pub(crate) async fn get_chat_list_item_by_id( (None, false) }; - let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?; let color = color_int_to_hex_string(chat.get_color(ctx).await?); Ok(ChatListItemFetchResult::ChatListItem { diff --git a/deltachat-jsonrpc/src/api/types/contact.rs b/deltachat-jsonrpc/src/api/types/contact.rs index 4ed4cf435..d67fc7fb1 100644 --- a/deltachat-jsonrpc/src/api/types/contact.rs +++ b/deltachat-jsonrpc/src/api/types/contact.rs @@ -20,6 +20,10 @@ pub struct ContactObject { name_and_addr: String, is_blocked: bool, is_verified: bool, + /// the address that verified this contact + verifier_addr: Option, + /// the id of the contact that verified this contact + verifier_id: Option, /// the contact's last seen timestamp last_seen: i64, was_seen_recently: bool, @@ -36,6 +40,18 @@ impl ContactObject { }; let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified; + let (verifier_addr, verifier_id) = if is_verified { + ( + contact.get_verifier_addr(context).await?, + contact + .get_verifier_id(context) + .await? + .map(|contact_id| contact_id.to_u32()), + ) + } else { + (None, None) + }; + Ok(ContactObject { address: contact.get_addr().to_owned(), color: color_int_to_hex_string(contact.get_color()), @@ -48,6 +64,8 @@ impl ContactObject { name_and_addr: contact.get_name_n_addr(), is_blocked: contact.is_blocked(), is_verified, + verifier_addr, + verifier_id, last_seen: contact.last_seen(), was_seen_recently: contact.was_seen_recently(), }) diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index c7e9cfe8d..0f35f0c6f 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -105,7 +105,7 @@ impl MessageObject { let sender_contact = Contact::load_from_db(context, message.get_from_id()).await?; let sender = ContactObject::try_from_dc_contact(context, sender_contact).await?; - let file_bytes = message.get_filebytes(context).await; + let file_bytes = message.get_filebytes(context).await?.unwrap_or_default(); let override_sender_name = message.get_override_sender_name(); let webxdc_info = if message.get_viewtype() == Viewtype::Webxdc { diff --git a/deltachat-jsonrpc/src/lib.rs b/deltachat-jsonrpc/src/lib.rs index 344e91afb..17429d8de 100644 --- a/deltachat-jsonrpc/src/lib.rs +++ b/deltachat-jsonrpc/src/lib.rs @@ -4,12 +4,13 @@ pub use yerpc; #[cfg(test)] mod tests { - use super::api::{Accounts, CommandApi}; use async_channel::unbounded; use futures::StreamExt; use tempfile::TempDir; use yerpc::{RpcClient, RpcSession}; + use super::api::{Accounts, CommandApi}; + #[tokio::test(flavor = "multi_thread")] async fn basic_json_rpc_functionality() -> anyhow::Result<()> { let tmp_dir = TempDir::new().unwrap().path().into(); diff --git a/deltachat-jsonrpc/src/webserver.rs b/deltachat-jsonrpc/src/webserver.rs index c2f770be7..9231069c5 100644 --- a/deltachat-jsonrpc/src/webserver.rs +++ b/deltachat-jsonrpc/src/webserver.rs @@ -1,6 +1,7 @@ -use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router}; use std::net::SocketAddr; use std::path::PathBuf; + +use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router}; use yerpc::axum::handle_ws_rpc; use yerpc::{RpcClient, RpcSession}; diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 9937dbabb..480685078 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -48,5 +48,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.104.0" + "version": "1.106.0" } \ No newline at end of file diff --git a/deltachat-jsonrpc/typescript/test/online.ts b/deltachat-jsonrpc/typescript/test/online.ts index 4bf7181c9..06c4c8954 100644 --- a/deltachat-jsonrpc/typescript/test/online.ts +++ b/deltachat-jsonrpc/typescript/test/online.ts @@ -12,7 +12,7 @@ describe("online tests", function () { let accountId1: number, accountId2: number; before(async function () { - this.timeout(12000); + this.timeout(60000); if (!process.env.DCC_NEW_TMP_EMAIL) { if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) { console.error( diff --git a/deltachat-rpc-client/examples/echobot_advanced.py b/deltachat-rpc-client/examples/echobot_advanced.py index 48e3d025b..303f3ee66 100644 --- a/deltachat-rpc-client/examples/echobot_advanced.py +++ b/deltachat-rpc-client/examples/echobot_advanced.py @@ -27,9 +27,7 @@ async def log_error(event): @hooks.on(events.MemberListChanged) async def on_memberlist_changed(event): - logging.info( - "member %s was %s", event.member, "added" if event.member_added else "removed" - ) + logging.info("member %s was %s", event.member, "added" if event.member_added else "removed") @hooks.on(events.GroupImageChanged) diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index 4b3bd3520..7797ac903 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -27,3 +27,13 @@ deltachat_rpc_client = [ [project.entry-points.pytest11] "deltachat_rpc_client.pytestplugin" = "deltachat_rpc_client.pytestplugin" + +[tool.black] +line-length = 120 + +[tool.ruff] +select = ["E", "F", "W", "N", "YTT", "B", "C4", "ISC", "ICN", "PT", "RET", "SIM", "TID", "ARG", "DTZ", "ERA", "PLC", "PLE", "PLW", "PIE", "COM"] +line-length = 120 + +[tool.isort] +profile = "black" diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py b/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py index ff15923b5..94edad50e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/__init__.py @@ -8,3 +8,18 @@ from .contact import Contact from .deltachat import DeltaChat from .message import Message from .rpc import Rpc + +__all__ = [ + "Account", + "AttrDict", + "Bot", + "Chat", + "Client", + "Contact", + "DeltaChat", + "EventType", + "Message", + "Rpc", + "run_bot_cli", + "run_client_cli", +] diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py b/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py index 562c62de1..f53d19c92 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/_utils.py @@ -30,12 +30,7 @@ class AttrDict(dict): """Dictionary that allows accessing values usin the "dot notation" as attributes.""" def __init__(self, *args, **kwargs) -> None: - super().__init__( - { - _camel_to_snake(key): _to_attrdict(value) - for key, value in dict(*args, **kwargs).items() - } - ) + super().__init__({_camel_to_snake(key): _to_attrdict(value) for key, value in dict(*args, **kwargs).items()}) def __getattr__(self, attr): if attr in self: @@ -51,7 +46,7 @@ class AttrDict(dict): async def run_client_cli( hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None, argv: Optional[list] = None, - **kwargs + **kwargs, ) -> None: """Run a simple command line app, using the given hooks. @@ -65,7 +60,7 @@ async def run_client_cli( async def run_bot_cli( hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None, argv: Optional[list] = None, - **kwargs + **kwargs, ) -> None: """Run a simple bot command line using the given hooks. @@ -80,7 +75,7 @@ async def _run_cli( client_type: Type["Client"], hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None, argv: Optional[list] = None, - **kwargs + **kwargs, ) -> None: from .deltachat import DeltaChat from .rpc import Rpc @@ -107,12 +102,9 @@ async def _run_cli( client = client_type(account, hooks) client.logger.debug("Running deltachat core %s", core_version) if not await client.is_configured(): - assert ( - args.email and args.password - ), "Account is not configured and email and password must be provided" - asyncio.create_task( - client.configure(email=args.email, password=args.password) - ) + assert args.email, "Account is not configured and email must be provided" + assert args.password, "Account is not configured and password must be provided" + asyncio.create_task(client.configure(email=args.email, password=args.password)) await client.run_forever() diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/account.py b/deltachat-rpc-client/src/deltachat_rpc_client/account.py index eab67d10d..44535da08 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/account.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/account.py @@ -89,9 +89,7 @@ class Account: """Configure an account.""" await self._rpc.configure(self.id) - async def create_contact( - self, obj: Union[int, str, Contact], name: Optional[str] = None - ) -> Contact: + async def create_contact(self, obj: Union[int, str, Contact], name: Optional[str] = None) -> Contact: """Create a new Contact or return an existing one. Calling this method will always result in the same @@ -120,10 +118,7 @@ class Account: async def get_blocked_contacts(self) -> List[AttrDict]: """Return a list with snapshots of all blocked contacts.""" contacts = await self._rpc.get_blocked_contacts(self.id) - return [ - AttrDict(contact=Contact(self, contact["id"]), **contact) - for contact in contacts - ] + return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts] async def get_contacts( self, @@ -148,10 +143,7 @@ class Account: if snapshot: contacts = await self._rpc.get_contacts(self.id, flags, query) - return [ - AttrDict(contact=Contact(self, contact["id"]), **contact) - for contact in contacts - ] + return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts] contacts = await self._rpc.get_contact_ids(self.id, flags, query) return [Contact(self, contact_id) for contact_id in contacts] @@ -192,9 +184,7 @@ class Account: if alldone_hint: flags |= ChatlistFlag.ADD_ALLDONE_HINT - entries = await self._rpc.get_chatlist_entries( - self.id, flags, query, contact and contact.id - ) + entries = await self._rpc.get_chatlist_entries(self.id, flags, query, contact and contact.id) if not snapshot: return [Chat(self, entry[0]) for entry in entries] diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index 4a2f4ae77..cdcefcf95 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -63,7 +63,7 @@ class Chat: """ if duration is not None: assert duration > 0, "Invalid duration" - dur: Union[str, dict] = dict(Until=duration) + dur: Union[str, dict] = {"Until": duration} else: dur = "Forever" await self._rpc.set_chat_mute_duration(self.account.id, self.id, dur) @@ -74,27 +74,19 @@ class Chat: async def pin(self) -> None: """Pin this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.PINNED - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.PINNED) async def unpin(self) -> None: """Unpin this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.NORMAL - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL) async def archive(self) -> None: """Archive this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.ARCHIVED - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.ARCHIVED) async def unarchive(self) -> None: """Unarchive this chat.""" - await self._rpc.set_chat_visibility( - self.account.id, self.id, ChatVisibility.NORMAL - ) + await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL) async def set_name(self, name: str) -> None: """Set name of this chat.""" @@ -133,9 +125,7 @@ class Chat: if isinstance(quoted_msg, Message): quoted_msg = quoted_msg.id - msg_id, _ = await self._rpc.misc_send_msg( - self.account.id, self.id, text, file, location, quoted_msg - ) + msg_id, _ = await self._rpc.misc_send_msg(self.account.id, self.id, text, file, location, quoted_msg) return Message(self.account, msg_id) async def send_text(self, text: str) -> Message: @@ -241,23 +231,17 @@ class Chat: timestamp_to: Optional[datetime] = None, ) -> List[AttrDict]: """Get list of location snapshots for the given contact in the given timespan.""" - time_from = ( - calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0 - ) + time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0 time_to = calendar.timegm(timestamp_to.utctimetuple()) if timestamp_to else 0 contact_id = contact.id if contact else 0 - result = await self._rpc.get_locations( - self.account.id, self.id, contact_id, time_from, time_to - ) + result = await self._rpc.get_locations(self.account.id, self.id, contact_id, time_from, time_to) locations = [] contacts: Dict[int, Contact] = {} for loc in result: loc = AttrDict(loc) loc["chat"] = self - loc["contact"] = contacts.setdefault( - loc.contact_id, Contact(self.account, loc.contact_id) - ) + loc["contact"] = contacts.setdefault(loc.contact_id, Contact(self.account, loc.contact_id)) loc["message"] = Message(self.account, loc.msg_id) locations.append(loc) return locations diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/client.py b/deltachat-rpc-client/src/deltachat_rpc_client/client.py index 4c6deafa1..393ac3cc7 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/client.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/client.py @@ -47,15 +47,11 @@ class Client: self._should_process_messages = 0 self.add_hooks(hooks or []) - def add_hooks( - self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]] - ) -> None: + def add_hooks(self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]]) -> None: for hook, event in hooks: self.add_hook(hook, event) - def add_hook( - self, hook: Callable, event: Union[type, EventFilter] = RawEvent - ) -> None: + def add_hook(self, hook: Callable, event: Union[type, EventFilter] = RawEvent) -> None: """Register hook for the given event filter.""" if isinstance(event, type): event = event() @@ -64,7 +60,7 @@ class Client: isinstance( event, (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged), - ) + ), ) self._hooks.setdefault(type(event), set()).add((hook, event)) @@ -76,7 +72,7 @@ class Client: isinstance( event, (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged), - ) + ), ) self._hooks.get(type(event), set()).remove((hook, event)) @@ -95,9 +91,7 @@ class Client: """Process events forever.""" await self.run_until(lambda _: False) - async def run_until( - self, func: Callable[[AttrDict], Union[bool, Coroutine]] - ) -> AttrDict: + async def run_until(self, func: Callable[[AttrDict], Union[bool, Coroutine]]) -> AttrDict: """Process events until the given callable evaluates to True. The callable should accept an AttrDict object representing the @@ -122,9 +116,7 @@ class Client: if stop: return event - async def _on_event( - self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent - ) -> None: + async def _on_event(self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent) -> None: for hook, evfilter in self._hooks.get(filter_type, []): if await evfilter.filter(event): try: @@ -133,11 +125,7 @@ class Client: self.logger.exception(ex) async def _parse_command(self, event: AttrDict) -> None: - cmds = [ - hook[1].command - for hook in self._hooks.get(NewMessage, []) - if hook[1].command - ] + cmds = [hook[1].command for hook in self._hooks.get(NewMessage, []) if hook[1].command] parts = event.message_snapshot.text.split(maxsplit=1) payload = parts[1] if len(parts) > 1 else "" cmd = parts.pop(0) @@ -202,11 +190,7 @@ class Client: for message in await self.account.get_fresh_messages_in_arrival_order(): snapshot = await message.get_snapshot() await self._on_new_msg(snapshot) - if ( - snapshot.is_info - and snapshot.system_message_type - != SystemMessageType.WEBXDC_INFO_MESSAGE - ): + if snapshot.is_info and snapshot.system_message_type != SystemMessageType.WEBXDC_INFO_MESSAGE: await self._handle_info_msg(snapshot) await snapshot.message.mark_seen() diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/events.py b/deltachat-rpc-client/src/deltachat_rpc_client/events.py index 146c89ea5..3801dcc72 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/events.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/events.py @@ -10,7 +10,7 @@ from .const import EventType def _tuple_of(obj, type_: type) -> tuple: if not obj: - return tuple() + return () if isinstance(obj, type_): obj = (obj,) @@ -39,7 +39,7 @@ class EventFilter(ABC): """Return True if two event filters are equal.""" def __ne__(self, other): - return not self.__eq__(other) + return not self == other async def _call_func(self, event) -> bool: if not self.func: @@ -65,9 +65,7 @@ class RawEvent(EventFilter): should be dispatched or not. """ - def __init__( - self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs - ): + def __init__(self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs): super().__init__(**kwargs) try: self.types = _tuple_of(types, EventType) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/message.py b/deltachat-rpc-client/src/deltachat_rpc_client/message.py index 86ddbc95e..d38d2cbfe 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/message.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/message.py @@ -49,22 +49,14 @@ class Message: """Mark the message as seen.""" await self._rpc.markseen_msgs(self.account.id, [self.id]) - async def send_webxdc_status_update( - self, update: Union[dict, str], description: str - ) -> None: + async def send_webxdc_status_update(self, update: Union[dict, str], description: str) -> None: """Send a webxdc status update. This message must be a webxdc.""" if not isinstance(update, str): update = json.dumps(update) - await self._rpc.send_webxdc_status_update( - self.account.id, self.id, update, description - ) + await self._rpc.send_webxdc_status_update(self.account.id, self.id, update, description) async def get_webxdc_status_updates(self, last_known_serial: int = 0) -> list: - return json.loads( - await self._rpc.get_webxdc_status_updates( - self.account.id, self.id, last_known_serial - ) - ) + return json.loads(await self._rpc.get_webxdc_status_updates(self.account.id, self.id, last_known_serial)) async def get_webxdc_info(self) -> dict: return await self._rpc.get_webxdc_info(self.account.id, self.id) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py index 4b8562af7..36969c9c1 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py @@ -12,8 +12,11 @@ from .rpc import Rpc async def get_temp_credentials() -> dict: url = os.getenv("DCC_NEW_TMP_EMAIL") assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set" + + # Replace default 5 minute timeout with a 1 minute timeout. + timeout = aiohttp.ClientTimeout(total=60) async with aiohttp.ClientSession() as session: - async with session.post(url) as response: + async with session.post(url, timeout=timeout) as response: return json.loads(await response.text()) @@ -64,9 +67,7 @@ class ACFactory: ) -> Message: if not from_account: from_account = (await self.get_online_accounts(1))[0] - to_contact = await from_account.create_contact( - await to_account.get_config("addr") - ) + to_contact = await from_account.create_contact(await to_account.get_config("addr")) if group: to_chat = await from_account.create_group(group) await to_chat.add_contact(to_contact) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py b/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py index 8a407308d..f15c1a29a 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py @@ -30,7 +30,7 @@ class Rpc: "deltachat-rpc-server", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, - **self._kwargs + **self._kwargs, ) self.id = 0 self.event_queues = {} @@ -46,7 +46,7 @@ class Rpc: await self.start() return self - async def __aexit__(self, exc_type, exc, tb): + async def __aexit__(self, _exc_type, _exc, _tb): await self.close() async def reader_loop(self) -> None: @@ -97,5 +97,6 @@ class Rpc: raise JsonRpcError(response["error"]) if "result" in response: return response["result"] + return None return method diff --git a/deltachat-rpc-client/tests/test_something.py b/deltachat-rpc-client/tests/test_something.py index d1208042e..1d02e1928 100644 --- a/deltachat-rpc-client/tests/test_something.py +++ b/deltachat-rpc-client/tests/test_something.py @@ -6,14 +6,14 @@ from deltachat_rpc_client import EventType, events from deltachat_rpc_client.rpc import JsonRpcError -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_system_info(rpc) -> None: system_info = await rpc.get_system_info() assert "arch" in system_info assert "deltachat_core_version" in system_info -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_email_address_validity(rpc) -> None: valid_addresses = [ "email@example.com", @@ -27,7 +27,7 @@ async def test_email_address_validity(rpc) -> None: assert not await rpc.check_email_validity(addr) -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_acfactory(acfactory) -> None: account = await acfactory.new_configured_account() while True: @@ -41,7 +41,7 @@ async def test_acfactory(acfactory) -> None: print("Successful configuration") -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_configure_starttls(acfactory) -> None: account = await acfactory.new_preconfigured_account() @@ -51,7 +51,7 @@ async def test_configure_starttls(acfactory) -> None: assert await account.is_configured() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_account(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -111,7 +111,7 @@ async def test_account(acfactory) -> None: await alice.stop_io() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_chat(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -177,7 +177,7 @@ async def test_chat(acfactory) -> None: await group.get_locations() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_contact(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -195,7 +195,7 @@ async def test_contact(acfactory) -> None: await alice_contact_bob.create_chat() -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_message(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) @@ -226,7 +226,7 @@ async def test_message(acfactory) -> None: await message.send_reaction("😎") -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_bot(acfactory) -> None: mock = MagicMock() user = (await acfactory.get_online_accounts(1))[0] @@ -237,25 +237,20 @@ async def test_bot(acfactory) -> None: hook = lambda e: mock.hook(e.msg_id), events.RawEvent(EventType.INCOMING_MSG) bot.add_hook(*hook) - event = await acfactory.process_message( - from_account=user, to_client=bot, text="Hello!" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="Hello!") mock.hook.assert_called_once_with(event.msg_id) bot.remove_hook(*hook) - track = lambda e: mock.hook(e.message_snapshot.id) + def track(e): + mock.hook(e.message_snapshot.id) mock.hook.reset_mock() hook = track, events.NewMessage(r"hello") bot.add_hook(*hook) bot.add_hook(track, events.NewMessage(command="/help")) - event = await acfactory.process_message( - from_account=user, to_client=bot, text="hello" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="hello") mock.hook.assert_called_with(event.msg_id) - event = await acfactory.process_message( - from_account=user, to_client=bot, text="hello!" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="hello!") mock.hook.assert_called_with(event.msg_id) await acfactory.process_message(from_account=user, to_client=bot, text="hey!") assert len(mock.hook.mock_calls) == 2 @@ -263,7 +258,5 @@ async def test_bot(acfactory) -> None: mock.hook.reset_mock() await acfactory.process_message(from_account=user, to_client=bot, text="hello") - event = await acfactory.process_message( - from_account=user, to_client=bot, text="/help" - ) + event = await acfactory.process_message(from_account=user, to_client=bot, text="/help") mock.hook.assert_called_once_with(event.msg_id) diff --git a/deltachat-rpc-client/tests/test_webxdc.py b/deltachat-rpc-client/tests/test_webxdc.py index 1bc5f02c1..22d9db0b4 100644 --- a/deltachat-rpc-client/tests/test_webxdc.py +++ b/deltachat-rpc-client/tests/test_webxdc.py @@ -3,16 +3,14 @@ import pytest from deltachat_rpc_client import EventType -@pytest.mark.asyncio +@pytest.mark.asyncio() async def test_webxdc(acfactory) -> None: alice, bob = await acfactory.get_online_accounts(2) bob_addr = await bob.get_config("addr") alice_contact_bob = await alice.create_contact(bob_addr, "Bob") alice_chat_bob = await alice_contact_bob.create_chat() - await alice_chat_bob.send_message( - text="Let's play chess!", file="../test-data/webxdc/chess.xdc" - ) + await alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc") while True: event = await bob.wait_for_event() diff --git a/deltachat-rpc-client/tox.ini b/deltachat-rpc-client/tox.ini index bea3b4603..1c222e9a9 100644 --- a/deltachat-rpc-client/tox.ini +++ b/deltachat-rpc-client/tox.ini @@ -2,6 +2,7 @@ isolated_build = true envlist = py3 + lint [testenv] commands = @@ -16,3 +17,13 @@ deps = pytest-asyncio aiohttp aiodns + +[testenv:lint] +skipsdist = True +skip_install = True +deps = + ruff + black +commands = + black --check src/ examples/ tests/ + ruff src/ examples/ tests/ diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index b6f10996d..753b99c81 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.104.0" +version = "1.106.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" @@ -19,7 +19,7 @@ anyhow = "1" env_logger = { version = "0.10.0" } futures-lite = "1.12.0" log = "0.4" -serde_json = "1.0.89" +serde_json = "1.0.91" serde = { version = "1.0", features = ["derive"] } -tokio = { version = "1.23.0", features = ["io-std"] } +tokio = { version = "1.23.1", features = ["io-std"] } yerpc = { version = "0.3.1", features = ["anyhow_expose"] } diff --git a/deltachat_derive/src/lib.rs b/deltachat_derive/src/lib.rs index e24c5cf39..4fed46d5d 100644 --- a/deltachat_derive/src/lib.rs +++ b/deltachat_derive/src/lib.rs @@ -1,9 +1,10 @@ #![recursion_limit = "128"] extern crate proc_macro; -use crate::proc_macro::TokenStream; use quote::quote; +use crate::proc_macro::TokenStream; + // For now, assume (not check) that these macroses are applied to enum without // data. If this assumption is violated, compiler error will point to // generated code, which is not very user-friendly. diff --git a/examples/simple.rs b/examples/simple.rs index d14b1ac48..0dd754448 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -1,5 +1,3 @@ -use tempfile::tempdir; - use deltachat::chat::{self, ChatId}; use deltachat::chatlist::*; use deltachat::config; @@ -8,6 +6,7 @@ use deltachat::context::*; use deltachat::message::Message; use deltachat::stock_str::StockStrings; use deltachat::{EventType, Events}; +use tempfile::tempdir; fn cb(event: EventType) { match event { diff --git a/format-flowed/src/lib.rs b/format-flowed/src/lib.rs index 52b6827e5..10f9400b5 100644 --- a/format-flowed/src/lib.rs +++ b/format-flowed/src/lib.rs @@ -24,7 +24,7 @@ fn format_line_flowed(line: &str, prefix: &str) -> String { let mut result = String::new(); let mut buffer = prefix.to_string(); - let mut after_space = false; + let mut after_space = prefix.ends_with(' '); for c in line.chars() { if c == ' ' { @@ -55,7 +55,7 @@ fn format_line_flowed(line: &str, prefix: &str) -> String { result + &buffer } -/// Returns text formatted according to RFC 3767 (format=flowed). +/// Returns text formatted according to RFC 3676 (format=flowed). /// /// This function accepts text separated by LF, but returns text /// separated by CRLF. @@ -70,23 +70,20 @@ pub fn format_flowed(text: &str) -> String { result += "\r\n"; } - let line_no_prefix = line - .strip_prefix('>') - .map(|line| line.strip_prefix(' ').unwrap_or(line)); - let is_quote = line_no_prefix.is_some(); - let line = line_no_prefix.unwrap_or(line).trim_end(); - let prefix = if is_quote { "> " } else { "" }; + let line = line.trim_end(); + let quote_depth = line.chars().take_while(|&c| c == '>').count(); + let (prefix, mut line) = line.split_at(quote_depth); - if prefix.len() + line.len() > 78 { - result += &format_line_flowed(line, prefix); - } else { - result += prefix; - if prefix.is_empty() && (line.starts_with('>') || line.starts_with(' ')) { - // Space stuffing, see RFC 3676 - result.push(' '); + let mut prefix = prefix.to_string(); + + if quote_depth > 0 { + if let Some(s) = line.strip_prefix(' ') { + line = s; + prefix += " "; } - result += line; } + + result += &format_line_flowed(line, &prefix); } result @@ -111,9 +108,6 @@ pub fn format_flowed_quote(text: &str) -> String { /// /// Lines must be separated by single LF. /// -/// Quote processing is not supported, it is assumed that they are -/// deleted during simplification. -/// /// Signature separator line is not processed here, it is assumed to /// be stripped beforehand. pub fn unformat_flowed(text: &str, delsp: bool) -> String { @@ -121,6 +115,12 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String { let mut skip_newline = true; for line in text.split('\n') { + let line = if !result.is_empty() && skip_newline { + line.trim_start_matches('>') + } else { + line + }; + // Revert space-stuffing let line = line.strip_prefix(' ').unwrap_or(line); @@ -150,8 +150,20 @@ mod tests { #[test] fn test_format_flowed() { + let text = ""; + assert_eq!(format_flowed(text), ""); + let text = "Foo bar baz"; - assert_eq!(format_flowed(text), "Foo bar baz"); + assert_eq!(format_flowed(text), text); + + let text = ">Foo bar"; + assert_eq!(format_flowed(text), text); + + let text = "> Foo bar"; + assert_eq!(format_flowed(text), text); + + let text = ">\n\nA"; + assert_eq!(format_flowed(text), ">\r\n\r\nA"); let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\ \n\ @@ -165,17 +177,33 @@ mod tests { let text = "> A quote"; assert_eq!(format_flowed(text), "> A quote"); + let text = "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A"; + assert_eq!( + format_flowed(text), + "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > \r\n> A" + ); + // Test space stuffing of wrapped lines let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\ > \n\ > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."; let expected = "> This is the Autocrypt Setup Message used to transfer your key between \r\n\ > clients.\r\n\ - > \r\n\ + >\r\n\ > To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\ > client and enter the setup code presented on the generating device."; assert_eq!(format_flowed(text), expected); + let text = ">> This is the Autocrypt Setup Message used to transfer your key between clients.\n\ + >> \n\ + >> To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device."; + let expected = ">> This is the Autocrypt Setup Message used to transfer your key between \r\n\ + >> clients.\r\n\ + >>\r\n\ + >> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\ + >> client and enter the setup code presented on the generating device."; + assert_eq!(format_flowed(text), expected); + // Test space stuffing of spaces. let text = " Foo bar baz"; assert_eq!(format_flowed(text), " Foo bar baz"); @@ -202,6 +230,12 @@ mod tests { let text = " Foo bar"; let expected = " Foo bar"; assert_eq!(unformat_flowed(text, false), expected); + + let text = + "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > \n> A"; + let expected = + "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A"; + assert_eq!(unformat_flowed(text, false), expected); } #[test] diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index b2f3f5f36..46118154d 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -233,6 +233,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" + [[package]] name = "base64ct" version = "1.5.3" @@ -702,7 +708,7 @@ dependencies = [ "async-smtp", "async_zip", "backtrace", - "base64 0.13.1", + "base64 0.20.0", "bitflags", "chrono", "deltachat_derive", @@ -719,7 +725,7 @@ dependencies = [ "kamadak-exif", "lettre_email", "libc", - "mailparse", + "mailparse 0.14.0", "native-tls", "num-derive", "num-traits", @@ -764,7 +770,7 @@ dependencies = [ "bolero", "deltachat", "format-flowed", - "mailparse", + "mailparse 0.13.8", ] [[package]] @@ -1691,6 +1697,17 @@ dependencies = [ "quoted_printable", ] +[[package]] +name = "mailparse" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b56570f5f8c0047260d1c8b5b331f62eb9c660b9dd4071a8c46f8c7d3f280aa" +dependencies = [ + "charset", + "data-encoding", + "quoted_printable", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -1887,9 +1904,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "opaque-debug" @@ -2195,9 +2212,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" dependencies = [ "memchr", ] @@ -2846,9 +2863,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.0" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", diff --git a/fuzz/fuzz_targets/fuzz_format_flowed.rs b/fuzz/fuzz_targets/fuzz_format_flowed.rs index f44bebf94..8f779a468 100644 --- a/fuzz/fuzz_targets/fuzz_format_flowed.rs +++ b/fuzz/fuzz_targets/fuzz_format_flowed.rs @@ -10,10 +10,7 @@ fn round_trip(input: &str) -> String { fn main() { check!().for_each(|data: &[u8]| { if let Ok(input) = std::str::from_utf8(data.into()) { - let mut input = input.to_string(); - - // Only consider inputs that don't contain quotes. - input.retain(|c| c != '>'); + let input = input.trim().to_string(); // Only consider inputs that are the result of unformatting format=flowed text. // At least this means that lines don't contain any trailing whitespace. diff --git a/node/test/test.js b/node/test/test.js index 3d6c93de9..d76566b04 100644 --- a/node/test/test.js +++ b/node/test/test.js @@ -1,33 +1,29 @@ // @ts-check -import DeltaChat, { Message } from '../dist' -import binding from '../binding' +import DeltaChat from '../dist' -import { deepEqual, deepStrictEqual, strictEqual } from 'assert' +import { deepStrictEqual, strictEqual } from 'assert' import chai, { expect } from 'chai' import chaiAsPromised from 'chai-as-promised' import { EventId2EventName, C } from '../dist/constants' import { join } from 'path' -import { mkdtempSync, statSync } from 'fs' -import { tmpdir } from 'os' +import { statSync } from 'fs' import { Context } from '../dist/context' +import fetch from 'node-fetch' chai.use(chaiAsPromised) -chai.config.truncateThreshold = 0; // Do not truncate assertion errors. +chai.config.truncateThreshold = 0 // Do not truncate assertion errors. async function createTempUser(url) { - const fetch = require('node-fetch') - async function postData(url = '') { // Default options are marked with * const response = await fetch(url, { method: 'POST', // *GET, POST, PUT, DELETE, etc. - mode: 'cors', // no-cors, *cors, same-origin - cache: 'no-cache', // *default, no-cache, reload, force-cache, only-if-cached - credentials: 'same-origin', // include, *same-origin, omit headers: { 'cache-control': 'no-cache', }, - referrerPolicy: 'no-referrer', // no-referrer, *client }) + if (!response.ok) { + throw new Error('request failed: ' + response.body.read()) + } return response.json() // parses JSON response into native JavaScript objects } diff --git a/package.json b/package.json index 3122efe66..25771812e 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.104.0" + "version": "1.106.0" } \ No newline at end of file diff --git a/python/doc/conf.py b/python/doc/conf.py index 55fdcd5dd..4647c6ae2 100644 --- a/python/doc/conf.py +++ b/python/doc/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index c5ebaed42..59705c379 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -34,8 +34,10 @@ class GroupTrackingPlugin: def ac_member_added(self, chat, contact, actor, message): print( "ac_member_added {} to chat {} from {}".format( - contact.addr, chat.id, actor or message.get_sender_contact().addr - ) + contact.addr, + chat.id, + actor or message.get_sender_contact().addr, + ), ) for member in chat.get_contacts(): print("chat member: {}".format(member.addr)) @@ -44,8 +46,10 @@ class GroupTrackingPlugin: def ac_member_removed(self, chat, contact, actor, message): print( "ac_member_removed {} from chat {} by {}".format( - contact.addr, chat.id, actor or message.get_sender_contact().addr - ) + contact.addr, + chat.id, + actor or message.get_sender_contact().addr, + ), ) diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index 3978417bf..e6b2d53f4 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -13,8 +13,8 @@ def datadir(): datadir = path.join("test-data") if datadir.isdir(): return datadir - else: - pytest.skip("test-data directory not found") + pytest.skip("test-data directory not found") + return None def test_echo_quit_plugin(acfactory, lp): @@ -47,7 +47,7 @@ def test_group_tracking_plugin(acfactory, lp): botproc.fnmatch_lines( """ *ac_configure_completed* - """ + """, ) ac1.add_account_plugin(FFIEventLogger(ac1)) ac2.add_account_plugin(FFIEventLogger(ac2)) @@ -61,7 +61,7 @@ def test_group_tracking_plugin(acfactory, lp): botproc.fnmatch_lines( """ *ac_chat_modified*bot test group* - """ + """, ) lp.sec("adding third member {}".format(ac2.get_config("addr"))) @@ -76,8 +76,9 @@ def test_group_tracking_plugin(acfactory, lp): """ *ac_member_added {}*from*{}* """.format( - contact3.addr, ac1.get_config("addr") - ) + contact3.addr, + ac1.get_config("addr"), + ), ) lp.sec("contact successfully added, now removing") @@ -86,6 +87,7 @@ def test_group_tracking_plugin(acfactory, lp): """ *ac_member_removed {}*from*{}* """.format( - contact3.addr, ac1.get_config("addr") - ) + contact3.addr, + ac1.get_config("addr"), + ), ) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9ff0c63e6..18dfe3e9b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -44,5 +44,9 @@ git_describe_command = "git describe --dirty --tags --long --match py-*.*" [tool.black] line-length = 120 +[tool.ruff] +select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM"] +line-length = 120 + [tool.isort] -profile = "black" \ No newline at end of file +profile = "black" diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 8a15ac942..227f88977 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -36,7 +36,8 @@ register_global_plugin(events) def run_cmdline(argv=None, account_plugins=None): """Run a simple default command line app, registering the specified - account plugins.""" + account plugins. + """ import argparse if argv is None: diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 995dd34b8..9ace6fc0e 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -102,8 +102,8 @@ def find_header(flags): printf("%s", _dc_header_file_location()); return 0; } - """ - ) + """, + ), ) cwd = os.getcwd() try: @@ -198,7 +198,7 @@ def ffibuilder(): typedef int... time_t; void free(void *ptr); extern int dc_event_has_string_data(int); - """ + """, ) function_defs = extract_functions(flags) defines = extract_defines(flags) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 2df4bf65b..76b5bea88 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -1,4 +1,4 @@ -""" Account class implementation. """ +"""Account class implementation.""" from __future__ import print_function @@ -39,7 +39,7 @@ def get_core_info(): ffi.gc( lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL), lib.dc_context_unref, - ) + ), ) @@ -172,10 +172,7 @@ class Account(object): namebytes = name.encode("utf8") if isinstance(value, (int, bool)): value = str(int(value)) - if value is not None: - valuebytes = value.encode("utf8") - else: - valuebytes = ffi.NULL + valuebytes = value.encode("utf8") if value is not None else ffi.NULL lib.dc_set_config(self._dc_context, namebytes, valuebytes) def get_config(self, name: str) -> str: @@ -225,9 +222,10 @@ class Account(object): return bool(lib.dc_is_configured(self._dc_context)) def is_open(self) -> bool: - """Determine if account is open + """Determine if account is open. - :returns True if account is open.""" + :returns True if account is open. + """ return bool(lib.dc_context_is_open(self._dc_context)) def set_avatar(self, img_path: Optional[str]) -> None: @@ -543,7 +541,7 @@ class Account(object): return from_dc_charpointer(res) def check_qr(self, qr): - """check qr code and return :class:`ScannedQRCode` instance representing the result""" + """check qr code and return :class:`ScannedQRCode` instance representing the result.""" res = ffi.gc(lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)), lib.dc_lot_unref) lot = DCLot(res) if lot.state() == const.DC_QR_ERROR: @@ -662,7 +660,7 @@ class Account(object): return lib.dc_all_work_done(self._dc_context) def start_io(self): - """start this account's IO scheduling (Rust-core async scheduler) + """start this account's IO scheduling (Rust-core async scheduler). If this account is not configured an Exception is raised. You need to call account.configure() and account.wait_configure_finish() @@ -705,12 +703,10 @@ class Account(object): """ lib.dc_maybe_network(self._dc_context) - def configure(self, reconfigure: bool = False) -> ConfigureTracker: + def configure(self) -> ConfigureTracker: """Start configuration process and return a Configtracker instance on which you can block with wait_finish() to get a True/False success value for the configuration process. - - :param reconfigure: deprecated, doesn't need to be checked anymore. """ if not self.get_config("addr") or not self.get_config("mail_pw"): raise MissingCredentials("addr or mail_pwd not set in config") @@ -733,7 +729,8 @@ class Account(object): def shutdown(self) -> None: """shutdown and destroy account (stop callback thread, close and remove - underlying dc_context).""" + underlying dc_context). + """ if self._dc_context is None: return diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index d546d4775..783f25304 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -1,4 +1,4 @@ -""" Chat and Location related API. """ +"""Chat and Location related API.""" import calendar import json @@ -37,7 +37,7 @@ class Chat(object): return self.id == getattr(other, "id", None) and self.account._dc_context == other.account._dc_context def __ne__(self, other) -> bool: - return not (self == other) + return not self == other def __repr__(self) -> str: return "".format(self.id, self.get_name()) @@ -74,19 +74,19 @@ class Chat(object): return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP def is_single(self) -> bool: - """Return True if this chat is a single/direct chat, False otherwise""" + """Return True if this chat is a single/direct chat, False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_SINGLE def is_mailinglist(self) -> bool: - """Return True if this chat is a mailing list, False otherwise""" + """Return True if this chat is a mailing list, False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_MAILINGLIST def is_broadcast(self) -> bool: - """Return True if this chat is a broadcast list, False otherwise""" + """Return True if this chat is a broadcast list, False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_BROADCAST def is_multiuser(self) -> bool: - """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise""" + """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise.""" return lib.dc_chat_get_type(self._dc_chat) in ( const.DC_CHAT_TYPE_GROUP, const.DC_CHAT_TYPE_MAILINGLIST, @@ -94,11 +94,11 @@ class Chat(object): ) def is_self_talk(self) -> bool: - """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise""" + """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise.""" return bool(lib.dc_chat_is_self_talk(self._dc_chat)) def is_device_talk(self) -> bool: - """Returns True if this chat is the "Device Messages" chat, False otherwise""" + """Returns True if this chat is the "Device Messages" chat, False otherwise.""" return bool(lib.dc_chat_is_device_talk(self._dc_chat)) def is_muted(self) -> bool: @@ -109,12 +109,12 @@ class Chat(object): return bool(lib.dc_chat_is_muted(self._dc_chat)) def is_pinned(self) -> bool: - """Return True if this chat is pinned, False otherwise""" + """Return True if this chat is pinned, False otherwise.""" return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_PINNED def is_archived(self) -> bool: """Return True if this chat is archived, False otherwise. - :returns: True if archived, False otherwise + :returns: True if archived, False otherwise. """ return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED @@ -136,7 +136,7 @@ class Chat(object): def can_send(self) -> bool: """Check if messages can be sent to a give chat. - This is not true eg. for the contact requests or for the device-talk + This is not true eg. for the contact requests or for the device-talk. :returns: True if the chat is writable, False otherwise """ @@ -167,7 +167,7 @@ class Chat(object): def get_color(self): """return the color of the chat. - :returns: color as 0x00rrggbb + :returns: color as 0x00rrggbb. """ return lib.dc_chat_get_color(self._dc_chat) @@ -178,21 +178,18 @@ class Chat(object): return json.loads(s) def mute(self, duration: Optional[int] = None) -> None: - """mutes the chat + """mutes the chat. :param duration: Number of seconds to mute the chat for. None to mute until unmuted again. :returns: None """ - if duration is None: - mute_duration = -1 - else: - mute_duration = duration + mute_duration = -1 if duration is None else duration ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration) if not bool(ret): raise ValueError("Call to dc_set_chat_mute_duration failed") def unmute(self) -> None: - """unmutes the chat + """unmutes the chat. :returns: None """ @@ -252,7 +249,8 @@ class Chat(object): def get_encryption_info(self) -> Optional[str]: """Return encryption info for this chat. - :returns: a string with encryption preferences of all chat members""" + :returns: a string with encryption preferences of all chat members + """ res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id) return from_dc_charpointer(res) @@ -463,7 +461,7 @@ class Chat(object): def get_contacts(self): """get all contacts for this chat. - :returns: list of :class:`deltachat.contact.Contact` objects for this chat + :returns: list of :class:`deltachat.contact.Contact` objects for this chat. """ from .contact import Contact @@ -547,19 +545,10 @@ class Chat(object): :param timespan_to: a datetime object or None (indicating up till now) :returns: list of :class:`deltachat.chat.Location` objects. """ - if timestamp_from is None: - time_from = 0 - else: - time_from = calendar.timegm(timestamp_from.utctimetuple()) - if timestamp_to is None: - time_to = 0 - else: - time_to = calendar.timegm(timestamp_to.utctimetuple()) + time_from = 0 if timestamp_from is None else calendar.timegm(timestamp_from.utctimetuple()) + time_to = 0 if timestamp_to is None else calendar.timegm(timestamp_to.utctimetuple()) - if contact is None: - contact_id = 0 - else: - contact_id = contact.id + contact_id = 0 if contact is None else contact.id dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to) return [ diff --git a/python/src/deltachat/contact.py b/python/src/deltachat/contact.py index 75c44b8e5..adf4cdbad 100644 --- a/python/src/deltachat/contact.py +++ b/python/src/deltachat/contact.py @@ -1,4 +1,4 @@ -""" Contact object. """ +"""Contact object.""" from datetime import date, datetime, timezone from typing import Optional @@ -28,7 +28,7 @@ class Contact(object): return self.account._dc_context == other.account._dc_context and self.id == other.id def __ne__(self, other): - return not (self == other) + return not self == other def __repr__(self): return "".format(self.id, self.addr, self.account._dc_context) @@ -76,7 +76,7 @@ class Contact(object): return lib.dc_contact_is_verified(self._dc_contact) def get_verifier(self, contact): - """Return the address of the contact that verified the contact""" + """Return the address of the contact that verified the contact.""" return from_dc_charpointer(lib.dc_contact_get_verifier_addr(contact._dc_contact)) def get_profile_image(self) -> Optional[str]: diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index 5c36d47f1..f2f4417c5 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -79,15 +79,17 @@ class DirectImap: def select_config_folder(self, config_name: str): """Return info about selected folder if it is - configured, otherwise None.""" + configured, otherwise None. + """ if "_" not in config_name: config_name = "configured_{}_folder".format(config_name) foldername = self.account.get_config(config_name) if foldername: return self.select_folder(foldername) + return None def list_folders(self) -> List[str]: - """return list of all existing folder names""" + """return list of all existing folder names.""" assert not self._idling return [folder.name for folder in self.conn.folder.list()] @@ -103,7 +105,7 @@ class DirectImap: def get_all_messages(self) -> List[MailMessage]: assert not self._idling - return [mail for mail in self.conn.fetch()] + return list(self.conn.fetch()) def get_unread_messages(self) -> List[str]: assert not self._idling @@ -221,5 +223,4 @@ class IdleManager: def done(self): """send idle-done to server if we are currently in idle mode.""" - res = self.direct_imap.conn.idle.stop() - return res + return self.direct_imap.conn.idle.stop() diff --git a/python/src/deltachat/events.py b/python/src/deltachat/events.py index 72b84a44a..9cd1caefe 100644 --- a/python/src/deltachat/events.py +++ b/python/src/deltachat/events.py @@ -30,6 +30,12 @@ class FFIEvent: self.data2 = data2 def __str__(self): + if self.name == "DC_EVENT_INFO": + return "INFO {data2}".format(data2=self.data2) + if self.name == "DC_EVENT_WARNING": + return "WARNING {data2}".format(data2=self.data2) + if self.name == "DC_EVENT_ERROR": + return "ERROR {data2}".format(data2=self.data2) return "{name} data1={data1} data2={data2}".format(**self.__dict__) @@ -128,7 +134,8 @@ class FFIEventTracker: def wait_for_connectivity(self, connectivity): """Wait for the specified connectivity. This only works reliably if the connectivity doesn't change - again too quickly, otherwise we might miss it.""" + again too quickly, otherwise we might miss it. + """ while 1: if self.account.get_connectivity() == connectivity: return @@ -136,12 +143,13 @@ class FFIEventTracker: def wait_for_connectivity_change(self, previous, expected_next): """Wait until the connectivity changes to `expected_next`. - Fails the test if it changes to something else.""" + Fails the test if it changes to something else. + """ while 1: current = self.account.get_connectivity() if current == expected_next: return - elif current != previous: + if current != previous: raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current)) self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED") @@ -176,7 +184,8 @@ class FFIEventTracker: - ac1 and ac2 are created - ac1 sends a message to ac2 - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message - - therefore no DC_EVENT_INCOMING_MSG is sent""" + - therefore no DC_EVENT_INCOMING_MSG is sent + """ self.get_info_contains("INBOX: Idle entering") def wait_next_incoming_message(self): @@ -186,14 +195,15 @@ class FFIEventTracker: def wait_next_messages_changed(self): """wait for and return next message-changed message or None - if the event contains no msgid""" + if the event contains no msgid + """ ev = self.get_matching("DC_EVENT_MSGS_CHANGED") if ev.data2 > 0: return self.account.get_message_by_id(ev.data2) return None def wait_next_reactions_changed(self): - """wait for and return next reactions-changed message""" + """wait for and return next reactions-changed message.""" ev = self.get_matching("DC_EVENT_REACTIONS_CHANGED") assert ev.data1 > 0 return self.account.get_message_by_id(ev.data2) @@ -285,10 +295,10 @@ class EventThread(threading.Thread): if data1 == 0 or data1 == 1000: success = data1 == 1000 comment = ffi_event.data2 - yield "ac_configure_completed", dict(success=success, comment=comment) + yield "ac_configure_completed", {"success": success, "comment": comment} elif name == "DC_EVENT_INCOMING_MSG": msg = account.get_message_by_id(ffi_event.data2) - yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg)) + yield map_system_message(msg) or ("ac_incoming_message", {"message": msg}) elif name == "DC_EVENT_MSGS_CHANGED": if ffi_event.data2 != 0: msg = account.get_message_by_id(ffi_event.data2) @@ -296,19 +306,19 @@ class EventThread(threading.Thread): res = map_system_message(msg) if res and res[0].startswith("ac_member"): yield res - yield "ac_outgoing_message", dict(message=msg) + yield "ac_outgoing_message", {"message": msg} elif msg.is_in_fresh(): yield map_system_message(msg) or ( "ac_incoming_message", - dict(message=msg), + {"message": msg}, ) elif name == "DC_EVENT_REACTIONS_CHANGED": assert ffi_event.data1 > 0 msg = account.get_message_by_id(ffi_event.data2) - yield "ac_reactions_changed", dict(message=msg) + yield "ac_reactions_changed", {"message": msg} elif name == "DC_EVENT_MSG_DELIVERED": msg = account.get_message_by_id(ffi_event.data2) - yield "ac_message_delivered", dict(message=msg) + yield "ac_message_delivered", {"message": msg} elif name == "DC_EVENT_CHAT_MODIFIED": chat = account.get_chat_by_id(ffi_event.data1) - yield "ac_chat_modified", dict(chat=chat) + yield "ac_chat_modified", {"chat": chat} diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 4d2fe6960..51d4a9c83 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -1,4 +1,4 @@ -""" Hooks for Python bindings to Delta Chat Core Rust CFFI""" +"""Hooks for Python bindings to Delta Chat Core Rust CFFI.""" import pluggy diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 0891b46cf..0e083ad86 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -1,4 +1,4 @@ -""" The Message object. """ +"""The Message object.""" import json import os @@ -59,10 +59,7 @@ class Message(object): :param view_type: the message type code or one of the strings: "text", "audio", "video", "file", "sticker", "videochat", "webxdc" """ - if isinstance(view_type, int): - view_type_code = view_type - else: - view_type_code = get_viewtype_code_from_name(view_type) + view_type_code = view_type if isinstance(view_type, int) else get_viewtype_code_from_name(view_type) return Message( account, ffi.gc(lib.dc_msg_new(account._dc_context, view_type_code), lib.dc_msg_unref), @@ -129,7 +126,7 @@ class Message(object): @props.with_doc def filemime(self) -> str: - """mime type of the file (if it exists)""" + """mime type of the file (if it exists).""" return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg)) def get_status_updates(self, serial: int = 0) -> list: @@ -141,7 +138,7 @@ class Message(object): :param serial: The last known serial. Pass 0 if there are no known serials to receive all updates. """ return json.loads( - from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial)) + from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial)), ) def send_status_update(self, json_data: Union[str, dict], description: str) -> bool: @@ -158,8 +155,11 @@ class Message(object): json_data = json.dumps(json_data, default=str) return bool( lib.dc_send_webxdc_status_update( - self.account._dc_context, self.id, as_dc_charpointer(json_data), as_dc_charpointer(description) - ) + self.account._dc_context, + self.id, + as_dc_charpointer(json_data), + as_dc_charpointer(description), + ), ) def send_reaction(self, reaction: str): @@ -232,16 +232,18 @@ class Message(object): ts = lib.dc_msg_get_received_timestamp(self._dc_msg) if ts: return datetime.fromtimestamp(ts, timezone.utc) + return None @props.with_doc def ephemeral_timer(self): - """Ephemeral timer in seconds + """Ephemeral timer in seconds. :returns: timer in seconds or None if there is no timer """ timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg) if timer: return timer + return None @props.with_doc def ephemeral_timestamp(self): @@ -255,23 +257,25 @@ class Message(object): @property def quoted_text(self) -> Optional[str]: - """Text inside the quote + """Text inside the quote. - :returns: Quoted text""" + :returns: Quoted text + """ return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg)) @property def quote(self): - """Quote getter + """Quote getter. - :returns: Quoted message, if found in the database""" + :returns: Quoted message, if found in the database + """ msg = lib.dc_msg_get_quoted_msg(self._dc_msg) if msg: return Message(self.account, ffi.gc(msg, lib.dc_msg_unref)) @quote.setter def quote(self, quoted_message): - """Quote setter""" + """Quote setter.""" lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg) def force_plaintext(self) -> None: @@ -286,7 +290,7 @@ class Message(object): :returns: email-mime message object (with headers only, no body). """ - import email.parser + import email mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id) if mime_headers: @@ -297,7 +301,7 @@ class Message(object): @property def error(self) -> Optional[str]: - """Error message""" + """Error message.""" return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg)) @property @@ -493,7 +497,8 @@ def get_viewtype_code_from_name(view_type_name): if code is not None: return code raise ValueError( - "message typecode not found for {!r}, " "available {!r}".format(view_type_name, list(_view_type_mapping.keys())) + "message typecode not found for {!r}, " + "available {!r}".format(view_type_name, list(_view_type_mapping.keys())), ) @@ -506,14 +511,11 @@ def map_system_message(msg): if msg.is_system_message(): res = parse_system_add_remove(msg.text) if not res: - return + return None action, affected, actor = res affected = msg.account.get_contact_by_addr(affected) - if actor == "me": - actor = None - else: - actor = msg.account.get_contact_by_addr(actor) - d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg) + actor = None if actor == "me" else msg.account.get_contact_by_addr(actor) + d = {"chat": msg.chat, "contact": affected, "actor": actor, "message": msg} return "ac_member_" + res[0], d @@ -528,8 +530,8 @@ def extract_addr(text): def parse_system_add_remove(text): """return add/remove info from parsing the given system message text. - returns a (action, affected, actor) triple""" - + returns a (action, affected, actor) triple + """ # You removed member a@b. # You added member a@b. # Member Me (x@y) removed by a@b. diff --git a/python/src/deltachat/props.py b/python/src/deltachat/props.py index ab21794ae..7dcf4a1aa 100644 --- a/python/src/deltachat/props.py +++ b/python/src/deltachat/props.py @@ -8,7 +8,7 @@ def with_doc(f): # copied over unmodified from # https://github.com/devpi/devpi/blob/master/common/devpi_common/types.py def cached(f): - """returns a cached property that is calculated by function f""" + """returns a cached property that is calculated by function f.""" def get(self): try: @@ -17,8 +17,9 @@ def cached(f): self._property_cache = {} except KeyError: pass - x = self._property_cache[f] = f(self) - return x + res = f(self) + self._property_cache[f] = res + return res def set(self, val): propcache = self.__dict__.setdefault("_property_cache", {}) diff --git a/python/src/deltachat/provider.py b/python/src/deltachat/provider.py index 76d5cf780..d760dbabb 100644 --- a/python/src/deltachat/provider.py +++ b/python/src/deltachat/provider.py @@ -9,7 +9,8 @@ class ProviderNotFoundError(Exception): class Provider(object): - """Provider information. + """ + Provider information. :param domain: The email to get the provider info for. """ diff --git a/python/src/deltachat/reactions.py b/python/src/deltachat/reactions.py index 9e9ed9555..92cdf1905 100644 --- a/python/src/deltachat/reactions.py +++ b/python/src/deltachat/reactions.py @@ -1,4 +1,4 @@ -""" The Reactions object. """ +"""The Reactions object.""" from .capi import ffi, lib from .cutil import from_dc_charpointer, iter_array diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 335e384a5..b61c1a0e4 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -29,7 +29,7 @@ def pytest_addoption(parser): "--liveconfig", action="store", default=None, - help="a file with >=2 lines where each line " "contains NAME=VALUE config settings for one account", + help="a file with >=2 lines where each line contains NAME=VALUE config settings for one account", ) group.addoption( "--ignored", @@ -124,7 +124,7 @@ def pytest_report_header(config, startdir): info["deltachat_core_version"], info["sqlite_version"], info["journal_mode"], - ) + ), ] cfg = config.option.liveconfig @@ -176,11 +176,11 @@ class TestProcess: try: yield self._configlist[index] except IndexError: - res = requests.post(liveconfig_opt) + res = requests.post(liveconfig_opt, timeout=60) if res.status_code != 200: pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text)) d = res.json() - config = dict(addr=d["email"], mail_pw=d["password"]) + config = {"addr": d["email"], "mail_pw": d["password"]} print("newtmpuser {}: addr={}".format(index, config["addr"])) self._configlist.append(config) yield config @@ -229,7 +229,7 @@ def write_dict_to_dir(dic, target_dir): path.write_bytes(content) -@pytest.fixture +@pytest.fixture() def data(request): class Data: def __init__(self) -> None: @@ -253,6 +253,7 @@ def data(request): if os.path.exists(fn): return fn print("WARNING: path does not exist: {!r}".format(fn)) + return None def read_path(self, bn, mode="r"): fn = self.get_path(bn) @@ -264,8 +265,11 @@ def data(request): class ACSetup: - """accounts setup helper to deal with multiple configure-process - and io & imap initialization phases. From tests, use the higher level + """ + Accounts setup helper to deal with multiple configure-process + and io & imap initialization phases. + + From tests, use the higher level public ACFactory methods instead of its private helper class. """ @@ -289,7 +293,7 @@ class ACSetup: self._account2state[account] = self.CONFIGURED self.log("added already configured account", account, account.get_config("addr")) - def start_configure(self, account, reconfigure=False): + def start_configure(self, account): """add an account and start its configure process.""" class PendingTracker: @@ -299,7 +303,7 @@ class ACSetup: account.add_account_plugin(PendingTracker(), name="pending_tracker") self._account2state[account] = self.CONFIGURING - account.configure(reconfigure=reconfigure) + account.configure() self.log("started configure on", account) def wait_one_configured(self, account): @@ -411,7 +415,8 @@ class ACFactory: acc.disable_logging() def get_next_liveconfig(self): - """Base function to get functional online configurations + """ + Base function to get functional online configurations where we can make valid SMTP and IMAP connections with. """ configdict = next(self._liveconfig_producer).copy() @@ -465,8 +470,7 @@ class ACFactory: if fname_pub and fname_sec: account._preconfigure_keypair(addr, fname_pub, fname_sec) return True - else: - print("WARN: could not use preconfigured keys for {!r}".format(addr)) + print("WARN: could not use preconfigured keys for {!r}".format(addr)) def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account: # do a pseudo-configured account @@ -476,14 +480,14 @@ class ACFactory: acname = ac._logid addr = "{}@offline.org".format(acname) ac.update_config( - dict( - addr=addr, - displayname=acname, - mail_pw="123", - configured_addr=addr, - configured_mail_pw="123", - configured="1", - ) + { + "addr": addr, + "displayname": acname, + "mail_pw": "123", + "configured_addr": addr, + "configured_mail_pw": "123", + "configured": "1", + }, ) self._preconfigure_key(ac, addr) self._acsetup.init_logging(ac) @@ -494,12 +498,12 @@ class ACFactory: configdict = self.get_next_liveconfig() else: # XXX we might want to transfer the key to the new account - configdict = dict( - addr=cloned_from.get_config("addr"), - mail_pw=cloned_from.get_config("mail_pw"), - imap_certificate_checks=cloned_from.get_config("imap_certificate_checks"), - smtp_certificate_checks=cloned_from.get_config("smtp_certificate_checks"), - ) + configdict = { + "addr": cloned_from.get_config("addr"), + "mail_pw": cloned_from.get_config("mail_pw"), + "imap_certificate_checks": cloned_from.get_config("imap_certificate_checks"), + "smtp_certificate_checks": cloned_from.get_config("smtp_certificate_checks"), + } configdict.update(kwargs) ac = self._get_cached_account(addr=configdict["addr"]) if cache else None if ac is not None: @@ -600,7 +604,7 @@ class ACFactory: acc._evtracker.wait_next_incoming_message() -@pytest.fixture +@pytest.fixture() def acfactory(request, tmpdir, testprocess, data): am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testprocess, data=data) yield am @@ -665,12 +669,12 @@ class BotProcess: ignored.append(line) -@pytest.fixture +@pytest.fixture() def tmp_db_path(tmpdir): return tmpdir.join("test.db").strpath -@pytest.fixture +@pytest.fixture() def lp(): class Printer: def sec(self, msg: str) -> None: diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 9f9cb4c53..199e5d5c9 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -77,11 +77,11 @@ class ConfigureTracker: self.account.remove_account_plugin(self) def wait_smtp_connected(self): - """wait until smtp is configured.""" + """Wait until SMTP is configured.""" self._smtp_finished.wait() def wait_imap_connected(self): - """wait until smtp is configured.""" + """Wait until IMAP is configured.""" self._imap_finished.wait() def wait_progress(self, data1=None): @@ -91,7 +91,8 @@ class ConfigureTracker: break def wait_finish(self, timeout=None): - """wait until configure is completed. + """ + Wait until configure is completed. Raise Exception if Configure failed """ diff --git a/python/tests/auditwheels.py b/python/tests/auditwheels.py index 1c34ab692..a56ca5821 100644 --- a/python/tests/auditwheels.py +++ b/python/tests/auditwheels.py @@ -15,5 +15,5 @@ if __name__ == "__main__": p, "-w", workspacedir, - ] + ], ) diff --git a/python/tests/test_0_complex_or_slow.py b/python/tests/test_0_complex_or_slow.py index 034839ee5..49e0ddf38 100644 --- a/python/tests/test_0_complex_or_slow.py +++ b/python/tests/test_0_complex_or_slow.py @@ -216,7 +216,7 @@ def test_fetch_existing(acfactory, lp, mvbox_move): # would also find the "Sent" folder, but it would be too late: # The sentbox thread, started by `start_io()`, would have seen that there is no # ConfiguredSentboxFolder and do nothing. - acfactory._acsetup.start_configure(ac1, reconfigure=True) + acfactory._acsetup.start_configure(ac1) acfactory.bring_accounts_online() assert_folders_configured(ac1) @@ -492,3 +492,48 @@ def test_multidevice_sync_seen(acfactory, lp): assert ac1_clone_message.is_in_seen # Test that the timer is started on the second device after synchronizing the seen status. assert "Expires: " in ac1_clone_message.get_message_info() + + +def test_see_new_verified_member_after_going_online(acfactory, tmpdir, lp): + """The test for the bug #3836: + - Alice has two devices, the second is offline. + - Alice creates a verified group and sends a QR invitation to Bob. + - Bob joins the group and sends a message there. Alice sees it. + - Alice's second devices goes online, but doesn't see Bob in the group. + """ + ac1, ac2 = acfactory.get_online_accounts(2) + ac2_addr = ac2.get_config("addr") + ac1_offl = acfactory.new_online_configuring_account(cloned_from=ac1) + for ac in [ac1, ac1_offl]: + ac.set_config("bcc_self", "1") + acfactory.bring_accounts_online() + dir = tmpdir.mkdir("exportdir") + ac1.export_self_keys(dir.strpath) + ac1_offl.import_self_keys(dir.strpath) + ac1_offl.stop_io() + + lp.sec("ac1: create verified-group QR, ac2 scans and joins") + chat = ac1.create_group_chat("hello", verified=True) + assert chat.is_protected() + qr = chat.get_join_qr() + lp.sec("ac2: start QR-code based join-group protocol") + chat2 = ac2.qr_join_chat(qr) + ac1._evtracker.wait_securejoin_inviter_progress(1000) + + lp.sec("ac2: sending message") + # Message can be sent only after a receipt of "vg-member-added" message. Just wait for + # "Member Me () added by ." message. + ac2._evtracker.wait_next_incoming_message() + msg_out = chat2.send_text("hello") + + lp.sec("ac1: receiving message") + msg_in = ac1._evtracker.wait_next_incoming_message() + assert msg_in.text == msg_out.text + assert msg_in.get_sender_contact().addr == ac2_addr + + lp.sec("ac1_offl: going online, receiving message") + ac1_offl.start_io() + ac1_offl._evtracker.wait_securejoin_inviter_progress(1000) + msg_in = ac1_offl._evtracker.wait_next_incoming_message() + assert msg_in.text == msg_out.text + assert msg_in.get_sender_contact().addr == ac2_addr diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 737bef158..5c2065644 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -32,7 +32,7 @@ def test_basic_imap_api(acfactory, tmpdir): imap2.shutdown() -@pytest.mark.ignored +@pytest.mark.ignored() def test_configure_generate_key(acfactory, lp): # A slow test which will generate new keys. acfactory.remove_preconfigured_keys() @@ -88,7 +88,7 @@ def test_export_import_self_keys(acfactory, tmpdir, lp): lp.indent(dir.strpath + os.sep + name) lp.sec("importing into existing account") ac2.import_self_keys(dir.strpath) - (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*", check_error=False) + (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*") assert key_id2 == key_id @@ -510,7 +510,7 @@ def test_send_and_receive_message_markseen(acfactory, lp): idle2.wait_for_seen() lp.step("1") - for i in range(2): + for _i in range(2): ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ") assert ev.data1 > const.DC_CHAT_ID_LAST_SPECIAL assert ev.data2 > const.DC_MSG_ID_LAST_SPECIAL @@ -529,7 +529,7 @@ def test_send_and_receive_message_markseen(acfactory, lp): pass # mark_seen_messages() has generated events before it returns -def test_moved_markseen(acfactory, lp): +def test_moved_markseen(acfactory): """Test that message already moved to DeltaChat folder is marked as seen.""" ac1 = acfactory.new_online_configuring_account() ac2 = acfactory.new_online_configuring_account(mvbox_move=True) @@ -553,7 +553,7 @@ def test_moved_markseen(acfactory, lp): ac2.mark_seen_messages([msg]) uid = idle2.wait_for_seen() - assert len([a for a in ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*")))]) == 1 + assert len(list(ac2.direct_imap.conn.fetch(AND(seen=True, uid=U(uid, "*"))))) == 1 def test_message_override_sender_name(acfactory, lp): @@ -832,7 +832,7 @@ def test_send_first_message_as_long_unicode_with_cr(acfactory, lp): lp.sec("sending multi-line non-unicode message from ac1 to ac2") text1 = ( "hello\nworld\nthis is a very long message that should be" - + " wrapped using format=flowed and unwrapped on the receiver" + " wrapped using format=flowed and unwrapped on the receiver" ) msg_out = chat.send_text(text1) assert not msg_out.is_encrypted() @@ -894,7 +894,7 @@ def test_dont_show_emails(acfactory, lp): message in Drafts that is moved to Sent later """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -908,7 +908,7 @@ def test_dont_show_emails(acfactory, lp): message in Sent """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -922,7 +922,7 @@ def test_dont_show_emails(acfactory, lp): Unknown message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -936,7 +936,21 @@ def test_dont_show_emails(acfactory, lp): Unknown & malformed message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), + ), + ) + ac1.direct_imap.append( + "Spam", + """ + From: delta + Subject: subj + To: {} + Message-ID: + Content-Type: text/plain; charset=utf-8 + + Unknown & malformed message in Spam + """.format( + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -950,7 +964,7 @@ def test_dont_show_emails(acfactory, lp): Actually interesting message in Spam """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) ac1.direct_imap.append( @@ -964,7 +978,7 @@ def test_dont_show_emails(acfactory, lp): Unknown message in Junk """.format( - ac1.get_config("configured_addr") + ac1.get_config("configured_addr"), ), ) @@ -1698,7 +1712,7 @@ def test_system_group_msg_from_blocked_user(acfactory, lp): assert contact.is_blocked() chat_on_ac2.remove_contact(ac1) ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED") - assert not ac1.get_self_contact() in chat_on_ac1.get_contacts() + assert ac1.get_self_contact() not in chat_on_ac1.get_contacts() def test_set_get_group_image(acfactory, data, lp): @@ -1772,7 +1786,7 @@ def test_connectivity(acfactory, lp): lp.sec( "Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " - + "all messages are fetched" + "all messages are fetched", ) ac1.direct_imap.select_config_folder("inbox") @@ -2134,7 +2148,7 @@ def test_group_quote(acfactory, lp): @pytest.mark.parametrize( - "folder,move,expected_destination,", + ("folder", "move", "expected_destination"), [ ( "xyz", @@ -2249,11 +2263,44 @@ def test_aeap_flow_verified(acfactory, lp): assert ac1new.get_config("addr") in [contact.addr for contact in msg_in_2.chat.get_contacts()] +def test_archived_muted_chat(acfactory, lp): + """If an archived and muted chat receives a new message, DC_EVENT_MSGS_CHANGED for + DC_CHAT_ID_ARCHIVED_LINK must be generated if the chat had only seen messages previously. + """ + ac1, ac2 = acfactory.get_online_accounts(2) + chat = acfactory.get_accepted_chat(ac1, ac2) + + lp.sec("ac1: send message to ac2") + chat.send_text("message0") + + lp.sec("wait for ac2 to receive message") + msg2 = ac2._evtracker.wait_next_incoming_message() + assert msg2.text == "message0" + msg2.mark_seen() + + chat2 = msg2.chat + chat2.archive() + chat2.mute() + + lp.sec("ac1: send another message to ac2") + chat.send_text("message1") + + lp.sec("wait for ac2 to receive DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK") + while 1: + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") + if ev.data1 == const.DC_CHAT_ID_ARCHIVED_LINK: + assert ev.data2 == 0 + archive = ac2.get_chat_by_id(const.DC_CHAT_ID_ARCHIVED_LINK) + assert archive.count_fresh_messages() == 1 + assert chat2.count_fresh_messages() == 1 + break + + class TestOnlineConfigureFails: def test_invalid_password(self, acfactory): configdict = acfactory.get_next_liveconfig() ac1 = acfactory.get_unconfigured_account() - ac1.update_config(dict(addr=configdict["addr"], mail_pw="123")) + ac1.update_config({"addr": configdict["addr"], "mail_pw": "123"}) configtracker = ac1.configure() configtracker.wait_progress(500) configtracker.wait_progress(0) diff --git a/python/tests/test_3_offline.py b/python/tests/test_3_offline.py index 98890287a..97dedfe37 100644 --- a/python/tests/test_3_offline.py +++ b/python/tests/test_3_offline.py @@ -15,7 +15,7 @@ from deltachat.tracker import ImexFailed @pytest.mark.parametrize( - "msgtext,res", + ("msgtext", "res"), [ ( "Member Me (tmp1@x.org) removed by tmp2@x.org.", @@ -108,7 +108,7 @@ class TestOfflineAccountBasic: def test_update_config(self, acfactory): ac1 = acfactory.get_unconfigured_account() - ac1.update_config(dict(mvbox_move=False)) + ac1.update_config({"mvbox_move": False}) assert ac1.get_config("mvbox_move") == "0" def test_has_savemime(self, acfactory): @@ -229,11 +229,11 @@ class TestOfflineContact: class TestOfflineChat: - @pytest.fixture + @pytest.fixture() def ac1(self, acfactory): return acfactory.get_pseudo_configured_account() - @pytest.fixture + @pytest.fixture() def chat1(self, ac1): return ac1.create_contact("some1@example.org", name="some1").create_chat() @@ -257,7 +257,7 @@ class TestOfflineChat: assert chat2.id == chat1.id assert chat2.get_name() == chat1.get_name() assert chat1 == chat2 - assert not (chat1 != chat2) + assert not chat1.__ne__(chat2) assert chat1 != chat3 for ichat in ac1.get_chats(): @@ -450,7 +450,7 @@ class TestOfflineChat: assert msg.filemime == "image/png" @pytest.mark.parametrize( - "fn,typein,typeout", + ("fn", "typein", "typeout"), [ ("r", None, "application/octet-stream"), ("r.txt", None, "text/plain"), @@ -458,7 +458,7 @@ class TestOfflineChat: ("r.txt", "image/png", "image/png"), ], ) - def test_message_file(self, ac1, chat1, data, lp, fn, typein, typeout): + def test_message_file(self, chat1, data, lp, fn, typein, typeout): lp.sec("sending file") fp = data.get_path(fn) msg = chat1.send_file(fp, typein) @@ -694,7 +694,7 @@ class TestOfflineChat: chat1.set_draft(None) assert chat1.get_draft() is None - def test_qr_setup_contact(self, acfactory, lp): + def test_qr_setup_contact(self, acfactory): ac1 = acfactory.get_pseudo_configured_account() ac2 = acfactory.get_pseudo_configured_account() qr = ac1.get_setup_contact_qr() diff --git a/python/tests/test_4_lowlevel.py b/python/tests/test_4_lowlevel.py index 7b0acd822..cb517670c 100644 --- a/python/tests/test_4_lowlevel.py +++ b/python/tests/test_4_lowlevel.py @@ -93,7 +93,7 @@ def test_empty_context(): capi.lib.dc_context_unref(ctx) -def test_dc_close_events(tmpdir, acfactory): +def test_dc_close_events(acfactory): ac1 = acfactory.get_unconfigured_account() # register after_shutdown function diff --git a/python/tox.ini b/python/tox.ini index b182c41a1..d01dd82e4 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -50,18 +50,14 @@ commands = skipsdist = True skip_install = True deps = - flake8 -# isort 5.11.0 is broken: https://github.com/PyCQA/isort/issues/2031 - isort<5.11.0 + ruff black # pygments required by rst-lint pygments restructuredtext_lint commands = - isort --check setup.py install_python_bindings.py src/deltachat examples/ tests/ black --check setup.py install_python_bindings.py src/deltachat examples/ tests/ - flake8 src/deltachat - flake8 tests/ examples/ + ruff src/deltachat tests/ examples/ rst-lint --encoding 'utf-8' README.rst [testenv:mypy] @@ -102,7 +98,3 @@ timeout = 150 timeout_func_only = True markers = ignored: ignore this test in default test runs, use --ignored to run. - -[flake8] -max-line-length = 120 -ignore = E203, E266, E501, W503 diff --git a/scripts/cleanup_devpi_indices.py b/scripts/cleanup_devpi_indices.py index 8e1eaa9ee..b32a50474 100644 --- a/scripts/cleanup_devpi_indices.py +++ b/scripts/cleanup_devpi_indices.py @@ -2,12 +2,13 @@ Remove old "dc" indices except for master which always stays. """ -from requests import Session import datetime -import sys import subprocess +import sys -MAXDAYS=7 +from requests import Session + +MAXDAYS = 7 session = Session() session.headers["Accept"] = "application/json" @@ -54,7 +55,8 @@ def run(): if not dates: print( "%s has no releases" % (baseurl + username + "/" + indexname), - file=sys.stderr) + file=sys.stderr, + ) date = datetime.datetime.now() else: date = datetime.datetime(*max(dates)) @@ -67,6 +69,5 @@ def run(): subprocess.check_call(["devpi", "index", "-y", "--delete", url]) - -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml index d902947bc..bd2074d4b 100644 --- a/scripts/concourse/docs_wheels.yml +++ b/scripts/concourse/docs_wheels.yml @@ -104,9 +104,6 @@ jobs: outputs: - name: py-docs path: ./python/doc/_build/ - # Source packages - - name: py-dist - path: ./python/.docker-tox/dist/ # Binary wheels - name: py-wheels path: ./python/.docker-tox/wheelhouse/ @@ -145,7 +142,6 @@ jobs: config: inputs: - name: py-wheels - - name: py-dist image_resource: type: registry-image source: @@ -162,7 +158,6 @@ jobs: devpi use https://m.devpi.net/dc/master devpi login ((devpi.login)) --password ((devpi.password)) devpi upload py-wheels/*manylinux201* - devpi upload py-dist/* - name: python-aarch64 plan: diff --git a/scripts/set_core_version.py b/scripts/set_core_version.py index 9a46a8278..8046081fa 100755 --- a/scripts/set_core_version.py +++ b/scripts/set_core_version.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -import os import json -import re +import os import pathlib +import re import subprocess from argparse import ArgumentParser @@ -23,7 +23,7 @@ def read_toml_version(relpath): res = regex_matches(relpath, rex) if res is not None: return res.group(1) - raise ValueError("no version found in {}".format(relpath)) + raise ValueError(f"no version found in {relpath}") def replace_toml_version(relpath, newversion): @@ -34,8 +34,8 @@ def replace_toml_version(relpath, newversion): for line in open(str(p)): m = rex.match(line) if m is not None: - print("{}: set version={}".format(relpath, newversion)) - f.write('version = "{}"\n'.format(newversion)) + print(f"{relpath}: set version={newversion}") + f.write(f'version = "{newversion}"\n') else: f.write(line) os.rename(tmp_path, str(p)) @@ -44,7 +44,7 @@ def replace_toml_version(relpath, newversion): def read_json_version(relpath): p = pathlib.Path(relpath) assert p.exists() - with open(p, "r") as f: + with open(p) as f: json_data = json.loads(f.read()) return json_data["version"] @@ -52,7 +52,7 @@ def read_json_version(relpath): def update_package_json(relpath, newversion): p = pathlib.Path(relpath) assert p.exists() - with open(p, "r") as f: + with open(p) as f: json_data = json.loads(f.read()) json_data["version"] = newversion with open(p, "w") as f: @@ -63,7 +63,7 @@ def main(): parser = ArgumentParser(prog="set_core_version") parser.add_argument("newversion") - json_list = ["package.json", "deltachat-jsonrpc/typescript/package.json"] + json_list = ["package.json", "deltachat-jsonrpc/typescript/package.json"] toml_list = [ "Cargo.toml", "deltachat-ffi/Cargo.toml", @@ -75,9 +75,9 @@ def main(): except SystemExit: print() for x in toml_list: - print("{}: {}".format(x, read_toml_version(x))) + print(f"{x}: {read_toml_version(x)}") for x in json_list: - print("{}: {}".format(x, read_json_version(x))) + print(f"{x}: {read_json_version(x)}") print() raise SystemExit("need argument: new version, example: 1.25.0") @@ -92,19 +92,19 @@ def main(): if "alpha" not in newversion: for line in open("CHANGELOG.md"): ## 1.25.0 - if line.startswith("## "): - if line[2:].strip().startswith(newversion): - break + if line.startswith("## ") and line[2:].strip().startswith(newversion): + break else: - raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion)) + raise SystemExit( + f"CHANGELOG.md contains no entry for version: {newversion}" + ) for toml_filename in toml_list: replace_toml_version(toml_filename, newversion) - + for json_filename in json_list: update_package_json(json_filename, newversion) - print("running cargo check") subprocess.call(["cargo", "check"]) @@ -114,13 +114,12 @@ def main(): print("after commit, on master make sure to: ") print("") - print(" git tag -a {}".format(newversion)) - print(" git push origin {}".format(newversion)) - print(" git tag -a py-{}".format(newversion)) - print(" git push origin py-{}".format(newversion)) + print(f" git tag -a {newversion}") + print(f" git push origin {newversion}") + print(f" git tag -a py-{newversion}") + print(f" git push origin py-{newversion}") print("") if __name__ == "__main__": main() - diff --git a/src/accounts.rs b/src/accounts.rs index 2380de04e..7b01de94e 100644 --- a/src/accounts.rs +++ b/src/accounts.rs @@ -367,13 +367,20 @@ impl Config { // Previous versions of the core stored absolute paths in account config. // Convert them to relative paths. + let mut modified = false; for account in &mut inner.accounts { if let Ok(new_dir) = account.dir.strip_prefix(dir) { account.dir = new_dir.to_path_buf(); + modified = true; } } - Ok(Config { file, inner }) + let config = Self { file, inner }; + if modified { + config.sync().await?; + } + + Ok(config) } /// Loads all accounts defined in the configuration file. @@ -502,7 +509,6 @@ impl AccountConfig { #[cfg(test)] mod tests { use super::*; - use crate::stock_str::{self, StockMessage}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/aheader.rs b/src/aheader.rs index 9899c1978..5e4268e6f 100644 --- a/src/aheader.rs +++ b/src/aheader.rs @@ -2,13 +2,12 @@ //! //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header). -use anyhow::{bail, Context as _, Error, Result}; use std::collections::BTreeMap; +use std::fmt; use std::str::FromStr; -use std::{fmt, str}; -use crate::contact::addr_cmp; -use crate::headerdef::{HeaderDef, HeaderDefMap}; +use anyhow::{bail, Context as _, Error, Result}; + use crate::key::{DcKey, SignedPublicKey}; /// Possible values for encryption preference @@ -36,7 +35,7 @@ impl fmt::Display for EncryptPreference { } } -impl str::FromStr for EncryptPreference { +impl FromStr for EncryptPreference { type Err = Error; fn from_str(s: &str) -> Result { @@ -69,29 +68,6 @@ impl Aheader { prefer_encrypt, } } - - /// Tries to parse Autocrypt header. - /// - /// If there is none, returns None. If the header is present but cannot be parsed, returns an - /// error. - pub fn from_headers( - wanted_from: &str, - headers: &[mailparse::MailHeader<'_>], - ) -> Result> { - if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) { - let header = Self::from_str(&value)?; - if !addr_cmp(&header.addr, wanted_from) { - bail!( - "Autocrypt header address {:?} is not {:?}", - header.addr, - wanted_from - ); - } - Ok(Some(header)) - } else { - Ok(None) - } - } } impl fmt::Display for Aheader { @@ -118,7 +94,7 @@ impl fmt::Display for Aheader { } } -impl str::FromStr for Aheader { +impl FromStr for Aheader { type Err = Error; fn from_str(s: &str) -> Result { diff --git a/src/authres.rs b/src/authres.rs index d2b67f46a..eb267ddc9 100644 --- a/src/authres.rs +++ b/src/authres.rs @@ -355,7 +355,6 @@ mod tests { use tokio::io::AsyncReadExt; use super::*; - use crate::aheader::EncryptPreference; use crate::e2ee; use crate::message; diff --git a/src/blob.rs b/src/blob.rs index 7b6109f2e..5de6a44b2 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -499,17 +499,15 @@ fn encoded_img_exceeds_bytes( #[cfg(test)] mod tests { - use fs::File; - use anyhow::Result; + use fs::File; use image::{GenericImageView, Pixel}; + use super::*; use crate::chat::{self, create_group_chat, ProtectionStatus}; use crate::message::Message; use crate::test_utils::{self, TestContext}; - use super::*; - fn check_image_size(path: impl AsRef, width: u32, height: u32) -> image::DynamicImage { tokio::task::block_in_place(move || { let img = image::open(path).expect("failed to open image"); diff --git a/src/chat.rs b/src/chat.rs index dab83faff..008d59958 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -531,20 +531,68 @@ impl ChatId { Ok(()) } - // Unarchives a chat that is archived and not muted. - // Needed when a message is added to a chat so that the chat gets a normal visibility again. - // Sending an appropriate event is up to the caller. - pub async fn unarchive_if_not_muted(self, context: &Context) -> Result<()> { + /// Unarchives a chat that is archived and not muted. + /// Needed after a message is added to a chat so that the chat gets a normal visibility again. + /// `msg_state` is the state of the message. Matters only for incoming messages currently. For + /// multiple outgoing messages the function may be called once with MessageState::Undefined. + /// Sending an appropriate event is up to the caller. + /// Also emits DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived + /// chats with unread messages increases (which is possible if the chat is muted). + pub async fn unarchive_if_not_muted( + self, + context: &Context, + msg_state: MessageState, + ) -> Result<()> { + if msg_state != MessageState::InFresh { + context + .sql + .execute( + "UPDATE chats SET archived=0 WHERE id=? AND archived=1 \ + AND NOT(muted_until=-1 OR muted_until>?)", + paramsv![self, time()], + ) + .await?; + return Ok(()); + } + let chat = Chat::load_from_db(context, self).await?; + if chat.visibility != ChatVisibility::Archived { + return Ok(()); + } + if chat.is_muted() { + let unread_cnt = context + .sql + .count( + "SELECT COUNT(*) + FROM msgs + WHERE state=? + AND hidden=0 + AND chat_id=?", + paramsv![MessageState::InFresh, self], + ) + .await?; + if unread_cnt == 1 { + // Added the first unread message in the chat. + context.emit_msgs_changed(DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)); + } + return Ok(()); + } context .sql - .execute( - "UPDATE chats SET archived=0 WHERE id=? AND archived=1 AND NOT(muted_until=-1 OR muted_until>?)", - paramsv![self, time()], - ) + .execute("UPDATE chats SET archived=0 WHERE id=?", paramsv![self]) .await?; Ok(()) } + /// Emits an appropriate event for a message. `important` is whether a notification should be + /// shown. + pub(crate) fn emit_msg_event(self, context: &Context, msg_id: MsgId, important: bool) { + if important { + context.emit_incoming_msg(self, msg_id); + } else { + context.emit_msgs_changed(self, msg_id); + } + } + /// Deletes a chat. pub async fn delete(self, context: &Context) -> Result<()> { ensure!( @@ -1129,7 +1177,7 @@ impl Chat { } } Err(err) => { - error!(context, "faild to load contacts for {}: {:?}", chat.id, err); + error!(context, "faild to load contacts for {}: {:#}", chat.id, err); } } chat.name = chat_name; @@ -2011,7 +2059,9 @@ async fn prepare_msg_common( msg.state = change_state_to; prepare_msg_blob(context, msg).await?; - chat_id.unarchive_if_not_muted(context).await?; + if !msg.hidden { + chat_id.unarchive_if_not_muted(context, msg.state).await?; + } msg.id = chat .prepare_msg_raw( context, @@ -2148,7 +2198,7 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result attach_selfavatar, Err(err) => { - warn!(context, "job: cannot get selfavatar-state: {}", err); + warn!(context, "job: cannot get selfavatar-state: {:#}", err); false } }; @@ -2210,27 +2260,27 @@ async fn create_send_msg_job(context: &Context, msg_id: MsgId) -> Result Result<()> { // "WHERE" below uses the index `(state, hidden, chat_id)`, see get_fresh_msg_cnt() for reasoning // the additional SELECT statement may speed up things as no write-blocking is needed. - let exists = context - .sql - .exists( - "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;", - paramsv![MessageState::InFresh, chat_id], - ) - .await?; - if !exists { - return Ok(()); - } + if chat_id.is_archived_link() { + let chat_ids_in_archive = context + .sql + .query_map( + "SELECT DISTINCT(m.chat_id) FROM msgs m + LEFT JOIN chats c ON m.chat_id=c.id + WHERE m.state=10 AND m.hidden=0 AND m.chat_id>9 AND c.blocked=0 AND c.archived=1", + paramsv![], + |row| row.get::<_, ChatId>(0), + |ids| ids.collect::, _>>().map_err(Into::into) + ) + .await?; + if chat_ids_in_archive.is_empty() { + return Ok(()); + } - context - .sql - .execute( - "UPDATE msgs - SET state=? - WHERE state=? - AND hidden=0 - AND chat_id=?;", - paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id], - ) - .await?; + context + .sql + .execute( + &format!( + "UPDATE msgs SET state=13 WHERE state=10 AND hidden=0 AND chat_id IN ({});", + sql::repeat_vars(chat_ids_in_archive.len()) + ), + rusqlite::params_from_iter(&chat_ids_in_archive), + ) + .await?; + for chat_id_in_archive in chat_ids_in_archive { + context.emit_event(EventType::MsgsNoticed(chat_id_in_archive)); + } + } else { + let exists = context + .sql + .exists( + "SELECT COUNT(*) FROM msgs WHERE state=? AND hidden=0 AND chat_id=?;", + paramsv![MessageState::InFresh, chat_id], + ) + .await?; + if !exists { + return Ok(()); + } + + context + .sql + .execute( + "UPDATE msgs + SET state=? + WHERE state=? + AND hidden=0 + AND chat_id=?;", + paramsv![MessageState::InNoticed, MessageState::InFresh, chat_id], + ) + .await?; + } context.emit_event(EventType::MsgsNoticed(chat_id)); @@ -2873,14 +2955,12 @@ pub(crate) async fn add_contact_to_chat_ex( Ok(true) } +/// Returns true if an avatar should be attached in the given chat. +/// +/// This function does not check if the avatar is set. +/// If avatar is not set and this function returns `true`, +/// a `Chat-User-Avatar: 0` header should be sent to reset the avatar. pub(crate) async fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Result { - // versions before 12/2019 already allowed to set selfavatar, however, it was never sent to others. - // to avoid sending out previously set selfavatars unexpectedly we added this additional check. - // it can be removed after some time. - if !context.sql.get_raw_config_bool("attach_selfavatar").await? { - return Ok(false); - } - let timestamp_some_days_ago = time() - DC_RESEND_USER_AVATAR_DAYS * 24 * 60 * 60; let needs_attach = context .sql @@ -3174,7 +3254,9 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) let mut created_msgs: Vec = Vec::new(); let mut curr_timestamp: i64; - chat_id.unarchive_if_not_muted(context).await?; + chat_id + .unarchive_if_not_muted(context, MessageState::Undefined) + .await?; if let Ok(mut chat) = Chat::load_from_db(context, chat_id).await { if let Some(reason) = chat.why_cant_send(context).await? { bail!("cannot send to {}: {}", chat_id, reason); @@ -3377,7 +3459,6 @@ pub async fn add_device_msg_with_importance( let rfc724_mid = create_outgoing_rfc724_mid(None, "@device"); msg.try_calc_and_set_dimensions(context).await.ok(); prepare_msg_blob(context, msg).await?; - chat_id.unarchive_if_not_muted(context).await?; let timestamp_sent = create_smeared_timestamp(context).await; @@ -3397,6 +3478,7 @@ pub async fn add_device_msg_with_importance( } } + let state = MessageState::InFresh; let row_id = context .sql .insert( @@ -3420,7 +3502,7 @@ pub async fn add_device_msg_with_importance( timestamp_sent, timestamp_sent, // timestamp_sent equals timestamp_rcvd msg.viewtype, - MessageState::InFresh, + state, msg.text.as_ref().cloned().unwrap_or_default(), msg.param.to_string(), rfc724_mid, @@ -3429,6 +3511,9 @@ pub async fn add_device_msg_with_importance( .await?; msg_id = MsgId::new(u32::try_from(row_id)?); + if !msg.hidden { + chat_id.unarchive_if_not_muted(context, state).await?; + } } if let Some(label) = label { @@ -3442,11 +3527,7 @@ pub async fn add_device_msg_with_importance( } if !msg_id.is_unset() { - if important { - context.emit_incoming_msg(chat_id, msg_id); - } else { - context.emit_msgs_changed(chat_id, msg_id); - } + chat_id.emit_msg_event(context, msg_id, important); } Ok(msg_id) @@ -3597,13 +3678,13 @@ pub(crate) async fn update_msg_text_and_timestamp( #[cfg(test)] mod tests { use super::*; - use crate::chatlist::{get_archived_cnt, Chatlist}; use crate::constants::{DC_GCL_ARCHIVED_ONLY, DC_GCL_NO_SPECIALS}; - use crate::contact::Contact; + use crate::contact::{Contact, ContactAddress}; use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_chat_info() { let t = TestContext::new().await; @@ -4501,6 +4582,91 @@ mod tests { Ok(()) } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_archive_fresh_msgs() -> Result<()> { + let t = TestContext::new_alice().await; + + async fn msg_from(t: &TestContext, name: &str, num: u32) -> Result<()> { + receive_imf( + t, + format!( + "From: {}@example.net\n\ + To: alice@example.org\n\ + Message-ID: <{}@example.org>\n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2022 19:37:57 +0000\n\ + \n\ + hello\n", + name, num + ) + .as_bytes(), + false, + ) + .await?; + Ok(()) + } + + // receive some messages in archived+muted chats + msg_from(&t, "bob", 1).await?; + let bob_chat_id = t.get_last_msg().await.get_chat_id(); + bob_chat_id.accept(&t).await?; + set_muted(&t, bob_chat_id, MuteDuration::Forever).await?; + bob_chat_id + .set_visibility(&t, ChatVisibility::Archived) + .await?; + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 0); + + msg_from(&t, "bob", 2).await?; + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1); + + msg_from(&t, "bob", 3).await?; + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1); + + msg_from(&t, "claire", 4).await?; + let claire_chat_id = t.get_last_msg().await.get_chat_id(); + claire_chat_id.accept(&t).await?; + set_muted(&t, claire_chat_id, MuteDuration::Forever).await?; + claire_chat_id + .set_visibility(&t, ChatVisibility::Archived) + .await?; + msg_from(&t, "claire", 5).await?; + msg_from(&t, "claire", 6).await?; + msg_from(&t, "claire", 7).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 3); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2); + + // mark one of the archived+muted chats as noticed: check that the archive-link counter is changed as well + marknoticed_chat(&t, claire_chat_id).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 1); + + // receive some more messages + msg_from(&t, "claire", 8).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 1); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(t.get_fresh_msgs().await?.len(), 0); + + msg_from(&t, "dave", 9).await?; + let dave_chat_id = t.get_last_msg().await.get_chat_id(); + dave_chat_id.accept(&t).await?; + assert_eq!(dave_chat_id.get_fresh_msg_cnt(&t).await?, 1); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 2); + assert_eq!(t.get_fresh_msgs().await?.len(), 1); + + // mark the archived-link as noticed: check that the real chats are noticed as well + marknoticed_chat(&t, DC_CHAT_ID_ARCHIVED_LINK).await?; + assert_eq!(bob_chat_id.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(claire_chat_id.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(dave_chat_id.get_fresh_msg_cnt(&t).await?, 1); + assert_eq!(DC_CHAT_ID_ARCHIVED_LINK.get_fresh_msg_cnt(&t).await?, 0); + assert_eq!(t.get_fresh_msgs().await?.len(), 1); + + Ok(()) + } + async fn get_chats_from_chat_list(ctx: &Context, listflags: usize) -> Vec { let chatlist = Chatlist::try_load(ctx, listflags, None, None) .await @@ -4569,6 +4735,46 @@ mod tests { assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_pinned_after_new_msgs() -> Result<()> { + let alice = TestContext::new_alice().await; + let bob = TestContext::new_bob().await; + let alice_chat_id = alice.create_chat(&bob).await.id; + let bob_chat_id = bob.create_chat(&alice).await.id; + + assert!(alice_chat_id + .set_visibility(&alice, ChatVisibility::Pinned) + .await + .is_ok()); + assert_eq!( + Chat::load_from_db(&alice, alice_chat_id) + .await? + .get_visibility(), + ChatVisibility::Pinned, + ); + + send_text_msg(&alice, alice_chat_id, "hi!".into()).await?; + assert_eq!( + Chat::load_from_db(&alice, alice_chat_id) + .await? + .get_visibility(), + ChatVisibility::Pinned, + ); + + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("hi!".into())); + let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.chat_id, alice_chat_id); + assert_eq!( + Chat::load_from_db(&alice, alice_chat_id) + .await? + .get_visibility(), + ChatVisibility::Pinned, + ); + Ok(()) + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_set_chat_name() { let t = TestContext::new().await; @@ -4616,15 +4822,21 @@ mod tests { let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?; assert!(!shall_attach_selfavatar(&t, chat_id).await?); - let (contact_id, _) = - Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::IncomingUnknownTo).await?; + let (contact_id, _) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("foo@bar.org")?, + Origin::IncomingUnknownTo, + ) + .await?; add_contact_to_chat(&t, chat_id, contact_id).await?; - assert!(!shall_attach_selfavatar(&t, chat_id).await?); - t.set_config(Config::Selfavatar, None).await?; // setting to None also forces re-sending assert!(shall_attach_selfavatar(&t, chat_id).await?); chat_id.set_selfavatar_timestamp(&t, time()).await?; assert!(!shall_attach_selfavatar(&t, chat_id).await?); + + t.set_config(Config::Selfavatar, None).await?; // setting to None also forces re-sending + assert!(shall_attach_selfavatar(&t, chat_id).await?); Ok(()) } @@ -4863,8 +5075,8 @@ mod tests { alice.set_config(Config::ShowEmails, Some("2")).await?; bob.set_config(Config::ShowEmails, Some("2")).await?; - let (contact_id, _) = - Contact::add_or_lookup(&alice, "", "bob@example.net", Origin::ManuallyCreated).await?; + let alice_bob_contact = alice.add_or_lookup_contact(&bob).await; + let contact_id = alice_bob_contact.id; let alice_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?; let alice_chat = Chat::load_from_db(&alice, alice_chat_id).await?; @@ -5126,7 +5338,7 @@ mod tests { assert_eq!(msg.get_filename(), Some(filename.to_string())); assert_eq!(msg.get_width(), w); assert_eq!(msg.get_height(), h); - assert!(msg.get_filebytes(&bob).await > 250); + assert!(msg.get_filebytes(&bob).await?.unwrap() > 250); Ok(()) } @@ -5574,8 +5786,13 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_create_for_contact_with_blocked() -> Result<()> { let t = TestContext::new().await; - let (contact_id, _) = - Contact::add_or_lookup(&t, "", "foo@bar.org", Origin::ManuallyCreated).await?; + let (contact_id, _) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("foo@bar.org")?, + Origin::ManuallyCreated, + ) + .await?; // create a blocked chat let chat_id_orig = diff --git a/src/chatlist.rs b/src/chatlist.rs index df80b15e2..e20025de9 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -365,7 +365,6 @@ pub async fn get_archived_cnt(context: &Context) -> Result { #[cfg(test)] mod tests { use super::*; - use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus}; use crate::message::Viewtype; use crate::receive_imf::receive_imf; diff --git a/src/config.rs b/src/config.rs index 40cddcc8d..0bca875f1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -292,9 +292,6 @@ impl Context { self.sql .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![]) .await?; - self.sql - .set_raw_config_bool("attach_selfavatar", true) - .await?; match value { Some(value) => { let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?; @@ -443,16 +440,15 @@ fn get_config_keys_string() -> String { #[cfg(test)] mod tests { - use super::*; - use std::str::FromStr; use std::string::ToString; + use num_traits::FromPrimitive; + + use super::*; use crate::constants; use crate::test_utils::TestContext; - use num_traits::FromPrimitive; - #[test] fn test_to_string() { assert_eq!(Config::MailServer.to_string(), "mail_server"); diff --git a/src/configure.rs b/src/configure.rs index 82403a76a..ebce79c5d 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -6,9 +6,12 @@ mod read_url; mod server_params; use anyhow::{bail, ensure, Context as _, Result}; +use auto_mozilla::moz_autoconfigure; +use auto_outlook::outlk_autodiscover; use futures::FutureExt; use futures_lite::FutureExt as _; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; +use server_params::{expand_param_vector, ServerParams}; use tokio::task; use crate::config::Config; @@ -28,10 +31,6 @@ use crate::stock_str; use crate::tools::{time, EmailAddress}; use crate::{chat, e2ee, provider}; -use auto_mozilla::moz_autoconfigure; -use auto_outlook::outlk_autodiscover; -use server_params::{expand_param_vector, ServerParams}; - macro_rules! progress { ($context:tt, $progress:expr, $comment:expr) => { assert!( @@ -565,13 +564,18 @@ async fn try_imap_one_param( provider_strict_tls: bool, ) -> Result { let inf = format!( - "imap: {}@{}:{} security={} certificate_checks={} oauth2={}", + "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}", param.user, param.server, param.port, param.security, param.certificate_checks, - param.oauth2 + param.oauth2, + if let Some(socks5_config) = socks5_config { + socks5_config.to_string() + } else { + "None".to_string() + } ); info!(context, "Trying: {}", inf); @@ -661,6 +665,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec -use quick_xml::events::{BytesStart, Event}; - use std::io::BufRead; use std::str::FromStr; -use crate::context::Context; -use crate::login_param::LoginParam; -use crate::provider::{Protocol, Socket}; +use quick_xml::events::{BytesStart, Event}; use super::read_url::read_url; use super::{Error, ServerParams}; +use crate::context::Context; +use crate::login_param::LoginParam; +use crate::provider::{Protocol, Socket}; #[derive(Debug)] struct Server { diff --git a/src/configure/auto_outlook.rs b/src/configure/auto_outlook.rs index d374f7016..8d42fd353 100644 --- a/src/configure/auto_outlook.rs +++ b/src/configure/auto_outlook.rs @@ -3,15 +3,14 @@ //! This module implements autoconfiguration via POX (Plain Old XML) interface to Autodiscover //! Service. Newer SOAP interface, introduced in Exchange 2010, is not used. -use quick_xml::events::Event; - use std::io::BufRead; -use crate::context::Context; -use crate::provider::{Protocol, Socket}; +use quick_xml::events::Event; use super::read_url::read_url; use super::{Error, ServerParams}; +use crate::context::Context; +use crate::provider::{Protocol, Socket}; /// Result of parsing a single `Protocol` tag. /// diff --git a/src/configure/read_url.rs b/src/configure/read_url.rs index f275025db..b0cdd989f 100644 --- a/src/configure/read_url.rs +++ b/src/configure/read_url.rs @@ -16,7 +16,7 @@ pub async fn read_url(context: &Context, url: &str) -> anyhow::Result { } pub async fn read_url_inner(context: &Context, url: &str) -> anyhow::Result { - let client = reqwest::Client::new(); + let client = crate::http::get_client()?; let mut url = url.to_string(); // Follow up to 10 http-redirects diff --git a/src/constants.rs b/src/constants.rs index dde6219f5..a7c60a8dd 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -68,6 +68,7 @@ impl Default for MediaQuality { } } +/// Type of the key to generate. #[derive( Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql, )] @@ -118,13 +119,13 @@ pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01; pub const DC_GCL_ADD_SELF: u32 = 0x02; // unchanged user avatars are resent to the recipients every some days -pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14; +pub(crate) const DC_RESEND_USER_AVATAR_DAYS: i64 = 14; // warn about an outdated app after a given number of days. // as we use the "provider-db generation date" as reference (that might not be updated very often) // and as not all system get speedy updates, // do not use too small value that will annoy users checking for nonexistant updates. -pub const DC_OUTDATED_WARNING_DAYS: i64 = 365; +pub(crate) const DC_OUTDATED_WARNING_DAYS: i64 = 365; /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again) pub const DC_CHAT_ID_TRASH: ChatId = ChatId::new(3); @@ -169,7 +170,7 @@ pub const DC_MSG_ID_DAYMARKER: u32 = 9; pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9; /// String that indicates that something is left out or truncated. -pub const DC_ELLIPSIS: &str = "[...]"; +pub(crate) const DC_ELLIPSIS: &str = "[...]"; // how many lines desktop can display when fullscreen (fullscreen at zoomlevel 1x) // (taken from "subjective" testing what looks ok) pub const DC_DESIRED_TEXT_LINES: usize = 38; @@ -186,11 +187,6 @@ pub const DC_DESIRED_TEXT_LINE_LEN: usize = 100; /// `char`s), not Unicode Grapheme Clusters. pub const DC_DESIRED_TEXT_LEN: usize = DC_DESIRED_TEXT_LINE_LEN * DC_DESIRED_TEXT_LINES; -// Flags for empty server job - -pub const DC_EMPTY_MVBOX: u32 = 0x01; -pub const DC_EMPTY_INBOX: u32 = 0x02; - // Flags for configuring IMAP and SMTP servers. // These flags are optional // and may be set together with the username, password etc. @@ -220,21 +216,7 @@ pub const BALANCED_IMAGE_SIZE: u32 = 1280; pub const WORSE_IMAGE_SIZE: u32 = 640; // this value can be increased if the folder configuration is changed and must be redone on next program start -pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3; - -// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks. -// this does not affect MIME'e `To:` header. -// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db. -pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50; - -pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)] -#[repr(u8)] -pub enum KeyType { - Public = 0, - Private = 1, -} +pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3; #[cfg(test)] mod tests { @@ -262,13 +244,6 @@ mod tests { assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap()); } - #[test] - fn test_keytype_values() { - // values may be written to disk and must not change - assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap()); - assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap()); - } - #[test] fn test_showemails_values() { // values may be written to disk and must not change diff --git a/src/contact.rs b/src/contact.rs index baf74b4ac..a1e546c67 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -1,11 +1,10 @@ //! Contacts module -#![allow(missing_docs)] - use std::cmp::Reverse; use std::collections::BinaryHeap; use std::convert::{TryFrom, TryInto}; use std::fmt; +use std::ops::Deref; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; @@ -38,6 +37,51 @@ use crate::{chat, stock_str}; /// Time during which a contact is considered as seen recently. const SEEN_RECENTLY_SECONDS: i64 = 600; +/// Valid contact address. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ContactAddress<'a>(&'a str); + +impl Deref for ContactAddress<'_> { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl AsRef for ContactAddress<'_> { + fn as_ref(&self) -> &str { + self.0 + } +} + +impl fmt::Display for ContactAddress<'_> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl<'a> ContactAddress<'a> { + /// Constructs a new contact address from string, + /// normalizing and validating it. + pub fn new(s: &'a str) -> Result { + let addr = addr_normalize(s); + if !may_be_valid_addr(addr) { + bail!("invalid address {:?}", s); + } + Ok(Self(addr)) + } +} + +/// Allow converting [`ContactAddress`] to an SQLite type. +impl rusqlite::types::ToSql for ContactAddress<'_> { + fn to_sql(&self) -> rusqlite::Result { + let val = rusqlite::types::Value::Text(self.0.to_string()); + let out = rusqlite::types::ToSqlOutput::Owned(val); + Ok(out) + } +} + /// Contact ID, including reserved IDs. /// /// Some contact IDs are reserved to identify special contacts. This @@ -48,12 +92,18 @@ const SEEN_RECENTLY_SECONDS: i64 = 600; pub struct ContactId(u32); impl ContactId { + /// Undefined contact. Used as a placeholder for trashed messages. pub const UNDEFINED: ContactId = ContactId::new(0); + /// The owner of the account. /// /// The email-address is set by `set_config` using "addr". pub const SELF: ContactId = ContactId::new(1); + + /// ID of the contact for info messages. pub const INFO: ContactId = ContactId::new(2); + + /// ID of the contact for device messages. pub const DEVICE: ContactId = ContactId::new(5); const LAST_SPECIAL: ContactId = ContactId::new(9); @@ -177,6 +227,8 @@ pub struct Contact { )] #[repr(u32)] pub enum Origin { + /// Unknown origin. Can be used as a minimum origin to specify that the caller does not care + /// about origin of the contact. Unknown = 0, /// The contact is a mailing list address, needed to unblock mailing lists @@ -257,12 +309,13 @@ pub(crate) enum Modifier { Created, } +/// Verification status of the contact. #[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)] #[repr(u8)] pub enum VerifiedStatus { /// Contact is not verified. Unverified = 0, - // TODO: is this a thing? + /// SELF has verified the fingerprint of a contact. Currently unused. Verified = 1, /// SELF and contact have verified their fingerprints in both directions; in the UI typically checkmarks are shown. BidirectVerified = 2, @@ -275,6 +328,7 @@ impl Default for VerifiedStatus { } impl Contact { + /// Loads a contact snapshot from the database. pub async fn load_from_db(context: &Context, contact_id: ContactId) -> Result { let mut contact = context .sql @@ -368,12 +422,14 @@ impl Contact { /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event. pub async fn create(context: &Context, name: &str, addr: &str) -> Result { let name = improve_single_line_input(name); - ensure!(!addr.is_empty(), "Cannot create contact with empty address"); let (name, addr) = sanitize_name_and_addr(&name, addr); + let addr = ContactAddress::new(&addr)?; let (contact_id, sth_modified) = - Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated).await?; + Contact::add_or_lookup(context, &name, addr, Origin::ManuallyCreated) + .await + .context("add_or_lookup")?; let blocked = Contact::is_blocked_load(context, contact_id).await?; match sth_modified { Modifier::None => {} @@ -458,10 +514,12 @@ impl Contact { /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name". /// /// Returns the contact_id and a `Modifier` value indicating if a modification occurred. + /// + /// Returns None if the contact with such address cannot exist. pub(crate) async fn add_or_lookup( context: &Context, name: &str, - addr: &str, + addr: ContactAddress<'_>, mut origin: Origin, ) -> Result<(ContactId, Modifier)> { let mut sth_modified = Modifier::None; @@ -469,22 +527,10 @@ impl Contact { ensure!(!addr.is_empty(), "Can not add_or_lookup empty address"); ensure!(origin != Origin::Unknown, "Missing valid origin"); - let addr = addr_normalize(addr).to_string(); - if context.is_self_addr(&addr).await? { return Ok((ContactId::SELF, sth_modified)); } - if !may_be_valid_addr(&addr) { - warn!( - context, - "Bad address \"{}\" for contact \"{}\".", - addr, - if !name.is_empty() { name } else { "" }, - ); - bail!("Bad address supplied: {:?}", addr); - } - let mut name = name; #[allow(clippy::collapsible_if)] if origin <= Origin::OutgoingTo { @@ -543,7 +589,7 @@ impl Contact { || row_authname.is_empty()); row_id = u32::try_from(id)?; - if origin as i32 >= row_origin as i32 && addr != row_addr { + if origin >= row_origin && addr.as_ref() != row_addr { update_addr = true; } if update_name || update_authname || update_addr || origin > row_origin { @@ -671,18 +717,25 @@ impl Contact { for (name, addr) in split_address_book(addr_book).into_iter() { let (name, addr) = sanitize_name_and_addr(name, addr); let name = normalize_name(&name); - match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await { - Err(err) => { - warn!( - context, - "Failed to add address {} from address book: {}", addr, err - ); - } - Ok((_, modified)) => { - if modified != Modifier::None { - modify_cnt += 1 + match ContactAddress::new(&addr) { + Ok(addr) => { + match Contact::add_or_lookup(context, &name, addr, Origin::AddressBook).await { + Ok((_, modified)) => { + if modified != Modifier::None { + modify_cnt += 1 + } + } + Err(err) => { + warn!( + context, + "Failed to add address {} from address book: {}", addr, err + ); + } } } + Err(err) => { + warn!(context, "{:#}.", err); + } } } if modify_cnt > 0 { @@ -847,6 +900,7 @@ impl Contact { Ok(()) } + /// Returns number of blocked contacts. pub async fn get_blocked_cnt(context: &Context) -> Result { let count = context .sql @@ -1138,23 +1192,16 @@ impl Contact { Ok(VerifiedStatus::Unverified) } - /// Return the address that verified the given contact - pub async fn get_verifier_addr( - context: &Context, - contact_id: &ContactId, - ) -> Result> { - let contact = Contact::load_from_db(context, *contact_id).await?; - - Ok(Peerstate::from_addr(context, contact.get_addr()) + /// Returns the address that verified the contact. + pub async fn get_verifier_addr(&self, context: &Context) -> Result> { + Ok(Peerstate::from_addr(context, self.get_addr()) .await? .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned()))) } - pub async fn get_verifier_id( - context: &Context, - contact_id: &ContactId, - ) -> Result> { - let verifier_addr = Contact::get_verifier_addr(context, contact_id).await?; + /// Returns the ContactId that verified the contact. + pub async fn get_verifier_id(&self, context: &Context) -> Result> { + let verifier_addr = self.get_verifier_addr(context).await?; if let Some(addr) = verifier_addr { Ok(Contact::lookup_id_by_addr(context, &addr, Origin::AddressBook).await?) } else { @@ -1162,7 +1209,7 @@ impl Contact { } } - /// Return the ContactId that verified the given contact + /// Returns the number of real (i.e. non-special) contacts in the database. pub async fn get_real_cnt(context: &Context) -> Result { if !context.sql.is_open().await { return Ok(0); @@ -1178,6 +1225,7 @@ impl Contact { Ok(count) } + /// Returns true if a contact with this ID exists. pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result { if contact_id.is_special() { return Ok(false); @@ -1193,6 +1241,7 @@ impl Contact { Ok(exists) } + /// Updates the origin of the contact, but only if new origin is higher than the current one. pub async fn scaleup_origin_by_id( context: &Context, contact_id: ContactId, @@ -1404,6 +1453,7 @@ pub(crate) async fn update_last_seen( ) .await? > 0 + && timestamp > time() - SEEN_RECENTLY_SECONDS { context.interrupt_recently_seen(contact_id, timestamp).await; } @@ -1453,6 +1503,7 @@ fn cat_fingerprint( } } +/// Compares two email addresses, normalizing them beforehand. pub fn addr_cmp(addr1: &str, addr2: &str) -> bool { let norm1 = addr_normalize(addr1).to_lowercase(); let norm2 = addr_normalize(addr2).to_lowercase(); @@ -1561,6 +1612,9 @@ impl RecentlySeenLoop { context, "Error receiving an interruption in recently seen loop: {}", err ); + // Maybe the sender side is closed. + // Terminate the loop to avoid looping indefinitely. + return; } Ok(Ok(RecentlySeenInterrupt { contact_id, @@ -1602,7 +1656,6 @@ impl RecentlySeenLoop { #[cfg(test)] mod tests { use super::*; - use crate::chat::{get_chat_contacts, send_text_msg, Chat}; use crate::chatlist::Chatlist; use crate::receive_imf::receive_imf; @@ -1680,7 +1733,7 @@ mod tests { let (id, _modified) = Contact::add_or_lookup( &context.ctx, "bob", - "user@example.org", + ContactAddress::new("user@example.org")?, Origin::IncomingReplyTo, ) .await?; @@ -1708,7 +1761,7 @@ mod tests { let (contact_bob_id, modified) = Contact::add_or_lookup( &context.ctx, "someone", - "user@example.org", + ContactAddress::new("user@example.org")?, Origin::ManuallyCreated, ) .await?; @@ -1743,6 +1796,18 @@ mod tests { Ok(()) } + #[test] + fn test_contact_address() -> Result<()> { + let alice_addr = "alice@example.org"; + let contact_address = ContactAddress::new(alice_addr)?; + assert_eq!(contact_address.as_ref(), alice_addr); + + let invalid_addr = "<> foobar"; + assert!(ContactAddress::new(invalid_addr).is_err()); + + Ok(()) + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_add_or_lookup() { // add some contacts, this also tests add_address_book() @@ -1758,10 +1823,14 @@ mod tests { assert_eq!(Contact::add_address_book(&t, book).await.unwrap(), 4); // check first added contact, this modifies authname because it is empty - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo) - .await - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bla foo", + ContactAddress::new("one@eins.org").unwrap(), + Origin::IncomingUnknownTo, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1773,10 +1842,14 @@ mod tests { assert_eq!(contact.get_name_n_addr(), "Name one (one@eins.org)"); // modify first added contact - let (contact_id_test, sth_modified) = - Contact::add_or_lookup(&t, "Real one", " one@eins.org ", Origin::ManuallyCreated) - .await - .unwrap(); + let (contact_id_test, sth_modified) = Contact::add_or_lookup( + &t, + "Real one", + ContactAddress::new(" one@eins.org ").unwrap(), + Origin::ManuallyCreated, + ) + .await + .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1785,10 +1858,14 @@ mod tests { assert!(!contact.is_blocked()); // check third added contact (contact without name) - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo) - .await - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("three@drei.sam").unwrap(), + Origin::IncomingUnknownTo, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::None); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1801,7 +1878,7 @@ mod tests { let (contact_id_test, sth_modified) = Contact::add_or_lookup( &t, "m. serious", - "three@drei.sam", + ContactAddress::new("three@drei.sam").unwrap(), Origin::IncomingUnknownFrom, ) .await @@ -1813,10 +1890,14 @@ mod tests { assert!(!contact.is_blocked()); // manually edit name of third contact (does not changed authorized name) - let (contact_id_test, sth_modified) = - Contact::add_or_lookup(&t, "schnucki", "three@drei.sam", Origin::ManuallyCreated) - .await - .unwrap(); + let (contact_id_test, sth_modified) = Contact::add_or_lookup( + &t, + "schnucki", + ContactAddress::new("three@drei.sam").unwrap(), + Origin::ManuallyCreated, + ) + .await + .unwrap(); assert_eq!(contact_id, contact_id_test); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1825,10 +1906,14 @@ mod tests { assert!(!contact.is_blocked()); // Fourth contact: - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo) - .await - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("alice@w.de").unwrap(), + Origin::IncomingUnknownTo, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::None); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -1963,9 +2048,13 @@ mod tests { assert!(Contact::delete(&alice, ContactId::SELF).await.is_err()); // Create Bob contact - let (contact_id, _) = - Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await?; + let (contact_id, _) = Contact::add_or_lookup( + &alice, + "Bob", + ContactAddress::new("bob@example.net")?, + Origin::ManuallyCreated, + ) + .await?; let chat = alice .create_chat_with_contact("Bob", "bob@example.net") .await; @@ -2038,10 +2127,14 @@ mod tests { let t = TestContext::new().await; // incoming mail `From: bob1 ` - this should init authname - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bob1", + ContactAddress::new("bob@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Created); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -2050,10 +2143,14 @@ mod tests { assert_eq!(contact.get_display_name(), "bob1"); // incoming mail `From: bob2 ` - this should update authname - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bob2", + ContactAddress::new("bob@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -2072,10 +2169,14 @@ mod tests { assert_eq!(contact.get_display_name(), "bob3"); // incoming mail `From: bob4 ` - this should update authname, manually given name is still "bob3" - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap(); + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "bob4", + ContactAddress::new("bob@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); assert!(!contact_id.is_special()); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); @@ -2100,7 +2201,7 @@ mod tests { let (contact_id_same, sth_modified) = Contact::add_or_lookup( &t, "claire1", - "claire@example.org", + ContactAddress::new("claire@example.org").unwrap(), Origin::IncomingUnknownFrom, ) .await @@ -2116,7 +2217,7 @@ mod tests { let (contact_id_same, sth_modified) = Contact::add_or_lookup( &t, "claire2", - "claire@example.org", + ContactAddress::new("claire@example.org").unwrap(), Origin::IncomingUnknownFrom, ) .await @@ -2138,26 +2239,38 @@ mod tests { let t = TestContext::new().await; // Incoming message from Bob. - let (contact_id, sth_modified) = - Contact::add_or_lookup(&t, "Bob", "bob@example.org", Origin::IncomingUnknownFrom) - .await?; + let (contact_id, sth_modified) = Contact::add_or_lookup( + &t, + "Bob", + ContactAddress::new("bob@example.org")?, + Origin::IncomingUnknownFrom, + ) + .await?; assert_eq!(sth_modified, Modifier::Created); let contact = Contact::load_from_db(&t, contact_id).await?; assert_eq!(contact.get_display_name(), "Bob"); // Incoming message from someone else with "Not Bob" in the "To:" field. - let (contact_id_same, sth_modified) = - Contact::add_or_lookup(&t, "Not Bob", "bob@example.org", Origin::IncomingUnknownTo) - .await?; + let (contact_id_same, sth_modified) = Contact::add_or_lookup( + &t, + "Not Bob", + ContactAddress::new("bob@example.org")?, + Origin::IncomingUnknownTo, + ) + .await?; assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); let contact = Contact::load_from_db(&t, contact_id).await?; assert_eq!(contact.get_display_name(), "Not Bob"); // Incoming message from Bob, changing the name back. - let (contact_id_same, sth_modified) = - Contact::add_or_lookup(&t, "Bob", "bob@example.org", Origin::IncomingUnknownFrom) - .await?; + let (contact_id_same, sth_modified) = Contact::add_or_lookup( + &t, + "Bob", + ContactAddress::new("bob@example.org")?, + Origin::IncomingUnknownFrom, + ) + .await?; assert_eq!(contact_id, contact_id_same); assert_eq!(sth_modified, Modifier::Modified); // This was None until the bugfix let contact = Contact::load_from_db(&t, contact_id).await?; @@ -2180,9 +2293,14 @@ mod tests { assert_eq!(contact.get_display_name(), "dave1"); // incoming mail `From: dave2 ` - this should update authname - Contact::add_or_lookup(&t, "dave2", "dave@example.org", Origin::IncomingUnknownFrom) - .await - .unwrap(); + Contact::add_or_lookup( + &t, + "dave2", + ContactAddress::new("dave@example.org").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap(); let contact = Contact::load_from_db(&t, contact_id).await.unwrap(); assert_eq!(contact.get_authname(), "dave2"); assert_eq!(contact.get_name(), "dave1"); @@ -2296,9 +2414,13 @@ mod tests { let encrinfo = Contact::get_encrinfo(&alice, ContactId::DEVICE).await; assert!(encrinfo.is_err()); - let (contact_bob_id, _modified) = - Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await?; + let (contact_bob_id, _modified) = Contact::add_or_lookup( + &alice, + "Bob", + ContactAddress::new("bob@example.net")?, + Origin::ManuallyCreated, + ) + .await?; let encrinfo = Contact::get_encrinfo(&alice, contact_bob_id).await?; assert_eq!(encrinfo, "No encryption"); @@ -2455,9 +2577,13 @@ CCCB 5AA9 F6E1 141C 9431 async fn test_last_seen() -> Result<()> { let alice = TestContext::new_alice().await; - let (contact_id, _) = - Contact::add_or_lookup(&alice, "Bob", "bob@example.net", Origin::ManuallyCreated) - .await?; + let (contact_id, _) = Contact::add_or_lookup( + &alice, + "Bob", + ContactAddress::new("bob@example.net")?, + Origin::ManuallyCreated, + ) + .await?; let contact = Contact::load_from_db(&alice, contact_id).await?; assert_eq!(contact.last_seen(), 0); @@ -2504,4 +2630,27 @@ Hi."#; Ok(()) } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_verified_by_none() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + let bob = tcm.bob().await; + + let contact_id = Contact::create(&alice, "Bob", "bob@example.net").await?; + let contact = Contact::get_by_id(&alice, contact_id).await?; + assert!(contact.get_verifier_addr(&alice).await?.is_none()); + assert!(contact.get_verifier_id(&alice).await?.is_none()); + + // Receive a message from Bob to create a peerstate. + let chat = bob.create_chat(&alice).await; + let sent_msg = bob.send_text(chat.id, "moin").await; + alice.recv_msg(&sent_msg).await; + + let contact = Contact::get_by_id(&alice, contact_id).await?; + assert!(contact.get_verifier_addr(&alice).await?.is_none()); + assert!(contact.get_verifier_id(&alice).await?.is_none()); + + Ok(()) + } } diff --git a/src/context.rs b/src/context.rs index 7e38a2106..329667ddf 100644 --- a/src/context.rs +++ b/src/context.rs @@ -383,7 +383,7 @@ impl Context { let mut lock = self.inner.scheduler.write().await; if lock.is_none() { match Scheduler::start(self.clone()).await { - Err(err) => error!(self, "Failed to start IO: {}", err), + Err(err) => error!(self, "Failed to start IO: {:#}", err), Ok(scheduler) => *lock = Some(scheduler), } } @@ -499,7 +499,7 @@ impl Context { match &*s { RunningState::Running { cancel_sender } => { if let Err(err) = cancel_sender.send(()).await { - warn!(self, "could not cancel ongoing: {:?}", err); + warn!(self, "could not cancel ongoing: {:#}", err); } info!(self, "Signaling the ongoing process to stop ASAP.",); *s = RunningState::ShallStop; @@ -861,8 +861,13 @@ pub fn get_version_str() -> &'static str { #[cfg(test)] mod tests { - use super::*; + use std::time::Duration; + use anyhow::Context as _; + use strum::IntoEnumIterator; + use tempfile::tempdir; + + use super::*; use crate::chat::{ get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration, }; @@ -873,10 +878,6 @@ mod tests { use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; use crate::tools::create_outgoing_rfc724_mid; - use anyhow::Context as _; - use std::time::Duration; - use strum::IntoEnumIterator; - use tempfile::tempdir; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_wrong_db() -> Result<()> { diff --git a/src/decrypt.rs b/src/decrypt.rs index 68bcba737..1165e3dd7 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -1,6 +1,7 @@ //! End-to-end decryption support. use std::collections::HashSet; +use std::str::FromStr; use anyhow::Result; use mailparse::ParsedMail; @@ -13,7 +14,6 @@ use crate::context::Context; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey}; use crate::keyring::Keyring; -use crate::log::LogExt; use crate::peerstate::Peerstate; use crate::pgp; @@ -72,9 +72,25 @@ pub(crate) async fn prepare_decryption( }); } - let autocrypt_header = Aheader::from_headers(from, &mail.headers) - .ok_or_log_msg(context, "Failed to parse Autocrypt header") - .flatten(); + let autocrypt_header = + if let Some(autocrypt_header_value) = mail.headers.get_header_value(HeaderDef::Autocrypt) { + match Aheader::from_str(&autocrypt_header_value) { + Ok(header) if addr_cmp(&header.addr, from) => Some(header), + Ok(header) => { + warn!( + context, + "Autocrypt header address {:?} is not {:?}.", header.addr, from + ); + None + } + Err(err) => { + warn!(context, "Failed to parse Autocrypt header: {:#}.", err); + None + } + } + } else { + None + }; let dkim_results = handle_authres(context, mail, from, message_time).await?; @@ -328,11 +344,10 @@ pub(crate) async fn get_autocrypt_peerstate( #[cfg(test)] mod tests { + use super::*; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; - use super::*; - #[test] fn test_has_decrypted_pgp_armor() { let data = b" -----BEGIN PGP MESSAGE-----"; diff --git a/src/download.rs b/src/download.rs index de127c7ae..cd08c01dd 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,9 +1,11 @@ //! # Download large messages manually. +use std::cmp::max; +use std::collections::BTreeMap; + use anyhow::{anyhow, Result}; use deltachat_derive::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; use crate::config::Config; use crate::context::Context; @@ -14,7 +16,6 @@ use crate::mimeparser::{MimeMessage, Part}; use crate::param::Params; use crate::tools::time; use crate::{job_try, stock_str, EventType}; -use std::cmp::max; /// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`. /// @@ -132,7 +133,7 @@ impl Job { /// Called in response to `Action::DownloadMsg`. pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status { if let Err(err) = imap.prepare(context).await { - warn!(context, "download: could not connect: {:?}", err); + warn!(context, "download: could not connect: {:#}", err); return Status::RetryNow; } @@ -264,14 +265,13 @@ impl MimeMessage { mod tests { use num_traits::FromPrimitive; + use super::*; use crate::chat::{get_chat_msgs, send_msg}; use crate::ephemeral::Timer; use crate::message::Viewtype; use crate::receive_imf::receive_imf_inner; use crate::test_utils::TestContext; - use super::*; - #[test] fn test_downloadstate_values() { // values may be written to disk and must not change diff --git a/src/e2ee.rs b/src/e2ee.rs index 62f699fdf..f145373e4 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -144,13 +144,12 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result { #[cfg(test)] mod tests { + use super::*; use crate::chat; use crate::message::{Message, Viewtype}; use crate::param::Param; use crate::test_utils::{bob_keypair, TestContext}; - use super::*; - mod ensure_secret_key_exists { use super::*; diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 1118ef077..0e8dfdf7a 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -64,6 +64,7 @@ #![allow(missing_docs)] +use std::cmp::max; use std::convert::{TryFrom, TryInto}; use std::num::ParseIntError; use std::str::FromStr; @@ -86,7 +87,6 @@ use crate::mimeparser::SystemMessage; use crate::sql::{self, params_iter}; use crate::stock_str; use crate::tools::{duration_to_str, time}; -use std::cmp::max; #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)] pub enum Timer { diff --git a/src/events.rs b/src/events.rs index b2744f4d5..6d2962341 100644 --- a/src/events.rs +++ b/src/events.rs @@ -283,7 +283,7 @@ pub enum EventType { /// @param data2 (int) Progress as: /// 300=vg-/vc-request received, typically shown as "bob@addr joins". /// 600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified". - /// 800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol. + /// 800=contact added to chat, shown as "bob@addr securely joined GROUP". Only for the verified-group-protocol. /// 1000=Protocol finished for this contact. SecurejoinInviterProgress { contact_id: ContactId, diff --git a/src/html.rs b/src/html.rs index ac59793fc..7e963e16b 100644 --- a/src/html.rs +++ b/src/html.rs @@ -7,12 +7,14 @@ //! `MsgId.get_html()` will return HTML - //! this allows nice quoting, handling linebreaks properly etc. -use futures::future::FutureExt; use std::future::Future; use std::pin::Pin; use anyhow::{Context as _, Result}; +use futures::future::FutureExt; use lettre_email::mime::{self, Mime}; +use lettre_email::PartBuilder; +use mailparse::ParsedContentType; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::message::{Message, MsgId}; @@ -20,8 +22,6 @@ use crate::mimeparser::parse_message_id; use crate::param::Param::SendHtml; use crate::plaintext::PlainText; use crate::{context::Context, message}; -use lettre_email::PartBuilder; -use mailparse::ParsedContentType; impl Message { /// Check if the message can be retrieved as HTML. @@ -250,7 +250,7 @@ impl MsgId { if !rawmime.is_empty() { match HtmlMsgParser::from_bytes(context, &rawmime).await { Err(err) => { - warn!(context, "get_html: parser error: {}", err); + warn!(context, "get_html: parser error: {:#}", err); Ok(None) } Ok(parser) => Ok(Some(parser.html)), diff --git a/src/http.rs b/src/http.rs new file mode 100644 index 000000000..17e73a10d --- /dev/null +++ b/src/http.rs @@ -0,0 +1,13 @@ +//! # HTTP module. + +use std::time::Duration; + +use anyhow::Result; + +const HTTP_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn get_client() -> Result { + Ok(reqwest::ClientBuilder::new() + .timeout(HTTP_TIMEOUT) + .build()?) +} diff --git a/src/imap.rs b/src/imap.rs index cb3843ee0..5eaf0fd6a 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -22,7 +22,7 @@ use crate::config::Config; use crate::constants::{ Blocked, Chattype, ShowEmails, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION, }; -use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin}; +use crate::contact::{normalize_name, Contact, ContactAddress, ContactId, Modifier, Origin}; use crate::context::Context; use crate::events::EventType; use crate::headerdef::{HeaderDef, HeaderDefMap}; @@ -308,6 +308,7 @@ impl Imap { if let Some(socks5_config) = &config.socks5_config { if config.lp.security == Socket::Starttls { Client::connect_starttls_socks5( + context, imap_server, imap_port, socks5_config.clone(), @@ -315,13 +316,18 @@ impl Imap { ) .await } else { - Client::connect_insecure_socks5((imap_server, imap_port), socks5_config.clone()) - .await + Client::connect_insecure_socks5( + context, + imap_server, + imap_port, + socks5_config.clone(), + ) + .await } } else if config.lp.security == Socket::Starttls { - Client::connect_starttls(imap_server, imap_port, config.strict_tls).await + Client::connect_starttls(context, imap_server, imap_port, config.strict_tls).await } else { - Client::connect_insecure((imap_server, imap_port)).await + Client::connect_insecure(context, imap_server, imap_port).await } } else { let config = &self.config; @@ -330,6 +336,7 @@ impl Imap { if let Some(socks5_config) = &config.socks5_config { Client::connect_secure_socks5( + context, imap_server, imap_port, config.strict_tls, @@ -337,7 +344,7 @@ impl Imap { ) .await } else { - Client::connect_secure(imap_server, imap_port, config.strict_tls).await + Client::connect_secure(context, imap_server, imap_port, config.strict_tls).await } }; @@ -554,7 +561,10 @@ impl Imap { folder: &str, ) -> Result { let session = self.session.as_mut().context("no session")?; - let newly_selected = session.select_or_create_folder(context, folder).await?; + let newly_selected = session + .select_or_create_folder(context, folder) + .await + .with_context(|| format!("failed to select or create folder {}", folder))?; let mailbox = session .selected_mailbox .as_mut() @@ -564,8 +574,12 @@ impl Imap { .uid_validity .with_context(|| format!("No UIDVALIDITY for folder {}", folder))?; - let old_uid_validity = get_uidvalidity(context, folder).await?; - let old_uid_next = get_uid_next(context, folder).await?; + let old_uid_validity = get_uidvalidity(context, folder) + .await + .with_context(|| format!("failed to get old UID validity for folder {}", folder))?; + let old_uid_next = get_uid_next(context, folder) + .await + .with_context(|| format!("failed to get old UID NEXT for folder {}", folder))?; if new_uid_validity == old_uid_validity { let new_emails = if newly_selected == NewlySelected::No { @@ -691,9 +705,11 @@ impl Imap { let old_uid_next = get_uid_next(context, folder).await?; let msgs = if fetch_existing_msgs { - self.prefetch_existing_msgs().await? + self.prefetch_existing_msgs() + .await + .context("prefetch_existing_msgs")? } else { - self.prefetch(old_uid_next).await? + self.prefetch(old_uid_next).await.context("prefetch")? }; let read_cnt = msgs.len(); @@ -756,7 +772,7 @@ impl Imap { fetch_response.flags(), show_emails, ) - .await? + .await.context("prefetch_should_download")? { match download_limit { Some(download_limit) => uids_fetch.push(( @@ -792,7 +808,8 @@ impl Imap { fetch_partially, fetch_existing_msgs, ) - .await?; + .await + .context("fetch_many_msgs")?; received_msgs.extend(received_msgs_in_batch); largest_uid_fetched = max( largest_uid_fetched, @@ -818,11 +835,13 @@ impl Imap { info!(context, "{} mails read from \"{}\".", read_cnt, folder); - let msg_ids = received_msgs + let msg_ids: Vec = received_msgs .iter() .flat_map(|m| m.msg_ids.clone()) .collect(); - context.emit_event(EventType::IncomingMsgBunch { msg_ids }); + if !msg_ids.is_empty() { + context.emit_event(EventType::IncomingMsgBunch { msg_ids }); + } chat::mark_old_messages_as_noticed(context, received_msgs).await?; @@ -1725,7 +1744,19 @@ async fn should_move_out_of_spam( }; // No chat found. let (from_id, blocked_contact, _origin) = - from_field_to_contact_id(context, &from, true).await?; + match from_field_to_contact_id(context, &from, true) + .await + .context("from_field_to_contact_id")? + { + Some(res) => res, + None => { + warn!( + context, + "Contact with From address {:?} cannot exist, not moving out of spam", from + ); + return Ok(false); + } + }; if blocked_contact { // Contact is blocked, leave the message in spam. return Ok(false); @@ -2015,7 +2046,10 @@ pub(crate) async fn prefetch_should_download( None => return Ok(false), }; let (_from_id, blocked_contact, origin) = - from_field_to_contact_id(context, &from, true).await?; + match from_field_to_contact_id(context, &from, true).await? { + Some(res) => res, + None => return Ok(false), + }; // prevent_rename=true as this might be a mailing list message and in this case it would be bad if we rename the contact. // (prevent_rename is the last argument of from_field_to_contact_id()) @@ -2335,33 +2369,41 @@ async fn add_all_recipients_as_contacts( .await .with_context(|| format!("could not select {}", mailbox))?; - let contacts = imap + let recipients = imap .get_all_recipients(context) .await .context("could not get recipients")?; let mut any_modified = false; - for contact in contacts { - let display_name_normalized = contact + for recipient in recipients { + let display_name_normalized = recipient .display_name .as_ref() .map(|s| normalize_name(s)) .unwrap_or_default(); - match Contact::add_or_lookup( + let recipient_addr = match ContactAddress::new(&recipient.addr) { + Err(err) => { + warn!( + context, + "Could not add contact for recipient with address {:?}: {:#}", + recipient.addr, + err + ); + continue; + } + Ok(recipient_addr) => recipient_addr, + }; + + let (_, modified) = Contact::add_or_lookup( context, &display_name_normalized, - &contact.addr, + recipient_addr, Origin::OutgoingTo, ) - .await - { - Ok((_, modified)) => { - if modified != Modifier::None { - any_modified = true; - } - } - Err(e) => warn!(context, "Could not add recipient: {}", e), + .await?; + if modified != Modifier::None { + any_modified = true; } } if any_modified { diff --git a/src/imap/client.rs b/src/imap/client.rs index 74624a119..e371a49c2 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -4,21 +4,18 @@ use std::{ }; use anyhow::{Context as _, Result}; - use async_imap::Client as ImapClient; use async_imap::Session as ImapSession; - use tokio::io::BufWriter; -use tokio::net::ToSocketAddrs; use super::capabilities::Capabilities; use super::session::Session; +use super::session::SessionStream; +use crate::context::Context; use crate::login_param::build_tls; use crate::net::connect_tcp; use crate::socks::Socks5Config; -use super::session::SessionStream; - /// IMAP write and read timeout in seconds. pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(30); @@ -91,8 +88,13 @@ impl Client { Ok(Session::new(session, capabilities)) } - pub async fn connect_secure(hostname: &str, port: u16, strict_tls: bool) -> Result { - let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?; + pub async fn connect_secure( + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + ) -> Result { + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); @@ -107,8 +109,8 @@ impl Client { Ok(Client { inner: client }) } - pub async fn connect_insecure(addr: impl ToSocketAddrs) -> Result { - let tcp_stream = connect_tcp(addr, IMAP_TIMEOUT).await?; + pub async fn connect_insecure(context: &Context, hostname: &str, port: u16) -> Result { + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, false).await?; let buffered_stream = BufWriter::new(tcp_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -120,12 +122,16 @@ impl Client { Ok(Client { inner: client }) } - pub async fn connect_starttls(hostname: &str, port: u16, strict_tls: bool) -> Result { - let tcp_stream = connect_tcp((hostname, port), IMAP_TIMEOUT).await?; + pub async fn connect_starttls( + context: &Context, + hostname: &str, + port: u16, + strict_tls: bool, + ) -> Result { + let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?; // Run STARTTLS command and convert the client back into a stream. - let session_stream: Box = Box::new(tcp_stream); - let mut client = ImapClient::new(session_stream); + let mut client = ImapClient::new(tcp_stream); let _greeting = client .read_response() .await @@ -150,12 +156,15 @@ impl Client { } pub async fn connect_secure_socks5( + context: &Context, domain: &str, port: u16, strict_tls: bool, socks5_config: Socks5Config, ) -> Result { - let socks5_stream = socks5_config.connect((domain, port), IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config + .connect(context, domain, port, IMAP_TIMEOUT, strict_tls) + .await?; let tls = build_tls(strict_tls); let tls_stream = tls.connect(domain, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); @@ -170,10 +179,14 @@ impl Client { } pub async fn connect_insecure_socks5( - target_addr: impl ToSocketAddrs, + context: &Context, + domain: &str, + port: u16, socks5_config: Socks5Config, ) -> Result { - let socks5_stream = socks5_config.connect(target_addr, IMAP_TIMEOUT).await?; + let socks5_stream = socks5_config + .connect(context, domain, port, IMAP_TIMEOUT, false) + .await?; let buffered_stream = BufWriter::new(socks5_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -186,18 +199,18 @@ impl Client { } pub async fn connect_starttls_socks5( + context: &Context, hostname: &str, port: u16, socks5_config: Socks5Config, strict_tls: bool, ) -> Result { let socks5_stream = socks5_config - .connect((hostname, port), IMAP_TIMEOUT) + .connect(context, hostname, port, IMAP_TIMEOUT, strict_tls) .await?; // Run STARTTLS command and convert the client back into a stream. - let session_stream: Box = Box::new(socks5_stream); - let mut client = ImapClient::new(session_stream); + let mut client = ImapClient::new(socks5_stream); let _greeting = client .read_response() .await diff --git a/src/imap/idle.rs b/src/imap/idle.rs index 78232982e..bcc605c63 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -1,12 +1,12 @@ -use super::Imap; +use std::time::{Duration, SystemTime}; use anyhow::{bail, Context as _, Result}; use async_channel::Receiver; use async_imap::extensions::idle::IdleResponse; use futures_lite::FutureExt; -use std::time::{Duration, SystemTime}; use super::session::Session; +use super::Imap; use crate::imap::client::IMAP_TIMEOUT; use crate::{context::Context, scheduler::InterruptInfo}; diff --git a/src/imap/scan_folders.rs b/src/imap/scan_folders.rs index d7c732244..991b6b38f 100644 --- a/src/imap/scan_folders.rs +++ b/src/imap/scan_folders.rs @@ -3,13 +3,12 @@ use std::{collections::BTreeMap, time::Instant}; use anyhow::{Context as _, Result}; use futures::stream::StreamExt; +use super::{get_folder_meaning, get_folder_meaning_by_name}; use crate::config::Config; use crate::imap::Imap; use crate::log::LogExt; use crate::{context::Context, imap::FolderMeaning}; -use super::{get_folder_meaning, get_folder_meaning_by_name}; - impl Imap { /// Returns true if folders were scanned, false if scanning was postponed. pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result { diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index 3f04f41a2..0d12f0238 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -1,8 +1,8 @@ -use super::session::Session as ImapSession; - -use crate::context::Context; use anyhow::Context as _; +use super::session::Session as ImapSession; +use crate::context::Context; + type Result = std::result::Result; #[derive(Debug, thiserror::Error)] @@ -109,13 +109,14 @@ impl ImapSession { Ok(newly_selected) => Ok(newly_selected), Err(err) => match err { Error::NoFolder(..) => { + info!(context, "Failed to select folder {} because it does not exist, trying to create it.", folder); self.create(folder).await.with_context(|| { format!("Couldn't select folder ('{}'), then create() failed", err) })?; - Ok(self.select_folder(context, Some(folder)).await?) + Ok(self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {}", folder))?) } - _ => Err(err.into()), + _ => Err(err).with_context(|| format!("failed to select folder {} with error other than NO, not trying to create it", folder)), }, } } diff --git a/src/imex.rs b/src/imex.rs index ea4033959..e77f7baa3 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -656,7 +656,7 @@ async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> { Ok(buf) => { let armored = std::string::String::from_utf8_lossy(&buf); if let Err(err) = set_self_key(context, &armored, set_default, false).await { - error!(context, "set_self_key: {}", err); + info!(context, "set_self_key: {}", err); continue; } } @@ -769,14 +769,13 @@ where #[cfg(test)] mod tests { - use super::*; + use ::pgp::armor::BlockType; + use super::*; use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE}; use crate::stock_str::StockMessage; use crate::test_utils::{alice_keypair, TestContext}; - use ::pgp::armor::BlockType; - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_render_setup_file() { let t = TestContext::new_alice().await; diff --git a/src/job.rs b/src/job.rs index e3b513fed..7f5528ca9 100644 --- a/src/job.rs +++ b/src/job.rs @@ -157,7 +157,7 @@ impl Job { /// Synchronizes UIDs for all folders. async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status { if let Err(err) = imap.prepare(context).await { - warn!(context, "could not connect: {:?}", err); + warn!(context, "could not connect: {:#}", err); return Status::RetryLater; } @@ -246,7 +246,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ time_offset ); job.save(context).await.unwrap_or_else(|err| { - error!(context, "failed to save job: {}", err); + error!(context, "failed to save job: {:#}", err); }); } else { info!( @@ -254,7 +254,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ "remove job {} as it exhausted {} retries", job, JOB_RETRIES ); job.delete(context).await.unwrap_or_else(|err| { - error!(context, "failed to delete job: {}", err); + error!(context, "failed to delete job: {:#}", err); }); } } @@ -269,7 +269,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ } job.delete(context).await.unwrap_or_else(|err| { - error!(context, "failed to delete job: {}", err); + error!(context, "failed to delete job: {:#}", err); }); } } @@ -403,7 +403,7 @@ LIMIT 1; Ok(job) => return Ok(job), Err(err) => { // Remove invalid job from the DB - info!(context, "cleaning up job, because of {}", err); + info!(context, "cleaning up job, because of {:#}", err); // TODO: improve by only doing a single query let id = context @@ -424,7 +424,6 @@ LIMIT 1; #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestContext; async fn insert_job(context: &Context, foreign_id: i64, valid: bool) { diff --git a/src/key.rs b/src/key.rs index e2fbe7c82..73967286c 100644 --- a/src/key.rs +++ b/src/key.rs @@ -11,6 +11,7 @@ use anyhow::{ensure, Context as _, Result}; use futures::Future; use num_traits::FromPrimitive; use pgp::composed::Deserializable; +pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; use pgp::ser::Serialize; use pgp::types::{KeyTrait, SecretKeyTrait}; use tokio::runtime::Handle; @@ -18,11 +19,9 @@ use tokio::runtime::Handle; use crate::config::Config; use crate::constants::KeyGenType; use crate::context::Context; -use crate::tools::{time, EmailAddress}; - // Re-export key types pub use crate::pgp::KeyPair; -pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; +use crate::tools::{time, EmailAddress}; /// Convenience trait for working with keys. /// @@ -30,17 +29,13 @@ pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; /// [SignedSecretKey] types and makes working with them a little /// easier in the deltachat world. pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone { - type KeyType: Serialize + Deserializable + KeyTrait + Clone; - /// Create a key from some bytes. - fn from_slice(bytes: &[u8]) -> Result { - Ok(::from_bytes(Cursor::new( - bytes, - ))?) + fn from_slice(bytes: &[u8]) -> Result { + Ok(::from_bytes(Cursor::new(bytes))?) } /// Create a key from a base64 string. - fn from_base64(data: &str) -> Result { + fn from_base64(data: &str) -> Result { // strip newlines and other whitespace let cleaned: String = data.split_whitespace().collect(); let bytes = base64::decode(cleaned.as_bytes())?; @@ -51,15 +46,15 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone { /// /// Returns the key and a map of any headers which might have been set in /// the ASCII-armored representation. - fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap)> { + fn from_asc(data: &str) -> Result<(Self, BTreeMap)> { let bytes = data.as_bytes(); - Self::KeyType::from_armor_single(Cursor::new(bytes)).context("rPGP error") + Self::from_armor_single(Cursor::new(bytes)).context("rPGP error") } /// Load the users' default key from the database. fn load_self<'a>( context: &'a Context, - ) -> Pin> + 'a + Send>>; + ) -> Pin> + 'a + Send>>; /// Serialise the key as bytes. fn to_bytes(&self) -> Vec { @@ -92,11 +87,9 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone { } impl DcKey for SignedPublicKey { - type KeyType = SignedPublicKey; - fn load_self<'a>( context: &'a Context, - ) -> Pin> + 'a + Send>> { + ) -> Pin> + 'a + Send>> { Box::pin(async move { let addr = context.get_primary_self_addr().await?; match context @@ -143,11 +136,9 @@ impl DcKey for SignedPublicKey { } impl DcKey for SignedSecretKey { - type KeyType = SignedSecretKey; - fn load_self<'a>( context: &'a Context, - ) -> Pin> + 'a + Send>> { + ) -> Pin> + 'a + Send>> { Box::pin(async move { match context .sql @@ -398,11 +389,12 @@ impl std::str::FromStr for Fingerprint { #[cfg(test)] mod tests { - use super::*; - use crate::test_utils::{alice_keypair, TestContext}; + use std::sync::Arc; use once_cell::sync::Lazy; - use std::sync::Arc; + + use super::*; + use crate::test_utils::{alice_keypair, TestContext}; static KEYPAIR: Lazy = Lazy::new(alice_keypair); diff --git a/src/keyring.rs b/src/keyring.rs index 192b8de80..fa5e9b5f4 100644 --- a/src/keyring.rs +++ b/src/keyring.rs @@ -19,7 +19,7 @@ where impl Keyring where - T: DcKey, + T: DcKey, { /// New empty keyring. pub fn new() -> Keyring { diff --git a/src/lib.rs b/src/lib.rs index a6e8eb8ae..831f6609e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,6 +66,7 @@ mod decrypt; pub mod download; mod e2ee; pub mod ephemeral; +mod http; mod imap; pub mod imex; mod scheduler; diff --git a/src/location.rs b/src/location.rs index bff558912..dc6bf3af6 100644 --- a/src/location.rs +++ b/src/location.rs @@ -337,7 +337,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64 ContactId::SELF, ] ).await { - warn!(context, "failed to store location {:?}", err); + warn!(context, "failed to store location {:#}", err); } else { info!(context, "stored location for chat {}", chat_id); continue_streaming = true; @@ -638,7 +638,7 @@ pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receive loop { let next_event = match maybe_send_locations(context).await { Err(err) => { - warn!(context, "maybe_send_locations failed: {}", err); + warn!(context, "maybe_send_locations failed: {:#}", err); Some(60) // Retry one minute later. } Ok(next_event) => next_event, diff --git a/src/log.rs b/src/log.rs index 51ada7480..c755484e6 100644 --- a/src/log.rs +++ b/src/log.rs @@ -155,9 +155,10 @@ impl LogExt for Result { #[cfg(test)] mod tests { - use crate::test_utils::TestContext; use anyhow::Result; + use crate::test_utils::TestContext; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_get_last_error() -> Result<()> { let t = TestContext::new().await; diff --git a/src/login_param.rs b/src/login_param.rs index feb2e4e73..bd5541047 100644 --- a/src/login_param.rs +++ b/src/login_param.rs @@ -332,7 +332,6 @@ pub fn build_tls(strict_tls: bool) -> async_native_tls::TlsConnector { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestContext; #[test] diff --git a/src/message.rs b/src/message.rs index d05b8b366..0bde53961 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1,7 +1,5 @@ //! # Messages and their identifiers. -#![allow(missing_docs)] - use std::collections::BTreeSet; use std::path::{Path, PathBuf}; @@ -237,11 +235,18 @@ impl Default for MessengerMessage { /// If you want an update, you have to recreate the object. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct Message { + /// Message ID. pub(crate) id: MsgId, + + /// `From:` contact ID. pub(crate) from_id: ContactId, + + /// ID of the first contact in the `To:` header. pub(crate) to_id: ContactId, pub(crate) chat_id: ChatId, pub(crate) viewtype: Viewtype, + + /// State of the message. pub(crate) state: MessageState, pub(crate) download_state: DownloadState, pub(crate) hidden: bool, @@ -263,6 +268,7 @@ pub struct Message { } impl Message { + /// Creates a new message with given view type. pub fn new(viewtype: Viewtype) -> Self { Message { viewtype, @@ -270,6 +276,7 @@ impl Message { } } + /// Loads message with given ID from the database. pub async fn load_from_db(context: &Context, id: MsgId) -> Result { ensure!( !id.is_special(), @@ -366,6 +373,12 @@ impl Message { Ok(msg) } + /// Returns the MIME type of an attached file if it exists. + /// + /// If the MIME type is not known, the function guesses the MIME type + /// from the extension. `application/octet-stream` is used as a fallback + /// if MIME type is not known, but `None` is only returned if no file + /// is attached. pub fn get_filemime(&self) -> Option { if let Some(m) = self.param.get(Param::MimeType) { return Some(m.to_string()); @@ -380,11 +393,12 @@ impl Message { None } + /// Returns the full path to the file associated with a message. pub fn get_file(&self, context: &Context) -> Option { self.param.get_path(Param::File, context).unwrap_or(None) } - pub async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> { + pub(crate) async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> { if self.viewtype.has_file() { let file_param = self.param.get_path(Param::File, context)?; if let Some(path_and_filename) = file_param { @@ -442,6 +456,8 @@ impl Message { self.param.set_float(Param::SetLongitude, longitude); } + /// Returns the message timestamp for display in the UI + /// as a unix timestamp in seconds. pub fn get_timestamp(&self) -> i64 { if 0 != self.timestamp_sent { self.timestamp_sent @@ -450,10 +466,12 @@ impl Message { } } + /// Returns the message ID. pub fn get_id(&self) -> MsgId { self.id } + /// Returns the ID of the contact who wrote the message. pub fn get_from_id(&self) -> ContactId { self.from_id } @@ -463,30 +481,40 @@ impl Message { self.chat_id } + /// Returns the type of the message. pub fn get_viewtype(&self) -> Viewtype { self.viewtype } + /// Returns the state of the message. pub fn get_state(&self) -> MessageState { self.state } + /// Returns the message receive time as a unix timestamp in seconds. pub fn get_received_timestamp(&self) -> i64 { self.timestamp_rcvd } + /// Returns the timestamp of the message for sorting. pub fn get_sort_timestamp(&self) -> i64 { self.timestamp_sort } + /// Returns the text of the message. pub fn get_text(&self) -> Option { self.text.as_ref().map(|s| s.to_string()) } + /// Returns message subject. pub fn get_subject(&self) -> &str { &self.subject } + /// Returns base file name without the path. + /// The base file name includes the extension. + /// + /// To get the full path, use [`Self::get_file()`]. pub fn get_filename(&self) -> Option { self.param .get(Param::File) @@ -494,26 +522,31 @@ impl Message { .map(|name| name.to_string_lossy().to_string()) } - pub async fn get_filebytes(&self, context: &Context) -> u64 { - match self.param.get_path(Param::File, context) { - Ok(Some(path)) => get_filebytes(context, &path).await, - Ok(None) => 0, - Err(_) => 0, + /// Returns the size of the file in bytes, if applicable. + pub async fn get_filebytes(&self, context: &Context) -> Result> { + if let Some(path) = self.param.get_path(Param::File, context)? { + Ok(Some(get_filebytes(context, &path).await?)) + } else { + Ok(None) } } + /// Returns width of associated image or video file. pub fn get_width(&self) -> i32 { self.param.get_int(Param::Width).unwrap_or_default() } + /// Returns height of associated image or video file. pub fn get_height(&self) -> i32 { self.param.get_int(Param::Height).unwrap_or_default() } + /// Returns duration of associated audio or video file. pub fn get_duration(&self) -> i32 { self.param.get_int(Param::Duration).unwrap_or_default() } + /// Returns true if padlock indicating message encryption should be displayed in the UI. pub fn get_showpadlock(&self) -> bool { self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0 } @@ -523,10 +556,12 @@ impl Message { self.param.get_bool(Param::Bot).unwrap_or_default() } + /// Return the ephemeral timer duration for a message. pub fn get_ephemeral_timer(&self) -> EphemeralTimer { self.ephemeral_timer } + /// Returns the timestamp of the epehemeral message removal. pub fn get_ephemeral_timestamp(&self) -> i64 { self.ephemeral_timestamp } @@ -564,6 +599,7 @@ impl Message { // C-data in the Java code (i.e. a `long` storing a C pointer) // - We can't make a param `SenderDisplayname` for messages as sometimes the display name of a contact changes, and we want to show // the same display name over all messages from the same sender. + /// Returns the name that should be shown over the message instead of the contact display ame. pub fn get_override_sender_name(&self) -> Option { self.param .get(Param::OverrideSenderDisplayname) @@ -572,11 +608,15 @@ impl Message { // Exposing this function over the ffi instead of get_override_sender_name() would mean that at least Android Java code has // to handle raw C-data (as it is done for msg_get_summary()) - pub fn get_sender_name(&self, contact: &Contact) -> String { + pub(crate) fn get_sender_name(&self, contact: &Contact) -> String { self.get_override_sender_name() .unwrap_or_else(|| contact.get_display_name().to_string()) } + /// Returns true if a message has a deviating timestamp. + /// + /// A message has a deviating timestamp when it is sent on + /// another day as received/sorted by. pub fn has_deviating_timestamp(&self) -> bool { let cnv_to_local = gm2local_offset(); let sort_timestamp = self.get_sort_timestamp() + cnv_to_local; @@ -585,14 +625,18 @@ impl Message { sort_timestamp / 86400 != send_timestamp / 86400 } + /// Returns true if the message was successfully delivered to the outgoing server or even + /// received a read receipt. pub fn is_sent(&self) -> bool { - self.state as i32 >= MessageState::OutDelivered as i32 + self.state >= MessageState::OutDelivered } + /// Returns true if the message is a forwarded message. pub fn is_forwarded(&self) -> bool { 0 != self.param.get_int(Param::Forwarded).unwrap_or_default() } + /// Returns true if the message is an informational message. pub fn is_info(&self) -> bool { let cmd = self.param.get_cmd(); self.from_id == ContactId::INFO @@ -600,10 +644,12 @@ impl Message { || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage } + /// Returns the type of an informational message. pub fn get_info_type(&self) -> SystemMessage { self.param.get_cmd() } + /// Returns true if the message is a system message. pub fn is_system_message(&self) -> bool { let cmd = self.param.get_cmd(); cmd != SystemMessage::Unknown @@ -621,6 +667,7 @@ impl Message { self.viewtype.has_file() && self.state == MessageState::OutPreparing } + /// Returns true if the message is an Autocrypt Setup Message. pub fn is_setupmessage(&self) -> bool { if self.viewtype != Viewtype::File { return false; @@ -629,6 +676,9 @@ impl Message { self.param.get_cmd() == SystemMessage::AutocryptSetupMessage } + /// Returns the first characters of the setup code. + /// + /// This is used to pre-fill the first entry field of the setup code. pub async fn get_setupcodebegin(&self, context: &Context) -> Option { if !self.is_setupmessage() { return None; @@ -649,7 +699,7 @@ impl Message { // add room to a webrtc_instance as defined by the corresponding config-value; // the result may still be prefixed by the type - pub fn create_webrtc_instance(instance: &str, room: &str) -> String { + pub(crate) fn create_webrtc_instance(instance: &str, room: &str) -> String { let (videochat_type, mut url) = Message::parse_webrtc_instance(instance); // make sure, there is a scheme in the url @@ -706,6 +756,7 @@ impl Message { } } + /// Returns videochat URL if the message is a videochat invitation. pub fn get_videochat_url(&self) -> Option { if self.viewtype == Viewtype::VideochatInvitation { if let Some(instance) = self.param.get(Param::WebrtcRoom) { @@ -715,6 +766,7 @@ impl Message { None } + /// Returns videochat type if the message is a videochat invitation. pub fn get_videochat_type(&self) -> Option { if self.viewtype == Viewtype::VideochatInvitation { if let Some(instance) = self.param.get(Param::WebrtcRoom) { @@ -724,10 +776,16 @@ impl Message { None } + /// Sets or unsets message text. pub fn set_text(&mut self, text: Option) { self.text = text; } + /// Sets the file associated with a message. + /// + /// This function does not use the file or check if it exists, + /// the file will only be used when the message is prepared + /// for sending. pub fn set_file(&mut self, file: impl ToString, filemime: Option<&str>) { self.param.set(Param::File, file); if let Some(filemime) = filemime { @@ -745,11 +803,13 @@ impl Message { } } + /// Sets the dimensions of associated image or video file. pub fn set_dimension(&mut self, width: i32, height: i32) { self.param.set_int(Param::Width, width); self.param.set_int(Param::Height, height); } + /// Sets the duration of associated audio or video file. pub fn set_duration(&mut self, duration: i32) { self.param.set_int(Param::Duration, duration); } @@ -759,6 +819,8 @@ impl Message { self.param.set_int(Param::Reaction, 1); } + /// Changes the message width, height or duration, + /// and stores it into the database. pub async fn latefiling_mediasize( &mut self, context: &Context, @@ -823,10 +885,12 @@ impl Message { Ok(()) } + /// Returns quoted message text, if any. pub fn quoted_text(&self) -> Option { self.param.get(Param::Quote).map(|s| s.to_string()) } + /// Returns quoted message, if any. pub async fn quoted_message(&self, context: &Context) -> Result> { if self.param.get(Param::Quote).is_some() && !self.is_forwarded() { return self.parent(context).await; @@ -834,6 +898,10 @@ impl Message { Ok(None) } + /// Returns parent message according to the `In-Reply-To` header + /// if it exists in the database and is not trashed. + /// + /// `References` header is not taken into account. pub async fn parent(&self, context: &Context) -> Result> { if let Some(in_reply_to) = &self.in_reply_to { if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? { @@ -854,6 +922,7 @@ impl Message { self.param.set_int(Param::ForcePlaintext, 1); } + /// Updates `param` column of the message in the database without changing other columns. pub async fn update_param(&self, context: &Context) -> Result<()> { context .sql @@ -893,12 +962,17 @@ impl Message { } } +/// State of the message. +/// For incoming messages, stores the information on whether the message was read or not. +/// For outgoing message, the message could be pending, already delivered or confirmed. #[derive( Debug, Clone, Copy, PartialEq, Eq, + PartialOrd, + Ord, FromPrimitive, ToPrimitive, ToSql, @@ -908,6 +982,7 @@ impl Message { )] #[repr(u32)] pub enum MessageState { + /// Undefined message state. Undefined = 0, /// Incoming *fresh* message. Fresh messages are neither noticed @@ -978,6 +1053,7 @@ impl std::fmt::Display for MessageState { } impl MessageState { + /// Returns true if the message can transition to `OutFailed` state from the current state. pub fn can_fail(self) -> bool { use MessageState::*; matches!( @@ -985,6 +1061,8 @@ impl MessageState { OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed. ) } + + /// Returns true for any outgoing message states. pub fn is_outgoing(self) -> bool { use MessageState::*; matches!( @@ -994,6 +1072,7 @@ impl MessageState { } } +/// Returns detailed message information in a multi-line text form. pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { let msg = Message::load_from_db(context, msg_id).await?; let rawtxt: Option = context @@ -1100,8 +1179,8 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { } if let Some(path) = msg.get_file(context) { - let bytes = get_filebytes(context, &path).await; - ret += &format!("\nFile: {}, {}, bytes\n", path.display(), bytes); + let bytes = get_filebytes(context, &path).await?; + ret += &format!("\nFile: {}, {} bytes\n", path.display(), bytes); } if msg.viewtype != Viewtype::Text { @@ -1158,7 +1237,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result { Ok(ret) } -pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> { +pub(crate) fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> { let extension: &str = &path.extension()?.to_str()?.to_lowercase(); let info = match extension { // before using viewtype other than Viewtype::File, @@ -1271,6 +1350,9 @@ pub async fn get_mime_headers(context: &Context, msg_id: MsgId) -> Result Result<()> { for msg_id in msg_ids.iter() { let msg = Message::load_from_db(context, *msg_id).await?; @@ -1318,6 +1400,7 @@ async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> Ok(()) } +/// Marks requested messages as seen. pub async fn markseen_msgs(context: &Context, msg_ids: Vec) -> Result<()> { if msg_ids.is_empty() { return Ok(()); @@ -1450,7 +1533,8 @@ pub(crate) async fn update_msg_state( // Context functions to work with messages -pub async fn exists(context: &Context, msg_id: MsgId) -> Result { +/// Returns true if given message ID exists in the database and is not trashed. +pub(crate) async fn exists(context: &Context, msg_id: MsgId) -> Result { if msg_id.is_special() { return Ok(false); } @@ -1467,7 +1551,7 @@ pub async fn exists(context: &Context, msg_id: MsgId) -> Result { } } -pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: &str) { +pub(crate) async fn set_msg_failed(context: &Context, msg_id: MsgId, error: &str) { if let Ok(mut msg) = Message::load_from_db(context, msg_id).await { if msg.state.can_fail() { msg.state = MessageState::OutFailed; @@ -1687,7 +1771,7 @@ pub async fn get_unblocked_msg_cnt(context: &Context) -> usize { { Ok(res) => res, Err(err) => { - error!(context, "get_unblocked_msg_cnt() failed. {}", err); + error!(context, "get_unblocked_msg_cnt() failed. {:#}", err); 0 } } @@ -1707,12 +1791,26 @@ pub async fn get_request_msg_cnt(context: &Context) -> usize { { Ok(res) => res, Err(err) => { - error!(context, "get_request_msg_cnt() failed. {}", err); + error!(context, "get_request_msg_cnt() failed. {:#}", err); 0 } } } +/// Estimates the number of messages that will be deleted +/// by the options `delete_device_after` or `delete_server_after`. +/// This is typically used to show the estimated impact to the user +/// before actually enabling deletion of old messages. +/// +/// If `from_server` is true, +/// estimate deletion count for server, +/// otherwise estimate deletion count for device. +/// +/// Count messages older than the given number of `seconds`. +/// +/// Returns the number of messages that are older than the given number of seconds. +/// This includes e-mails downloaded due to the `show_emails` option. +/// Messages in the "saved messages" folder are not counted as they will not be deleted automatically. pub async fn estimate_deletion_cnt( context: &Context, from_server: bool, @@ -1801,6 +1899,7 @@ pub(crate) async fn rfc724_mid_exists( )] #[repr(u32)] pub enum Viewtype { + /// Unknown message type. Unknown = 0, /// Text message. @@ -1883,13 +1982,12 @@ impl Viewtype { mod tests { use num_traits::FromPrimitive; + use super::*; use crate::chat::{marknoticed_chat, ChatItem}; use crate::chatlist::Chatlist; use crate::receive_imf::receive_imf; use crate::test_utils as test; - use crate::test_utils::TestContext; - - use super::*; + use crate::test_utils::{TestContext, TestContextManager}; #[test] fn test_guess_msgtype_from_suffix() { @@ -2374,8 +2472,9 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_format_flowed_round_trip() -> Result<()> { - let alice = TestContext::new_alice().await; - let bob = TestContext::new_bob().await; + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + let bob = tcm.bob().await; let chat = alice.create_chat(&bob).await; let text = " Foo bar"; @@ -2388,6 +2487,11 @@ mod tests { let received = bob.recv_msg(&sent).await; assert_eq!(received.text.as_deref(), Some(text)); + let text = "> xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx > A"; + let sent = alice.send_text(chat.id, text).await; + let received = bob.recv_msg(&sent).await; + assert_eq!(received.text.as_deref(), Some(text)); + let python_program = "\ def hello(): return 'Hello, world!'"; diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 43f3e5886..14e6563d2 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -76,6 +76,7 @@ pub struct MimeFactory<'a> { /// and must be deleted if the message is actually queued for sending. sync_ids_to_delete: Option, + /// True if the avatar should be attached. attach_selfavatar: bool, } @@ -689,7 +690,9 @@ impl<'a> MimeFactory<'a> { .fold(message, |message, header| message.header(header)); // Add gossip headers in chats with multiple recipients - if peerstates.len() > 1 && self.should_do_gossip(context).await? { + if (peerstates.len() > 1 || context.get_config_bool(Config::BccSelf).await?) + && self.should_do_gossip(context).await? + { for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) { if peerstate.peek_key(min_verified).is_some() { if let Some(header) = peerstate.render_gossip_header(min_verified) { @@ -722,9 +725,11 @@ impl<'a> MimeFactory<'a> { )); if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { - info!(context, "mimefactory: outgoing message mime:"); - let raw_message = message.clone().build().as_string(); - println!("{}", raw_message); + info!( + context, + "mimefactory: unencrypted message mime-body:\n{}", + message.clone().build().as_string(), + ); } let encrypted = encrypt_helper @@ -782,6 +787,14 @@ impl<'a> MimeFactory<'a> { .into_iter() .fold(outer_message, |message, header| message.header(header)); + if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { + info!( + context, + "mimefactory: outgoing message mime-body:\n{}", + outer_message.clone().build().as_string(), + ); + } + let MimeFactory { last_added_location_id, .. @@ -905,6 +918,17 @@ impl<'a> MimeFactory<'a> { "Secure-Join".to_string(), "vg-member-added".to_string(), )); + // FIXME: Old clients require Secure-Join-Fingerprint header. Remove this + // eventually. + let fingerprint = Peerstate::from_addr(context, email_to_add) + .await? + .context("No peerstate found in db")? + .public_key_fingerprint + .context("No public key fingerprint in db for the member to add")?; + headers.protected.push(Header::new( + "Secure-Join-Fingerprint".into(), + fingerprint.hex(), + )); } } SystemMessage::GroupNameChanged => { @@ -1432,7 +1456,7 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool async fn is_file_size_okay(context: &Context, msg: &Message) -> Result { match msg.param.get_path(Param::File, context)? { Some(path) => { - let bytes = get_filebytes(context, &path).await; + let bytes = get_filebytes(context, &path).await?; Ok(bytes <= UPPER_LIMIT_FILE_SIZE) } None => Ok(false), @@ -1484,18 +1508,17 @@ fn maybe_encode_words(words: &str) -> String { mod tests { use mailparse::{addrparse_header, MailHeaderMap}; + use super::*; use crate::chat::ChatId; use crate::chat::{ self, add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg, ProtectionStatus, }; use crate::chatlist::Chatlist; - use crate::contact::Origin; + use crate::contact::{ContactAddress, Origin}; use crate::mimeparser::MimeMessage; use crate::receive_imf::receive_imf; use crate::test_utils::{get_chat_msg, TestContext}; - - use super::*; #[test] fn test_render_email_address() { let display_name = "ä space"; @@ -1817,11 +1840,15 @@ mod tests { } async fn first_subject_str(t: TestContext) -> String { - let contact_id = - Contact::add_or_lookup(&t, "Dave", "dave@example.com", Origin::ManuallyCreated) - .await - .unwrap() - .0; + let contact_id = Contact::add_or_lookup( + &t, + "Dave", + ContactAddress::new("dave@example.com").unwrap(), + Origin::ManuallyCreated, + ) + .await + .unwrap() + .0; let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap(); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index b6f7fd25f..f8af8b702 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -246,14 +246,17 @@ impl MimeMessage { mail_raw = raw; let decrypted_mail = mailparse::parse_mail(&mail_raw)?; if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { - info!(context, "decrypted message mime-body:"); - println!("{}", String::from_utf8_lossy(&mail_raw)); + info!( + context, + "decrypted message mime-body:\n{}", + String::from_utf8_lossy(&mail_raw), + ); } (Ok(decrypted_mail), signatures, true) } Ok(None) => (Ok(mail), HashSet::new(), false), Err(err) => { - warn!(context, "decryption failed: {}", err); + warn!(context, "decryption failed: {:#}", err); (Err(err), HashSet::new(), false) } }; @@ -382,7 +385,7 @@ impl MimeMessage { typ: Viewtype::Text, msg_raw: Some(txt.clone()), msg: txt, - error: Some(format!("Decrypting failed: {}", err)), + error: Some(format!("Decrypting failed: {:#}", err)), ..Default::default() }; parser.parts.push(part); @@ -682,7 +685,7 @@ impl MimeMessage { Err(err) => { warn!( context, - "Could not save decoded avatar to blob file: {}", err + "Could not save decoded avatar to blob file: {:#}", err ); None } @@ -989,7 +992,7 @@ impl MimeMessage { let decoded_data = match mail.get_body() { Ok(decoded_data) => decoded_data, Err(err) => { - warn!(context, "Invalid body parsed {:?}", err); + warn!(context, "Invalid body parsed {:#}", err); // Note that it's not always an error - might be no data return Ok(false); } @@ -1009,7 +1012,7 @@ impl MimeMessage { let decoded_data = match mail.get_body() { Ok(decoded_data) => decoded_data, Err(err) => { - warn!(context, "Invalid body parsed {:?}", err); + warn!(context, "Invalid body parsed {:#}", err); // Note that it's not always an error - might be no data return Ok(false); } @@ -1141,7 +1144,7 @@ impl MimeMessage { if filename.starts_with("location") || filename.starts_with("message") { let parsed = location::Kml::parse(decoded_data) .map_err(|err| { - warn!(context, "failed to parse kml part: {}", err); + warn!(context, "failed to parse kml part: {:#}", err); }) .ok(); if filename.starts_with("location") { @@ -1159,7 +1162,7 @@ impl MimeMessage { self.sync_items = context .parse_sync_items(serialized) .map_err(|err| { - warn!(context, "failed to parse sync data: {}", err); + warn!(context, "failed to parse sync data: {:#}", err); }) .ok(); return Ok(()); @@ -1181,7 +1184,7 @@ impl MimeMessage { Err(err) => { error!( context, - "Could not add blob for mime part {}, error {}", filename, err + "Could not add blob for mime part {}, error {:#}", filename, err ); return Ok(()); } @@ -1226,7 +1229,7 @@ impl MimeMessage { Err(err) => { warn!( context, - "PGP key attachment is not an ASCII-armored file: {}", err, + "PGP key attachment is not an ASCII-armored file: {:#}", err ); return Ok(false); } @@ -1952,6 +1955,8 @@ where mod tests { #![allow(clippy::indexing_slicing)] + use mailparse::ParsedMail; + use super::*; use crate::{ chatlist::Chatlist, @@ -1961,7 +1966,6 @@ mod tests { receive_imf::receive_imf, test_utils::TestContext, }; - use mailparse::ParsedMail; impl AvatarAction { pub fn is_change(&self) -> bool { @@ -3147,7 +3151,7 @@ On 2020-10-25, Bob wrote: assert_eq!(msg.is_dc_message, MessengerMessage::No); assert_eq!(msg.chat_blocked, Blocked::Request); assert_eq!(msg.state, MessageState::InFresh); - assert_eq!(msg.get_filebytes(&t).await, 2115); + assert_eq!(msg.get_filebytes(&t).await.unwrap().unwrap(), 2115); assert!(msg.get_file(&t).is_some()); assert_eq!(msg.get_filename().unwrap(), "avatar64x64.png"); assert_eq!(msg.get_width(), 64); diff --git a/src/net.rs b/src/net.rs index 6c0c7dd0b..c1065ea9e 100644 --- a/src/net.rs +++ b/src/net.rs @@ -1,25 +1,180 @@ ///! # Common network utilities. +use std::net::{IpAddr, SocketAddr}; use std::pin::Pin; +use std::str::FromStr; use std::time::Duration; -use anyhow::{Context as _, Result}; -use tokio::net::{TcpStream, ToSocketAddrs}; +use anyhow::{Context as _, Error, Result}; +use tokio::net::{lookup_host, TcpStream}; use tokio::time::timeout; use tokio_io_timeout::TimeoutStream; +use crate::context::Context; +use crate::tools::time; + +async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result { + let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) + .await + .context("connection timeout")? + .context("connection failure")?; + Ok(tcp_stream) +} + +async fn lookup_host_with_timeout( + hostname: &str, + port: u16, + timeout_val: Duration, +) -> Result> { + let res = timeout(timeout_val, lookup_host((hostname, port))) + .await + .context("DNS lookup timeout")? + .context("DNS lookup failure")?; + Ok(res.collect()) +} + +/// Looks up hostname and port using DNS and updates the address resolution cache. +/// +/// If `load_cache` is true, appends cached results not older than 30 days to the end. +async fn lookup_host_with_cache( + context: &Context, + hostname: &str, + port: u16, + timeout_val: Duration, + load_cache: bool, +) -> Result> { + let now = time(); + let mut resolved_addrs = match lookup_host_with_timeout(hostname, port, timeout_val).await { + Ok(res) => res, + Err(err) => { + warn!( + context, + "DNS resolution for {}:{} failed: {:#}.", hostname, port, err + ); + Vec::new() + } + }; + + for addr in resolved_addrs.iter() { + let ip_string = addr.ip().to_string(); + if ip_string == hostname { + // IP address resolved into itself, not interesting to cache. + continue; + } + + info!(context, "Resolved {}:{} into {}.", hostname, port, &addr); + + // Update the cache. + context + .sql + .execute( + "INSERT INTO dns_cache + (hostname, address, timestamp) + VALUES (?, ?, ?) + ON CONFLICT (hostname, address) + DO UPDATE SET timestamp=excluded.timestamp", + paramsv![hostname, ip_string, now], + ) + .await?; + } + + if load_cache { + for cached_address in context + .sql + .query_map( + "SELECT address + FROM dns_cache + WHERE hostname = ? + AND ? < timestamp + 30 * 24 * 3600 + ORDER BY timestamp DESC", + paramsv![hostname, now], + |row| { + let address: String = row.get(0)?; + Ok(address) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await? + { + match IpAddr::from_str(&cached_address) { + Ok(ip_addr) => { + let addr = SocketAddr::new(ip_addr, port); + if !resolved_addrs.contains(&addr) { + resolved_addrs.push(addr); + } + } + Err(err) => { + warn!( + context, + "Failed to parse cached address {:?}: {:#}.", cached_address, err + ); + } + } + } + } + + Ok(resolved_addrs) +} + /// Returns a TCP connection stream with read/write timeouts set /// and Nagle's algorithm disabled with `TCP_NODELAY`. /// /// `TCP_NODELAY` ensures writing to the stream always results in immediate sending of the packet /// to the network, which is important to reduce the latency of interactive protocols such as IMAP. +/// +/// If `load_cache` is true, may use cached DNS results. +/// Because the cache may be poisoned with incorrect results by networks hijacking DNS requests, +/// this option should only be used when connection is authenticated, +/// for example using TLS. +/// If TLS is not used or invalid TLS certificates are allowed, +/// this option should be disabled. pub(crate) async fn connect_tcp( - addr: impl ToSocketAddrs, + context: &Context, + host: &str, + port: u16, timeout_val: Duration, + load_cache: bool, ) -> Result>>> { - let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) - .await - .context("connection timeout")? - .context("connection failure")?; + let mut tcp_stream = None; + let mut last_error = None; + + for resolved_addr in + lookup_host_with_cache(context, host, port, timeout_val, load_cache).await? + { + match connect_tcp_inner(resolved_addr, timeout_val).await { + Ok(stream) => { + tcp_stream = Some(stream); + + // Maximize priority of this cached entry. + context + .sql + .execute( + "UPDATE dns_cache + SET timestamp = ? + WHERE address = ?", + paramsv![time(), resolved_addr.ip().to_string()], + ) + .await?; + break; + } + Err(err) => { + warn!( + context, + "Failed to connect to {}: {:#}.", resolved_addr, err + ); + last_error = Some(err); + } + } + } + + let tcp_stream = match tcp_stream { + Some(tcp_stream) => tcp_stream, + None => { + return Err(last_error.unwrap_or_else(|| Error::msg("no DNS resolution results"))); + } + }; // Disable Nagle's algorithm. tcp_stream.set_nodelay(true)?; diff --git a/src/oauth2.rs b/src/oauth2.rs index d48cb45a6..bde9f2240 100644 --- a/src/oauth2.rs +++ b/src/oauth2.rs @@ -158,7 +158,7 @@ pub async fn get_oauth2_access_token( } // ... and POST - let client = reqwest::Client::new(); + let client = crate::http::get_client()?; let response: Response = match client.post(post_url).form(&post_param).send().await { Ok(resp) => match resp.json().await { @@ -284,7 +284,14 @@ impl Oauth2 { // "verified_email": true, // "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg" // } - let response = match reqwest::get(userinfo_url).await { + let client = match crate::http::get_client() { + Ok(cl) => cl, + Err(err) => { + warn!(context, "failed to get HTTP client: {}", err); + return None; + } + }; + let response = match client.get(userinfo_url).send().await { Ok(response) => response, Err(err) => { warn!(context, "failed to get userinfo: {}", err); @@ -345,7 +352,6 @@ fn normalize_addr(addr: &str) -> &str { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::TestContext; #[test] diff --git a/src/param.rs b/src/param.rs index 69e587e18..55bf5f930 100644 --- a/src/param.rs +++ b/src/param.rs @@ -435,14 +435,13 @@ impl<'a> ParamsFile<'a> { #[cfg(test)] mod tests { - use super::*; - use std::path::Path; use std::str::FromStr; use anyhow::Result; use tokio::fs; + use super::*; use crate::test_utils::TestContext; #[test] diff --git a/src/peerstate.rs b/src/peerstate.rs index 2699bf6eb..c3f3b1a68 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -3,13 +3,15 @@ #![allow(missing_docs)] use std::collections::HashSet; -use std::fmt; + +use anyhow::{Context as _, Error, Result}; +use num_traits::FromPrimitive; use crate::aheader::{Aheader, EncryptPreference}; use crate::chat::{self, Chat}; use crate::chatlist::Chatlist; use crate::constants::Chattype; -use crate::contact::{addr_cmp, Contact, Origin}; +use crate::contact::{addr_cmp, Contact, ContactAddress, Origin}; use crate::context::Context; use crate::events::EventType; use crate::key::{DcKey, Fingerprint, SignedPublicKey}; @@ -17,8 +19,6 @@ use crate::message::Message; use crate::mimeparser::SystemMessage; use crate::sql::Sql; use crate::stock_str; -use anyhow::{Context as _, Result}; -use num_traits::FromPrimitive; #[derive(Debug)] pub enum PeerstateKeyType { @@ -35,6 +35,7 @@ pub enum PeerstateVerifiedStatus { } /// Peerstate represents the state of an Autocrypt peer. +#[derive(Debug, PartialEq, Eq)] pub struct Peerstate { pub addr: String, pub last_seen: i64, @@ -52,44 +53,6 @@ pub struct Peerstate { pub verifier: Option, } -impl PartialEq for Peerstate { - fn eq(&self, other: &Peerstate) -> bool { - self.addr == other.addr - && self.last_seen == other.last_seen - && self.last_seen_autocrypt == other.last_seen_autocrypt - && self.prefer_encrypt == other.prefer_encrypt - && self.public_key == other.public_key - && self.public_key_fingerprint == other.public_key_fingerprint - && self.gossip_key == other.gossip_key - && self.gossip_timestamp == other.gossip_timestamp - && self.gossip_key_fingerprint == other.gossip_key_fingerprint - && self.verified_key == other.verified_key - && self.verified_key_fingerprint == other.verified_key_fingerprint - && self.fingerprint_changed == other.fingerprint_changed - } -} - -impl Eq for Peerstate {} - -impl fmt::Debug for Peerstate { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("Peerstate") - .field("addr", &self.addr) - .field("last_seen", &self.last_seen) - .field("last_seen_autocrypt", &self.last_seen_autocrypt) - .field("prefer_encrypt", &self.prefer_encrypt) - .field("public_key", &self.public_key) - .field("public_key_fingerprint", &self.public_key_fingerprint) - .field("gossip_key", &self.gossip_key) - .field("gossip_timestamp", &self.gossip_timestamp) - .field("gossip_key_fingerprint", &self.gossip_key_fingerprint) - .field("verified_key", &self.verified_key) - .field("verified_key_fingerprint", &self.verified_key_fingerprint) - .field("fingerprint_changed", &self.fingerprint_changed) - .finish() - } -} - impl Peerstate { pub fn from_header(header: &Aheader, message_time: i64) -> Self { Peerstate { @@ -223,7 +186,10 @@ impl Peerstate { .transpose() .unwrap_or_default(), fingerprint_changed: false, - verifier: row.get("verifier")?, + verifier: { + let verifier: Option = row.get("verifier")?; + verifier.filter(|verifier| !verifier.is_empty()) + }, }; Ok(res) @@ -369,43 +335,48 @@ impl Peerstate { /// verifier: /// The address which verifies the given contact /// If we are verifying the contact, use that contacts address - /// Returns whether the value of the key has changed pub fn set_verified( &mut self, which_key: PeerstateKeyType, - fingerprint: &Fingerprint, + fingerprint: Fingerprint, verified: PeerstateVerifiedStatus, verifier: String, - ) -> bool { + ) -> Result<()> { if verified == PeerstateVerifiedStatus::BidirectVerified { match which_key { PeerstateKeyType::PublicKey => { if self.public_key_fingerprint.is_some() - && self.public_key_fingerprint.as_ref().unwrap() == fingerprint + && self.public_key_fingerprint.as_ref().unwrap() == &fingerprint { self.verified_key = self.public_key.clone(); - self.verified_key_fingerprint = self.public_key_fingerprint.clone(); + self.verified_key_fingerprint = Some(fingerprint); self.verifier = Some(verifier); - true + Ok(()) } else { - false + Err(Error::msg(format!( + "{} is not peer's public key fingerprint", + fingerprint, + ))) } } PeerstateKeyType::GossipKey => { if self.gossip_key_fingerprint.is_some() - && self.gossip_key_fingerprint.as_ref().unwrap() == fingerprint + && self.gossip_key_fingerprint.as_ref().unwrap() == &fingerprint { self.verified_key = self.gossip_key.clone(); - self.verified_key_fingerprint = self.gossip_key_fingerprint.clone(); + self.verified_key_fingerprint = Some(fingerprint); self.verifier = Some(verifier); - true + Ok(()) } else { - false + Err(Error::msg(format!( + "{} is not peer's gossip key fingerprint", + fingerprint, + ))) } } } } else { - false + Err(Error::msg("BidirectVerified required")) } } @@ -450,7 +421,7 @@ impl Peerstate { self.verified_key.as_ref().map(|k| k.to_bytes()), self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()), self.addr, - self.verifier, + self.verifier.as_deref().unwrap_or(""), ], ) .await?; @@ -542,14 +513,31 @@ impl Peerstate { if (chat.typ == Chattype::Group && chat.is_protected()) || chat.typ == Chattype::Broadcast { - chat::remove_from_chat_contacts_table(context, *chat_id, contact_id).await?; - - let (new_contact_id, _) = - Contact::add_or_lookup(context, "", new_addr, Origin::IncomingUnknownFrom) + match ContactAddress::new(new_addr) { + Ok(new_addr) => { + let (new_contact_id, _) = Contact::add_or_lookup( + context, + "", + new_addr, + Origin::IncomingUnknownFrom, + ) .await?; - chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]).await?; + chat::remove_from_chat_contacts_table(context, *chat_id, contact_id) + .await?; + chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]) + .await?; - context.emit_event(EventType::ChatModified(*chat_id)); + context.emit_event(EventType::ChatModified(*chat_id)); + } + Err(err) => { + warn!( + context, + "New address {:?} is not vaild, not doing AEAP: {:#}.", + new_addr, + err + ) + } + } } } diff --git a/src/pgp.rs b/src/pgp.rs index 93762e68a..cacdecae0 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -382,11 +382,12 @@ pub async fn symm_decrypt( #[cfg(test)] mod tests { - use super::*; - use crate::test_utils::{alice_keypair, bob_keypair}; use once_cell::sync::Lazy; use tokio::sync::OnceCell; + use super::*; + use crate::test_utils::{alice_keypair, bob_keypair}; + #[test] fn test_split_armored_data_1() { let (typ, _headers, base64) = split_armored_data( diff --git a/src/plaintext.rs b/src/plaintext.rs index 5bdeffed4..13dd1ac7c 100644 --- a/src/plaintext.rs +++ b/src/plaintext.rs @@ -2,9 +2,10 @@ #![allow(missing_docs)] -use crate::simplify::split_lines; use once_cell::sync::Lazy; +use crate::simplify::split_lines; + #[derive(Debug)] pub struct PlainText { pub text: String, diff --git a/src/provider.rs b/src/provider.rs index 4d8ee74d8..5d81b642b 100644 --- a/src/provider.rs +++ b/src/provider.rs @@ -4,13 +4,14 @@ mod data; -use crate::config::Config; -use crate::context::Context; -use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED}; use anyhow::Result; use chrono::{NaiveDateTime, NaiveTime}; use trust_dns_resolver::{config, AsyncResolver, TokioAsyncResolver}; +use crate::config::Config; +use crate::context::Context; +use crate::provider::data::{PROVIDER_DATA, PROVIDER_IDS, PROVIDER_UPDATED}; + #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)] #[repr(u8)] pub enum Status { @@ -195,10 +196,11 @@ pub fn get_provider_update_timestamp() -> i64 { mod tests { #![allow(clippy::indexing_slicing)] + use chrono::NaiveDate; + use super::*; use crate::test_utils::TestContext; use crate::tools::time; - use chrono::NaiveDate; #[test] fn test_get_provider_by_domain_unexistant() { diff --git a/src/provider/data.rs b/src/provider/data.rs index 3b8b4b642..21084f821 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -1,12 +1,13 @@ // file generated by src/provider/update.py +use std::collections::HashMap; + +use once_cell::sync::Lazy; + use crate::provider::Protocol::*; use crate::provider::Socket::*; use crate::provider::UsernamePattern::*; use crate::provider::{Config, ConfigDefault, Oauth2Authorizer, Provider, Server, Status}; -use std::collections::HashMap; - -use once_cell::sync::Lazy; // 163.md: 163.com static P_163: Lazy = Lazy::new(|| Provider { @@ -526,7 +527,7 @@ static P_GMX_NET: Lazy = Lazy::new(|| Provider { oauth2_authorizer: None, }); -// hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, hermes.radio +// hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, ec.hermes.radio, ec1.hermes.radio, ec2.hermes.radio, ec3.hermes.radio, ec4.hermes.radio, ec5.hermes.radio, ec6.hermes.radio, ec7.hermes.radio, ec8.hermes.radio, ec9.hermes.radio, ec10.hermes.radio, ec11.hermes.radio, ec12.hermes.radio, ec13.hermes.radio, ec14.hermes.radio, ec15.hermes.radio, hermes.radio static P_HERMES_RADIO: Lazy = Lazy::new(|| Provider { id: "hermes.radio", status: Status::Ok, @@ -902,6 +903,35 @@ static P_NAVER: Lazy = Lazy::new(|| Provider { oauth2_authorizer: None, }); +// nubo.coop.md: nubo.coop +static P_NUBO_COOP: Lazy = Lazy::new(|| Provider { + id: "nubo.coop", + status: Status::Ok, + before_login_hint: "", + after_login_hint: "", + overview_page: "https://providers.delta.chat/nubo-coop", + server: vec![ + Server { + protocol: Imap, + socket: Ssl, + hostname: "mail.nubo.coop", + port: 993, + username_pattern: Email, + }, + Server { + protocol: Smtp, + socket: Ssl, + hostname: "mail.nubo.coop", + port: 465, + username_pattern: Email, + }, + ], + config_defaults: None, + strict_tls: true, + max_smtp_rcpt_to: None, + oauth2_authorizer: None, +}); + // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com, outlook.de static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { id: "outlook.com", @@ -931,6 +961,35 @@ static P_OUTLOOK_COM: Lazy = Lazy::new(|| Provider { oauth2_authorizer: None, }); +// ouvaton.coop.md: ouvaton.org +static P_OUVATON_COOP: Lazy = Lazy::new(|| Provider { + id: "ouvaton.coop", + status: Status::Ok, + before_login_hint: "", + after_login_hint: "", + overview_page: "https://providers.delta.chat/ouvaton-coop", + server: vec![ + Server { + protocol: Imap, + socket: Ssl, + hostname: "imap.ouvaton.coop", + port: 993, + username_pattern: Email, + }, + Server { + protocol: Smtp, + socket: Ssl, + hostname: "smtp.ouvaton.coop", + port: 465, + username_pattern: Email, + }, + ], + config_defaults: None, + strict_tls: true, + max_smtp_rcpt_to: None, + oauth2_authorizer: None, +}); + // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us static P_POSTEO: Lazy = Lazy::new(|| Provider { id: "posteo", @@ -1659,6 +1718,22 @@ pub(crate) static PROVIDER_DATA: Lazy> ("ka13.hermes.radio", &*P_HERMES_RADIO), ("ka14.hermes.radio", &*P_HERMES_RADIO), ("ka15.hermes.radio", &*P_HERMES_RADIO), + ("ec.hermes.radio", &*P_HERMES_RADIO), + ("ec1.hermes.radio", &*P_HERMES_RADIO), + ("ec2.hermes.radio", &*P_HERMES_RADIO), + ("ec3.hermes.radio", &*P_HERMES_RADIO), + ("ec4.hermes.radio", &*P_HERMES_RADIO), + ("ec5.hermes.radio", &*P_HERMES_RADIO), + ("ec6.hermes.radio", &*P_HERMES_RADIO), + ("ec7.hermes.radio", &*P_HERMES_RADIO), + ("ec8.hermes.radio", &*P_HERMES_RADIO), + ("ec9.hermes.radio", &*P_HERMES_RADIO), + ("ec10.hermes.radio", &*P_HERMES_RADIO), + ("ec11.hermes.radio", &*P_HERMES_RADIO), + ("ec12.hermes.radio", &*P_HERMES_RADIO), + ("ec13.hermes.radio", &*P_HERMES_RADIO), + ("ec14.hermes.radio", &*P_HERMES_RADIO), + ("ec15.hermes.radio", &*P_HERMES_RADIO), ("hermes.radio", &*P_HERMES_RADIO), ("hey.com", &*P_HEY_COM), ("i.ua", &*P_I_UA), @@ -1681,12 +1756,14 @@ pub(crate) static PROVIDER_DATA: Lazy> ("mailo.com", &*P_MAILO_COM), ("nauta.cu", &*P_NAUTA_CU), ("naver.com", &*P_NAVER), + ("nubo.coop", &*P_NUBO_COOP), ("hotmail.com", &*P_OUTLOOK_COM), ("outlook.com", &*P_OUTLOOK_COM), ("office365.com", &*P_OUTLOOK_COM), ("outlook.com.tr", &*P_OUTLOOK_COM), ("live.com", &*P_OUTLOOK_COM), ("outlook.de", &*P_OUTLOOK_COM), + ("ouvaton.org", &*P_OUVATON_COOP), ("posteo.de", &*P_POSTEO), ("posteo.af", &*P_POSTEO), ("posteo.at", &*P_POSTEO), @@ -1861,7 +1938,9 @@ pub(crate) static PROVIDER_IDS: Lazy> = ("mailo.com", &*P_MAILO_COM), ("nauta.cu", &*P_NAUTA_CU), ("naver", &*P_NAVER), + ("nubo.coop", &*P_NUBO_COOP), ("outlook.com", &*P_OUTLOOK_COM), + ("ouvaton.coop", &*P_OUVATON_COOP), ("posteo", &*P_POSTEO), ("protonmail", &*P_PROTONMAIL), ("qq", &*P_QQ), @@ -1891,4 +1970,4 @@ pub(crate) static PROVIDER_IDS: Lazy> = }); pub static PROVIDER_UPDATED: Lazy = - Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2022, 7, 5).unwrap()); + Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 1, 6).unwrap()); diff --git a/src/provider/update.py b/src/provider/update.py index 9031e85b4..b67e9fb59 100755 --- a/src/provider/update.py +++ b/src/provider/update.py @@ -190,6 +190,6 @@ if __name__ == "__main__": now = datetime.datetime.utcnow() out_all += "pub static PROVIDER_UPDATED: Lazy = "\ - "Lazy::new(|| chrono::NaiveDate::from_ymd("+str(now.year)+", "+str(now.month)+", "+str(now.day)+"));\n" + "Lazy::new(|| chrono::NaiveDate::from_ymd_opt("+str(now.year)+", "+str(now.month)+", "+str(now.day)+").unwrap());\n" print(out_all) diff --git a/src/qr.rs b/src/qr.rs index 2813c762d..7c14c4de0 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -3,18 +3,21 @@ #![allow(missing_docs)] mod dclogin_scheme; -pub use dclogin_scheme::LoginOptions; +use std::collections::BTreeMap; use anyhow::{anyhow, bail, ensure, Context as _, Error, Result}; +pub use dclogin_scheme::LoginOptions; use once_cell::sync::Lazy; use percent_encoding::percent_decode_str; use serde::Deserialize; -use std::collections::BTreeMap; +use self::dclogin_scheme::configure_from_login_qr; use crate::chat::{self, get_chat_id_by_grpid, ChatIdBlocked}; use crate::config::Config; use crate::constants::Blocked; -use crate::contact::{addr_normalize, may_be_valid_addr, Contact, ContactId, Origin}; +use crate::contact::{ + addr_normalize, may_be_valid_addr, Contact, ContactAddress, ContactId, Origin, +}; use crate::context::Context; use crate::key::Fingerprint; use crate::message::Message; @@ -22,8 +25,6 @@ use crate::peerstate::Peerstate; use crate::tools::time; use crate::{token, EventType}; -use self::dclogin_scheme::configure_from_login_qr; - const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase const DCACCOUNT_SCHEME: &str = "DCACCOUNT:"; pub(super) const DCLOGIN_SCHEME: &str = "DCLOGIN:"; @@ -221,14 +222,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { .context("Can't load peerstate")?; if let (Some(addr), Some(invitenumber), Some(authcode)) = (&addr, invitenumber, authcode) { - let contact_id = Contact::add_or_lookup(context, &name, addr, Origin::UnhandledQrScan) + let addr = ContactAddress::new(addr)?; + let (contact_id, _) = Contact::add_or_lookup(context, &name, addr, Origin::UnhandledQrScan) .await - .map(|(id, _)| id) .with_context(|| format!("failed to add or lookup contact for address {:?}", addr))?; if let (Some(grpid), Some(grpname)) = (grpid, grpname) { if context - .is_self_addr(addr) + .is_self_addr(&addr) .await .with_context(|| format!("can't check if address {:?} is our address", addr))? { @@ -261,7 +262,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { authcode, }) } - } else if context.is_self_addr(addr).await? { + } else if context.is_self_addr(&addr).await? { if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await { Ok(Qr::WithdrawVerifyContact { contact_id, @@ -287,10 +288,11 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result { } } else if let Some(addr) = addr { if let Some(peerstate) = peerstate { - let contact_id = - Contact::add_or_lookup(context, &name, &peerstate.addr, Origin::UnhandledQrScan) + let peerstate_addr = ContactAddress::new(&peerstate.addr)?; + let (contact_id, _) = + Contact::add_or_lookup(context, &name, peerstate_addr, Origin::UnhandledQrScan) .await - .map(|(id, _)| id)?; + .context("add_or_lookup")?; let chat = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request) .await .context("Failed to create (new) chat for contact")?; @@ -373,7 +375,7 @@ struct CreateAccountErrorResponse { #[allow(clippy::indexing_slicing)] async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> { let url_str = &qr[DCACCOUNT_SCHEME.len()..]; - let response = reqwest::Client::new().post(url_str).send().await?; + let response = crate::http::get_client()?.post(url_str).send().await?; let response_status = response.status(); let response_text = response.text().await.with_context(|| { format!( @@ -530,11 +532,11 @@ async fn decode_mailto(context: &Context, qr: &str) -> Result { }; let addr = normalize_address(addr)?; - let name = "".to_string(); + let name = ""; Qr::from_address( context, name, - addr, + &addr, if draft.is_empty() { None } else { Some(draft) }, ) .await @@ -554,8 +556,8 @@ async fn decode_smtp(context: &Context, qr: &str) -> Result { }; let addr = normalize_address(addr)?; - let name = "".to_string(); - Qr::from_address(context, name, addr, None).await + let name = ""; + Qr::from_address(context, name, &addr, None).await } /// Extract address for the matmsg scheme. @@ -579,8 +581,8 @@ async fn decode_matmsg(context: &Context, qr: &str) -> Result { }; let addr = normalize_address(addr)?; - let name = "".to_string(); - Qr::from_address(context, name, addr, None).await + let name = ""; + Qr::from_address(context, name, &addr, None).await } static VCARD_NAME_RE: Lazy = @@ -609,18 +611,19 @@ async fn decode_vcard(context: &Context, qr: &str) -> Result { bail!("Bad e-mail address"); }; - Qr::from_address(context, name, addr, None).await + Qr::from_address(context, &name, &addr, None).await } impl Qr { pub async fn from_address( context: &Context, - name: String, - addr: String, + name: &str, + addr: &str, draft: Option, ) -> Result { + let addr = ContactAddress::new(addr)?; let (contact_id, _) = - Contact::add_or_lookup(context, &name, &addr, Origin::UnhandledQrScan).await?; + Contact::add_or_lookup(context, name, addr, Origin::UnhandledQrScan).await?; Ok(Qr::Addr { contact_id, draft }) } } @@ -638,14 +641,14 @@ fn normalize_address(addr: &str) -> Result { #[cfg(test)] mod tests { - use super::*; + use anyhow::Result; + use super::*; use crate::aheader::EncryptPreference; use crate::chat::{create_group_chat, ProtectionStatus}; use crate::key::DcKey; use crate::securejoin::get_securejoin_qr; use crate::test_utils::{alice_keypair, TestContext}; - use anyhow::Result; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_decode_http() -> Result<()> { diff --git a/src/qr/dclogin_scheme.rs b/src/qr/dclogin_scheme.rs index c7e9e8a76..c7b7613ee 100644 --- a/src/qr/dclogin_scheme.rs +++ b/src/qr/dclogin_scheme.rs @@ -1,13 +1,13 @@ use std::collections::HashMap; -use crate::config::Config; -use crate::context::Context; -use crate::provider::Socket; -use crate::{contact, login_param::CertificateChecks}; use anyhow::{bail, Context as _, Result}; use num_traits::cast::ToPrimitive; use super::{Qr, DCLOGIN_SCHEME}; +use crate::config::Config; +use crate::context::Context; +use crate::provider::Socket; +use crate::{contact, login_param::CertificateChecks}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum LoginOptions { @@ -221,9 +221,10 @@ pub(crate) async fn configure_from_login_qr( #[cfg(test)] mod test { + use anyhow::{self, bail}; + use super::{decode_login, LoginOptions}; use crate::{login_param::CertificateChecks, provider::Socket, qr::Qr}; - use anyhow::{self, bail}; macro_rules! login_options_just_pw { ($pw: expr) => { diff --git a/src/quota.rs b/src/quota.rs index eba52899f..2d71ca13f 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -2,9 +2,10 @@ #![allow(missing_docs)] +use std::collections::BTreeMap; + use anyhow::{anyhow, Context as _, Result}; use async_imap::types::{Quota, QuotaResource}; -use std::collections::BTreeMap; use crate::chat::add_device_msg_with_importance; use crate::config::Config; @@ -134,7 +135,7 @@ impl Context { /// Called in response to `Action::UpdateRecentQuota`. pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result { if let Err(err) = imap.prepare(self).await { - warn!(self, "could not connect: {:?}", err); + warn!(self, "could not connect: {:#}", err); return Ok(Status::RetryNow); } @@ -162,7 +163,7 @@ impl Context { self.set_config(Config::QuotaExceeding, None).await?; } } - Err(err) => warn!(self, "cannot get highest quota usage: {:?}", err), + Err(err) => warn!(self, "cannot get highest quota usage: {:#}", err), } } diff --git a/src/reaction.rs b/src/reaction.rs index 72a096030..fdd193ccd 100644 --- a/src/reaction.rs +++ b/src/reaction.rs @@ -287,10 +287,9 @@ pub async fn get_msg_reactions(context: &Context, msg_id: MsgId) -> Result Result> { info!(context, "Receiving message, seen={}...", seen); - if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" { - info!(context, "receive_imf: incoming message mime-body:"); - println!("{}", String::from_utf8_lossy(imf_raw)); + if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { + info!( + context, + "receive_imf: incoming message mime-body:\n{}", + String::from_utf8_lossy(imf_raw), + ); } let mut mime_parser = match MimeMessage::from_bytes_with_partial(context, imf_raw, is_partial_download).await { Err(err) => { - warn!(context, "receive_imf: can't parse MIME: {}", err); + warn!(context, "receive_imf: can't parse MIME: {:#}", err); let msg_ids; if !rfc724_mid.starts_with(GENERATED_PREFIX) { let row_id = context @@ -170,7 +173,16 @@ pub(crate) async fn receive_imf_inner( // If this is a mailing list email (i.e. list_id_header is some), don't change the displayname because in // a mailing list the sender displayname sometimes does not belong to the sender email address. let (from_id, _from_id_blocked, incoming_origin) = - from_field_to_contact_id(context, &mime_parser.from, prevent_rename).await?; + match from_field_to_contact_id(context, &mime_parser.from, prevent_rename).await? { + Some(contact_id_res) => contact_id_res, + None => { + warn!( + context, + "receive_imf: From field does not contain an acceptable address" + ); + return Ok(None); + } + }; let incoming = from_id != ContactId::SELF; @@ -253,7 +265,7 @@ pub(crate) async fn receive_imf_inner( if from_id == ContactId::SELF { if mime_parser.was_encrypted() { if let Err(err) = context.execute_sync_items(sync_items).await { - warn!(context, "receive_imf cannot execute sync items: {}", err); + warn!(context, "receive_imf cannot execute sync items: {:#}", err); } } else { warn!(context, "sync items are not encrypted."); @@ -268,7 +280,7 @@ pub(crate) async fn receive_imf_inner( .receive_status_update(from_id, insert_msg_id, status_update) .await { - warn!(context, "receive_imf cannot update status: {}", err); + warn!(context, "receive_imf cannot update status: {:#}", err); } } @@ -278,7 +290,7 @@ pub(crate) async fn receive_imf_inner( .update_contacts_timestamp(from_id, Param::AvatarTimestamp, sent_timestamp) .await? { - match contact::set_profile_image( + if let Err(err) = contact::set_profile_image( context, from_id, avatar_action, @@ -286,12 +298,10 @@ pub(crate) async fn receive_imf_inner( ) .await { - Ok(()) => { - context.emit_event(EventType::ChatModified(chat_id)); - } - Err(err) => { - warn!(context, "receive_imf cannot update profile image: {}", err); - } + warn!( + context, + "receive_imf cannot update profile image: {:#}", err + ); }; } } @@ -317,7 +327,7 @@ pub(crate) async fn receive_imf_inner( ) .await { - warn!(context, "cannot update contact status: {}", err); + warn!(context, "cannot update contact status: {:#}", err); } } @@ -346,11 +356,7 @@ pub(crate) async fn receive_imf_inner( } else if !chat_id.is_trash() { let fresh = received_msg.state == MessageState::InFresh; for msg_id in &received_msg.msg_ids { - if incoming && fresh { - context.emit_incoming_msg(chat_id, *msg_id); - } else { - context.emit_msgs_changed(chat_id, *msg_id); - }; + chat_id.emit_msg_event(context, *msg_id, incoming && fresh); } } @@ -366,26 +372,39 @@ pub(crate) async fn receive_imf_inner( /// Also returns whether it is blocked or not and its origin. /// /// * `prevent_rename`: passed through to `add_or_lookup_contacts_by_address_list()` +/// +/// Returns `None` if From field does not contain a valid contact address. pub async fn from_field_to_contact_id( context: &Context, from: &SingleInfo, prevent_rename: bool, -) -> Result<(ContactId, bool, Origin)> { +) -> Result> { let display_name = if prevent_rename { Some("") } else { from.display_name.as_deref() }; + let from_addr = match ContactAddress::new(&from.addr) { + Ok(from_addr) => from_addr, + Err(err) => { + warn!( + context, + "Cannot create a contact for the given From field: {:#}.", err + ); + return Ok(None); + } + }; + let from_id = add_or_lookup_contact_by_addr( context, display_name, - &from.addr, + from_addr, Origin::IncomingUnknownFrom, ) .await?; if from_id == ContactId::SELF { - Ok((ContactId::SELF, false, Origin::OutgoingBcc)) + Ok(Some((ContactId::SELF, false, Origin::OutgoingBcc))) } else { let mut from_id_blocked = false; let mut incoming_origin = Origin::Unknown; @@ -393,7 +412,7 @@ pub async fn from_field_to_contact_id( from_id_blocked = contact.blocked; incoming_origin = contact.origin; } - Ok((from_id, from_id_blocked, incoming_origin)) + Ok(Some((from_id, from_id_blocked, incoming_origin))) } } @@ -495,7 +514,7 @@ async fn add_parts( securejoin_seen = false; } Err(err) => { - warn!(context, "Error in Secure-Join message handling: {}", err); + warn!(context, "Error in Secure-Join message handling: {:#}", err); chat_id = Some(DC_CHAT_ID_TRASH); securejoin_seen = true; } @@ -730,7 +749,7 @@ async fn add_parts( chat_id = None; } Err(err) => { - warn!(context, "Error in Secure-Join watching: {}", err); + warn!(context, "Error in Secure-Join watching: {:#}", err); chat_id = Some(DC_CHAT_ID_TRASH); } } @@ -870,7 +889,7 @@ async fn add_parts( Err(err) => { warn!( context, - "can't parse ephemeral timer \"{}\": {}", value, err + "can't parse ephemeral timer \"{}\": {:#}", value, err ); EphemeralTimer::Disabled } @@ -926,7 +945,7 @@ async fn add_parts( { warn!( context, - "failed to modify timer for chat {}: {}", chat_id, err + "failed to modify timer for chat {}: {:#}", chat_id, err ); } else { info!( @@ -975,7 +994,7 @@ async fn add_parts( if chat.is_protected() || new_status.is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id, to_ids).await { - warn!(context, "verification problem: {}", err); + warn!(context, "verification problem: {:#}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } else { @@ -1216,7 +1235,7 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, replace_msg_id.delete_from_db(context).await?; } - chat_id.unarchive_if_not_muted(context).await?; + chat_id.unarchive_if_not_muted(context, state).await?; info!( context, @@ -1487,7 +1506,7 @@ async fn create_or_lookup_group( let create_protected = if mime_parser.get_header(HeaderDef::ChatVerified).is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id, to_ids).await { - warn!(context, "verification problem: {}", err); + warn!(context, "verification problem: {:#}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } @@ -1713,7 +1732,7 @@ async fn apply_group_changes( if mime_parser.get_header(HeaderDef::ChatVerified).is_some() { if let Err(err) = check_verified_properties(context, mime_parser, from_id, to_ids).await { - warn!(context, "verification problem: {}", err); + warn!(context, "verification problem: {:#}", err); let s = format!("{}. See 'Info' for more details", err); mime_parser.repl_msg_by_error(&s); } @@ -1953,6 +1972,13 @@ async fn apply_mailinglist_changes( } let listid = &chat.grpid; + let list_post = match ContactAddress::new(list_post) { + Ok(list_post) => list_post, + Err(err) => { + warn!(context, "Invalid List-Post: {:#}.", err); + return Ok(()); + } + }; let (contact_id, _) = Contact::add_or_lookup(context, "", list_post, Origin::Hidden).await?; let mut contact = Contact::load_from_db(context, contact_id).await?; @@ -1962,7 +1988,7 @@ async fn apply_mailinglist_changes( } if let Some(old_list_post) = chat.param.get(Param::ListPost) { - if list_post != old_list_post { + if list_post.as_ref() != old_list_post { // Apparently the mailing list is using a different List-Post header in each message. // Make the mailing list read-only because we would't know which message the user wants to reply to. chat.param.remove(Param::ListPost); @@ -2171,10 +2197,10 @@ async fn check_verified_properties( if let Some(fp) = fp { peerstate.set_verified( PeerstateKeyType::GossipKey, - &fp, + fp, PeerstateVerifiedStatus::BidirectVerified, contact.get_addr().to_owned(), - ); + )?; peerstate.save_to_db(&context.sql).await?; is_verified = true; } @@ -2293,8 +2319,13 @@ async fn add_or_lookup_contacts_by_address_list( continue; } let display_name = info.display_name.as_deref(); - contact_ids - .insert(add_or_lookup_contact_by_addr(context, display_name, addr, origin).await?); + if let Ok(addr) = ContactAddress::new(addr) { + let contact_id = + add_or_lookup_contact_by_addr(context, display_name, addr, origin).await?; + contact_ids.insert(contact_id); + } else { + warn!(context, "Contact with address {:?} cannot exist.", addr); + } } Ok(contact_ids.into_iter().collect::>()) @@ -2304,17 +2335,17 @@ async fn add_or_lookup_contacts_by_address_list( async fn add_or_lookup_contact_by_addr( context: &Context, display_name: Option<&str>, - addr: &str, + addr: ContactAddress<'_>, origin: Origin, ) -> Result { - if context.is_self_addr(addr).await? { + if context.is_self_addr(&addr).await? { return Ok(ContactId::SELF); } let display_name_normalized = display_name.map(normalize_name).unwrap_or_default(); - let (row_id, _modified) = + let (contact_id, _modified) = Contact::add_or_lookup(context, &display_name_normalized, addr, origin).await?; - Ok(row_id) + Ok(contact_id) } #[cfg(test)] diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index a92c8aead..c026a4811 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -1,7 +1,6 @@ use tokio::fs; use super::*; - use crate::aheader::EncryptPreference; use crate::chat::get_chat_contacts; use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility}; @@ -425,11 +424,15 @@ async fn test_escaped_recipients() { .await .unwrap(); - let carl_contact_id = - Contact::add_or_lookup(&t, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom) - .await - .unwrap() - .0; + let carl_contact_id = Contact::add_or_lookup( + &t, + "Carl", + ContactAddress::new("carl@host.tld").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap() + .0; receive_imf( &t, @@ -467,11 +470,15 @@ async fn test_cc_to_contact() { .await .unwrap(); - let carl_contact_id = - Contact::add_or_lookup(&t, "garabage", "carl@host.tld", Origin::IncomingUnknownFrom) - .await - .unwrap() - .0; + let carl_contact_id = Contact::add_or_lookup( + &t, + "garabage", + ContactAddress::new("carl@host.tld").unwrap(), + Origin::IncomingUnknownFrom, + ) + .await + .unwrap() + .0; receive_imf( &t, @@ -2054,7 +2061,7 @@ async fn test_duplicate_message() -> Result<()> { let bob_contact_id = Contact::add_or_lookup( &alice, "Bob", - "bob@example.org", + ContactAddress::new("bob@example.org").unwrap(), Origin::IncomingUnknownFrom, ) .await? @@ -2109,9 +2116,14 @@ Second signature"; async fn test_ignore_footer_status_from_mailinglist() -> Result<()> { let t = TestContext::new_alice().await; t.set_config(Config::ShowEmails, Some("2")).await?; - let bob_id = Contact::add_or_lookup(&t, "", "bob@example.net", Origin::IncomingUnknownCc) - .await? - .0; + let bob_id = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("bob@example.net").unwrap(), + Origin::IncomingUnknownCc, + ) + .await? + .0; let bob = Contact::load_from_db(&t, bob_id).await?; assert_eq!(bob.get_status(), ""); assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 0); @@ -2523,13 +2535,8 @@ Second thread."#; // Alice adds Fiona to both ad hoc groups. let fiona = TestContext::new_fiona().await; - let (alice_fiona_contact_id, _) = Contact::add_or_lookup( - &alice, - "Fiona", - "fiona@example.net", - Origin::IncomingUnknownTo, - ) - .await?; + let alice_fiona_contact = alice.add_or_lookup_contact(&fiona).await; + let alice_fiona_contact_id = alice_fiona_contact.id; chat::add_contact_to_chat(&alice, alice_first_msg.chat_id, alice_fiona_contact_id).await?; let alice_first_invite = alice.pop_sent_msg().await; diff --git a/src/scheduler.rs b/src/scheduler.rs index 92e569010..356c40e07 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -4,6 +4,7 @@ use futures::try_join; use futures_lite::FutureExt; use tokio::task; +use self::connectivity::ConnectivityStore; use crate::config::Config; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; @@ -17,8 +18,6 @@ use crate::sql; use crate::tools::time; use crate::tools::{duration_to_str, maybe_add_time_based_warnings}; -use self::connectivity::ConnectivityStore; - pub(crate) mod connectivity; /// Job and connection scheduler. diff --git a/src/scheduler/connectivity.rs b/src/scheduler/connectivity.rs index 630053864..f6c6c4678 100644 --- a/src/scheduler/connectivity.rs +++ b/src/scheduler/connectivity.rs @@ -3,6 +3,8 @@ use core::fmt; use std::{ops::Deref, sync::Arc}; +use anyhow::{anyhow, Result}; +use humansize::{format_size, BINARY}; use tokio::sync::{Mutex, RwLockReadGuard}; use crate::events::EventType; @@ -13,8 +15,6 @@ use crate::quota::{ use crate::tools::time; use crate::{config::Config, scheduler::Scheduler, stock_str, tools}; use crate::{context::Context, log::LogExt}; -use anyhow::{anyhow, Result}; -use humansize::{format_size, BINARY}; #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)] pub enum Connectivity { diff --git a/src/securejoin.rs b/src/securejoin.rs index 77d743c17..dfb68b525 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -30,10 +30,11 @@ mod bob; mod bobstate; mod qrinvite; -use crate::token::Namespace; use bobstate::BobState; use qrinvite::QrInvite; +use crate::token::Namespace; + pub const NON_ALPHANUMERIC_WITHOUT_DOT: &AsciiSet = &NON_ALPHANUMERIC.remove(b'.'); macro_rules! inviter_progress { @@ -415,7 +416,7 @@ pub(crate) async fn handle_securejoin_handshake( .await? .get_addr() .to_owned(); - if mark_peer_as_verified(context, &fingerprint, contact_addr) + if mark_peer_as_verified(context, fingerprint.clone(), contact_addr) .await .is_err() { @@ -455,6 +456,8 @@ pub(crate) async fn handle_securejoin_handshake( } None => bail!("Chat {} not found", &field_grpid), } + inviter_progress!(context, contact_id, 800); + inviter_progress!(context, contact_id, 1000); } else { // Alice -> Bob secure_connection_established( @@ -503,9 +506,6 @@ pub(crate) async fn handle_securejoin_handshake( return Ok(HandshakeMessage::Ignore); } if join_vg { - // Responsible for showing "$Bob securely joined $group" message - inviter_progress!(context, contact_id, 800); - inviter_progress!(context, contact_id, 1000); let field_grpid = mime_message .get_header(HeaderDef::SecureJoinGroup) .map(|s| s.as_str()) @@ -579,40 +579,103 @@ pub(crate) async fn observe_securejoin_on_other_device( .await?; return Ok(HandshakeMessage::Ignore); } - let fingerprint: Fingerprint = - match mime_message.get_header(HeaderDef::SecureJoinFingerprint) { - Some(fp) => fp.parse()?, + let addr = Contact::load_from_db(context, contact_id) + .await? + .get_addr() + .to_string(); + if mime_message.gossiped_addr.contains(&addr) { + let mut peerstate = match Peerstate::from_addr(context, &addr).await? { + Some(p) => p, None => { could_not_establish_secure_connection( - context, - contact_id, - info_chat_id(context, contact_id).await?, - "Fingerprint not provided, please update Delta Chat on all your devices.", - ) - .await?; + context, + contact_id, + info_chat_id(context, contact_id).await?, + &format!("No peerstate in db for '{}' at step {}", &addr, step), + ) + .await?; return Ok(HandshakeMessage::Ignore); } }; - if mark_peer_as_verified( - context, - &fingerprint, - Contact::load_from_db(context, contact_id) - .await? - .get_addr() - .to_owned(), - ) - .await - .is_err() + let fingerprint = match peerstate.gossip_key_fingerprint.clone() { + Some(fp) => fp, + None => { + could_not_establish_secure_connection( + context, + contact_id, + info_chat_id(context, contact_id).await?, + &format!( + "No gossip key fingerprint in db for '{}' at step {}", + &addr, step, + ), + ) + .await?; + return Ok(HandshakeMessage::Ignore); + } + }; + if let Err(err) = peerstate.set_verified( + PeerstateKeyType::GossipKey, + fingerprint, + PeerstateVerifiedStatus::BidirectVerified, + addr, + ) { + could_not_establish_secure_connection( + context, + contact_id, + info_chat_id(context, contact_id).await?, + &format!("Could not mark peer as verified at step {}: {}", step, err), + ) + .await?; + return Ok(HandshakeMessage::Ignore); + } + peerstate.prefer_encrypt = EncryptPreference::Mutual; + peerstate.save_to_db(&context.sql).await.unwrap_or_default(); + } else if let Some(fingerprint) = + mime_message.get_header(HeaderDef::SecureJoinFingerprint) { + // FIXME: Old versions of DC send this header instead of gossips. Remove this + // eventually. + let fingerprint = fingerprint.parse()?; + if mark_peer_as_verified( + context, + fingerprint, + Contact::load_from_db(context, contact_id) + .await? + .get_addr() + .to_owned(), + ) + .await + .is_err() + { + could_not_establish_secure_connection( + context, + contact_id, + info_chat_id(context, contact_id).await?, + format!("Fingerprint mismatch on observing {}.", step).as_ref(), + ) + .await?; + return Ok(HandshakeMessage::Ignore); + } + } else { could_not_establish_secure_connection( context, contact_id, info_chat_id(context, contact_id).await?, - format!("Fingerprint mismatch on observing {}.", step).as_ref(), + &format!( + "No gossip header for '{}' at step {}, please update Delta Chat on all \ + your devices.", + &addr, step, + ), ) .await?; return Ok(HandshakeMessage::Ignore); } + if step.as_str() == "vg-member-added" { + inviter_progress!(context, contact_id, 800); + } + if step.as_str() == "vg-member-added" || step.as_str() == "vc-contact-confirm" { + inviter_progress!(context, contact_id, 1000); + } Ok(if step.as_str() == "vg-member-added" { HandshakeMessage::Propagate } else { @@ -653,25 +716,25 @@ async fn could_not_establish_secure_connection( async fn mark_peer_as_verified( context: &Context, - fingerprint: &Fingerprint, + fingerprint: Fingerprint, verifier: String, ) -> Result<(), Error> { - if let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, fingerprint).await? { - if peerstate.set_verified( + if let Some(ref mut peerstate) = Peerstate::from_fingerprint(context, &fingerprint).await? { + if let Err(err) = peerstate.set_verified( PeerstateKeyType::PublicKey, fingerprint, PeerstateVerifiedStatus::BidirectVerified, verifier, ) { - peerstate.prefer_encrypt = EncryptPreference::Mutual; - peerstate.save_to_db(&context.sql).await.unwrap_or_default(); - return Ok(()); + error!(context, "Could not mark peer as verified: {}", err); + return Err(err); } + peerstate.prefer_encrypt = EncryptPreference::Mutual; + peerstate.save_to_db(&context.sql).await.unwrap_or_default(); + Ok(()) + } else { + bail!("no peerstate in db for fingerprint {}", fingerprint.hex()); } - bail!( - "could not mark peer as verified for fingerprint {}", - fingerprint.hex() - ); } /* ****************************************************************************** @@ -705,11 +768,12 @@ fn encrypted_and_signed( #[cfg(test)] mod tests { use super::*; - use crate::chat; use crate::chat::ProtectionStatus; use crate::chatlist::Chatlist; use crate::constants::{Chattype, DC_GCM_ADDDAYMARKER}; + use crate::contact::ContactAddress; + use crate::contact::VerifiedStatus; use crate::peerstate::Peerstate; use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, TestContextManager}; @@ -1003,7 +1067,7 @@ mod tests { let (contact_bob_id, _modified) = Contact::add_or_lookup( &alice.ctx, "Bob", - "bob@example.net", + ContactAddress::new("bob@example.net")?, Origin::ManuallyCreated, ) .await?; diff --git a/src/securejoin/bob.rs b/src/securejoin/bob.rs index 9162220ca..674fa94f7 100644 --- a/src/securejoin/bob.rs +++ b/src/securejoin/bob.rs @@ -5,6 +5,9 @@ use anyhow::{Context as _, Result}; +use super::bobstate::{BobHandshakeStage, BobState}; +use super::qrinvite::QrInvite; +use super::HandshakeMessage; use crate::chat::{is_contact_in_chat, ChatId, ProtectionStatus}; use crate::constants::{Blocked, Chattype}; use crate::contact::Contact; @@ -14,10 +17,6 @@ use crate::mimeparser::MimeMessage; use crate::tools::time; use crate::{chat, stock_str}; -use super::bobstate::{BobHandshakeStage, BobState}; -use super::qrinvite::QrInvite; -use super::HandshakeMessage; - /// Starts the securejoin protocol with the QR `invite`. /// /// This will try to start the securejoin protocol for the given QR `invite`. If it diff --git a/src/securejoin/bobstate.rs b/src/securejoin/bobstate.rs index e1145a006..ead6b1c4f 100644 --- a/src/securejoin/bobstate.rs +++ b/src/securejoin/bobstate.rs @@ -10,6 +10,8 @@ use anyhow::{Error, Result}; use rusqlite::Connection; +use super::qrinvite::QrInvite; +use super::{encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified}; use crate::chat::{self, ChatId}; use crate::contact::{Contact, Origin}; use crate::context::Context; @@ -21,9 +23,6 @@ use crate::mimeparser::{MimeMessage, SystemMessage}; use crate::param::Param; use crate::sql::Sql; -use super::qrinvite::QrInvite; -use super::{encrypted_and_signed, fingerprint_equals_sender, mark_peer_as_verified}; - /// The stage of the [`BobState`] securejoin handshake protocol state machine. /// /// This does not concern itself with user interactions, only represents what happened to @@ -368,7 +367,7 @@ impl BobState { } mark_peer_as_verified( context, - self.invite.fingerprint(), + self.invite.fingerprint().clone(), mime_message.from.addr.to_string(), ) .await?; diff --git a/src/simplify.rs b/src/simplify.rs index 1c30bc2ef..7709dbf37 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -285,9 +285,10 @@ fn is_plain_quote(buf: &str) -> bool { #[cfg(test)] mod tests { - use super::*; use proptest::prelude::*; + use super::*; + proptest! { #[test] // proptest does not support [[:graphical:][:space:]] regex. diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 829e114b0..6a4c48f9e 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -1,16 +1,21 @@ //! # SMTP message sending -use super::Smtp; -use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport}; - -use crate::config::Config; -use crate::constants::DEFAULT_MAX_SMTP_RCPT_TO; -use crate::context::Context; -use crate::events::EventType; use std::time::Duration; +use async_smtp::{EmailAddress, Envelope, SendableEmail, Transport}; + +use super::Smtp; +use crate::config::Config; +use crate::context::Context; +use crate::events::EventType; + pub type Result = std::result::Result; +// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks. +// this does not affect MIME'e `To:` header. +// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db. +pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("Envelope error: {}", _0)] diff --git a/src/socks.rs b/src/socks.rs index e7ea20730..58d7aa1db 100644 --- a/src/socks.rs +++ b/src/socks.rs @@ -4,15 +4,17 @@ use std::fmt; use std::pin::Pin; use std::time::Duration; -use crate::net::connect_tcp; use anyhow::Result; pub use async_smtp::ServerAddress; -use tokio::net::{self, TcpStream}; +use fast_socks5::client::{Config, Socks5Stream}; +use fast_socks5::util::target_addr::ToTargetAddr; +use fast_socks5::AuthenticationMethod; +use fast_socks5::Socks5Command; +use tokio::net::TcpStream; use tokio_io_timeout::TimeoutStream; use crate::context::Context; -use fast_socks5::client::{Config, Socks5Stream}; -use fast_socks5::AuthenticationMethod; +use crate::net::connect_tcp; #[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct Socks5Config { @@ -54,12 +56,18 @@ impl Socks5Config { } } + /// If `load_dns_cache` is true, loads cached DNS resolution results. + /// Use this only if the connection is going to be protected with TLS checks. pub async fn connect( &self, - target_addr: impl net::ToSocketAddrs, + context: &Context, + target_host: &str, + target_port: u16, timeout_val: Duration, + load_dns_cache: bool, ) -> Result>>>> { - let tcp_stream = connect_tcp(target_addr, timeout_val).await?; + let tcp_stream = + connect_tcp(context, &self.host, self.port, timeout_val, load_dns_cache).await?; let authentication_method = if let Some((username, password)) = self.user_password.as_ref() { @@ -70,8 +78,12 @@ impl Socks5Config { } else { None }; - let socks_stream = + let mut socks_stream = Socks5Stream::use_stream(tcp_stream, authentication_method, Config::default()).await?; + let target_addr = (target_host, target_port).to_target_addr()?; + socks_stream + .request(Socks5Command::TCPConnect, target_addr) + .await?; Ok(socks_stream) } diff --git a/src/sql.rs b/src/sql.rs index f24008390..778a79107 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -626,26 +626,26 @@ pub async fn housekeeping(context: &Context) -> Result<()> { if let Err(err) = remove_unused_files(context).await { warn!( context, - "Housekeeping: cannot remove unusued files: {}", err + "Housekeeping: cannot remove unusued files: {:#}", err ); } if let Err(err) = start_ephemeral_timers(context).await { warn!( context, - "Housekeeping: cannot start ephemeral timers: {}", err + "Housekeeping: cannot start ephemeral timers: {:#}", err ); } if let Err(err) = prune_tombstones(&context.sql).await { warn!( context, - "Housekeeping: Cannot prune message tombstones: {}", err + "Housekeeping: Cannot prune message tombstones: {:#}", err ); } if let Err(err) = deduplicate_peerstates(&context.sql).await { - warn!(context, "Failed to deduplicate peerstates: {}", err) + warn!(context, "Failed to deduplicate peerstates: {:#}", err) } context.schedule_quota_update().await?; @@ -874,11 +874,10 @@ pub fn repeat_vars(count: usize) -> String { mod tests { use async_channel as channel; + use super::*; use crate::config::Config; use crate::{test_utils::TestContext, EventType}; - use super::*; - #[test] fn test_maybe_add_file() { let mut files = Default::default(); diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index fde27051c..4611e9f7f 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -671,6 +671,18 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid); ) .await?; } + if dbversion < 97 { + sql.execute_migration( + "CREATE TABLE dns_cache ( + hostname TEXT NOT NULL, + address TEXT NOT NULL, -- IPv4 or IPv6 address + timestamp INTEGER NOT NULL, + UNIQUE (hostname, address) + )", + 97, + ) + .await?; + } let new_version = sql .get_raw_config_int(VERSION_CFG) diff --git a/src/stock_str.rs b/src/stock_str.rs index 2e93962d6..aa7ac2903 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::sync::Arc; use anyhow::{bail, Result}; +use humansize::{format_size, BINARY}; use strum::EnumProperty as EnumPropertyTrait; use strum_macros::EnumProperty; use tokio::sync::RwLock; @@ -19,7 +20,6 @@ use crate::context::Context; use crate::message::{Message, Viewtype}; use crate::param::Param; use crate::tools::timestamp_to_str; -use humansize::{format_size, BINARY}; #[derive(Debug, Clone)] pub struct StockStrings { @@ -1308,13 +1308,12 @@ impl Accounts { mod tests { use num_traits::ToPrimitive; + use super::*; use crate::chat::delete_and_reset_all_device_msgs; use crate::chat::Chat; use crate::chatlist::Chatlist; use crate::test_utils::TestContext; - use super::*; - #[test] fn test_enum_mapping() { assert_eq!(StockMessage::NoMessages.to_usize().unwrap(), 1); diff --git a/src/summary.rs b/src/summary.rs index 5b4d3f0bc..5fc91c61e 100644 --- a/src/summary.rs +++ b/src/summary.rs @@ -1,5 +1,8 @@ //! # Message summary for chatlist. +use std::borrow::Cow; +use std::fmt; + use crate::chat::Chat; use crate::constants::Chattype; use crate::contact::{Contact, ContactId}; @@ -9,8 +12,6 @@ use crate::mimeparser::SystemMessage; use crate::param::Param; use crate::stock_str; use crate::tools::truncate; -use std::borrow::Cow; -use std::fmt; /// Prefix displayed before message and separated by ":" in the chatlist. #[derive(Debug)] diff --git a/src/sync.rs b/src/sync.rs index 041980ab0..b3835e499 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,10 @@ //! # Synchronize items between devices. +use anyhow::Result; +use lettre_email::mime::{self}; +use lettre_email::PartBuilder; +use serde::{Deserialize, Serialize}; + use crate::chat::{Chat, ChatId}; use crate::config::Config; use crate::constants::Blocked; @@ -12,10 +17,6 @@ use crate::sync::SyncData::{AddQrToken, DeleteQrToken}; use crate::token::Namespace; use crate::tools::time; use crate::{chat, stock_str, token}; -use anyhow::Result; -use lettre_email::mime::{self}; -use lettre_email::PartBuilder; -use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize)] pub(crate) struct QrTokenData { @@ -260,12 +261,13 @@ impl Context { #[cfg(test)] mod tests { + use anyhow::bail; + use super::*; use crate::chat::Chat; use crate::chatlist::Chatlist; use crate::test_utils::TestContext; use crate::token::Namespace; - use anyhow::bail; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_is_sync_sending_enabled() -> Result<()> { diff --git a/src/test_utils.rs b/src/test_utils.rs index ecf219d5c..f5bce1cf3 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -14,6 +14,7 @@ use chat::ChatItem; use once_cell::sync::Lazy; use rand::Rng; use tempfile::{tempdir, TempDir}; +use tokio::runtime::Handle; use tokio::sync::RwLock; use tokio::task; @@ -21,8 +22,8 @@ use crate::chat::{self, Chat, ChatId}; use crate::chatlist::Chatlist; use crate::config::Config; use crate::constants::Chattype; -use crate::constants::{DC_GCM_ADDDAYMARKER, DC_MSG_ID_DAYMARKER}; -use crate::contact::{Contact, ContactId, Modifier, Origin}; +use crate::constants::{DC_GCL_NO_SPECIALS, DC_GCM_ADDDAYMARKER, DC_MSG_ID_DAYMARKER}; +use crate::contact::{Contact, ContactAddress, ContactId, Modifier, Origin}; use crate::context::Context; use crate::events::{Event, EventType, Events}; use crate::key::{self, DcKey, KeyPair, KeyPairUse}; @@ -263,7 +264,6 @@ impl TestContext { Self::builder().configure_fiona().build().await } - #[allow(dead_code)] /// Print current chat state. pub async fn print_chats(&self) { println!("\n========== Chats of {}: ==========", self.name()); @@ -502,7 +502,7 @@ impl TestContext { /// Gets the most recent message over all chats. pub async fn get_last_msg(&self) -> Message { - let chats = Chatlist::try_load(&self.ctx, 0, None, None) + let chats = Chatlist::try_load(&self.ctx, DC_GCL_NO_SPECIALS, None, None) .await .expect("failed to load chatlist"); // 0 is correct in the next line (as opposed to `chats.len() - 1`, which would be the last element): @@ -523,13 +523,14 @@ impl TestContext { .await .unwrap_or_default() .unwrap_or_default(); - let addr = other.ctx.get_primary_self_addr().await.unwrap(); + let primary_self_addr = other.ctx.get_primary_self_addr().await.unwrap(); + let addr = ContactAddress::new(&primary_self_addr).unwrap(); // MailinglistAddress is the lowest allowed origin, we'd prefer to not modify the // origin when creating this contact. let (contact_id, modified) = - Contact::add_or_lookup(self, &name, &addr, Origin::MailinglistAddress) + Contact::add_or_lookup(self, &name, addr, Origin::MailinglistAddress) .await - .unwrap(); + .expect("add_or_lookup"); match modified { Modifier::None => (), Modifier::Modified => warn!(&self.ctx, "Contact {} modified by TestContext", &addr), @@ -702,6 +703,19 @@ impl Deref for TestContext { } } +impl Drop for TestContext { + fn drop(&mut self) { + task::block_in_place(move || { + if let Ok(handle) = Handle::try_current() { + // Print the chats if runtime still exists. + handle.block_on(async move { + self.print_chats().await; + }); + } + }); + } +} + pub enum LogEvent { /// Logged event. Event(Event), @@ -1079,4 +1093,12 @@ mod tests { bob.ctx.emit_event(EventType::Info("there".into())); // panic!("Both fail"); } + + /// Checks that dropping the `TestContext` after the runtime does not panic, + /// e.g. that `TestContext::drop` does not assume the runtime still exists. + #[test] + fn test_new_test_context() { + let runtime = tokio::runtime::Runtime::new().expect("unable to create tokio runtime"); + runtime.block_on(TestContext::new()); + } } diff --git a/src/tools.rs b/src/tools.rs index b766dc73e..ca935cf1e 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -9,7 +9,6 @@ use std::fmt; use std::io::Cursor; use std::path::{Path, PathBuf}; use std::str::from_utf8; - use std::time::{Duration, SystemTime}; use anyhow::{bail, Error, Result}; @@ -277,6 +276,12 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time /// - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header /// - the group-id should be a string with the characters [a-zA-Z0-9\-_] pub(crate) fn create_id() -> String { + const URL_SAFE_ENGINE: base64::engine::fast_portable::FastPortable = + base64::engine::fast_portable::FastPortable::from( + &base64::alphabet::URL_SAFE, + base64::engine::fast_portable::NO_PAD, + ); + // ThreadRng implements CryptoRng trait and is supposed to be cryptographically secure. let mut rng = thread_rng(); @@ -285,7 +290,7 @@ pub(crate) fn create_id() -> String { rng.fill(&mut arr[..]); // Take 11 base64 characters containing 66 random bits. - base64::encode_config(arr, base64::URL_SAFE) + base64::encode_engine(arr, &URL_SAFE_ENGINE) .chars() .take(11) .collect() @@ -358,12 +363,10 @@ pub(crate) fn get_abs_path(context: &Context, path: impl AsRef) -> PathBuf } } -pub(crate) async fn get_filebytes(context: &Context, path: impl AsRef) -> u64 { +pub(crate) async fn get_filebytes(context: &Context, path: impl AsRef) -> Result { let path_abs = get_abs_path(context, &path); - match fs::metadata(&path_abs).await { - Ok(meta) => meta.len(), - Err(_err) => 0, - } + let meta = fs::metadata(&path_abs).await?; + Ok(meta.len()) } pub(crate) async fn delete_file(context: &Context, path: impl AsRef) -> bool { @@ -699,7 +702,6 @@ mod tests { #![allow(clippy::indexing_slicing)] use super::*; - use crate::{ config::Config, message::get_msg_info, receive_imf::receive_imf, test_utils::TestContext, }; @@ -1004,11 +1006,12 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; assert_eq!(EmailAddress::new("@d.tt").is_ok(), false); } - use crate::chatlist::Chatlist; - use crate::{chat, test_utils}; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use proptest::prelude::*; + use crate::chatlist::Chatlist; + use crate::{chat, test_utils}; + proptest! { #[test] fn test_truncate( @@ -1049,7 +1052,7 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; .is_ok()); assert!(file_exist!(context, "$BLOBDIR/foobar")); assert!(!file_exist!(context, "$BLOBDIR/foobarx")); - assert_eq!(get_filebytes(context, "$BLOBDIR/foobar").await, 7); + assert_eq!(get_filebytes(context, "$BLOBDIR/foobar").await.unwrap(), 7); let abs_path = context .get_blobdir() diff --git a/src/update_helper.rs b/src/update_helper.rs index bfe46d500..9136b23df 100644 --- a/src/update_helper.rs +++ b/src/update_helper.rs @@ -1,10 +1,11 @@ //! # Functions to update timestamps. +use anyhow::Result; + use crate::chat::{Chat, ChatId}; use crate::contact::{Contact, ContactId}; use crate::context::Context; use crate::param::{Param, Params}; -use anyhow::Result; impl Context { /// Updates a contact's timestamp, if reasonable. diff --git a/src/webxdc.rs b/src/webxdc.rs index 7d4bb4d03..992597736 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -1,7 +1,5 @@ //! # Handle webxdc messages. -#![allow(missing_docs)] - use std::convert::TryFrom; use std::path::Path; @@ -31,6 +29,7 @@ use crate::{chat, EventType}; /// In the future, that may be useful to avoid new Webxdc being loaded on old Delta Chats. const WEBXDC_API_VERSION: u32 = 1; +/// Suffix used to recognize webxdc files. pub const WEBXDC_SUFFIX: &str = "xdc"; const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png"; @@ -55,20 +54,44 @@ const WEBXDC_RECEIVING_LIMIT: u64 = 4194304; #[derive(Debug, Deserialize)] #[non_exhaustive] struct WebxdcManifest { + /// Webxdc name, used on icons or page titles. name: Option, + + /// Minimum API version required to run this webxdc. min_api: Option, + + /// Optional URL of webxdc source code. source_code_url: Option, + + /// If the webxdc requests network access. request_internet_access: Option, } /// Parsed information from WebxdcManifest and fallbacks. #[derive(Debug, Serialize)] pub struct WebxdcInfo { + /// The name of the app. + /// Defaults to filename if not set in the manifest. pub name: String, + + /// Filename of the app icon. pub icon: String, + + /// If the webxdc represents a document and allows to edit it, + /// this is the document name. + /// Otherwise an empty string. pub document: String, + + /// Short description of the webxdc state. + /// For example, "7 votes". pub summary: String, + + /// URL of webxdc source code or an empty string. pub source_code_url: String, + + /// If the webxdc is allowed to access the network. + /// It should request access, be encrypted + /// and sent to self for this. pub internet_access: bool, } @@ -747,6 +770,7 @@ impl Message { #[cfg(test)] mod tests { + use super::*; use crate::chat::{ add_contact_to_chat, create_broadcast_list, create_group_chat, forward_msgs, remove_contact_from_chat, resend_msgs, send_msg, send_text_msg, ChatId, ProtectionStatus, @@ -758,8 +782,6 @@ mod tests { use crate::receive_imf::{receive_imf, receive_imf_inner}; use crate::test_utils::TestContext; - use super::*; - #[allow(clippy::assertions_on_constants)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_webxdc_file_limits() -> Result<()> {