From ee81d61988b15a91b185a15096cd3fa1ff655987 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 01:31:09 +0000 Subject: [PATCH 01/34] Fix ruff warnings --- .../src/deltachat_rpc_client/chat.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index 7dac7d2f4..6a3e70971 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -182,19 +182,23 @@ class Chat: """Add contacts to this group.""" for cnt in contact: if isinstance(cnt, str): - cnt = (await self.account.create_contact(cnt)).id + contact_id = (await self.account.create_contact(cnt)).id elif not isinstance(cnt, int): - cnt = cnt.id - await self._rpc.add_contact_to_chat(self.account.id, self.id, cnt) + contact_id = cnt.id + else: + contact_id = cnt + await self._rpc.add_contact_to_chat(self.account.id, self.id, contact_id) async def remove_contact(self, *contact: Union[int, str, Contact]) -> None: """Remove members from this group.""" for cnt in contact: if isinstance(cnt, str): - cnt = (await self.account.create_contact(cnt)).id + contact_id = (await self.account.create_contact(cnt)).id elif not isinstance(cnt, int): - cnt = cnt.id - await self._rpc.remove_contact_from_chat(self.account.id, self.id, cnt) + contact_id = cnt.id + else: + contact_id = cnt + await self._rpc.remove_contact_from_chat(self.account.id, self.id, contact_id) async def get_contacts(self) -> List[Contact]: """Get the contacts belonging to this chat. @@ -230,9 +234,9 @@ class Chat: locations = [] contacts: Dict[int, Contact] = {} for loc in result: - loc = AttrDict(loc) - loc["chat"] = self - loc["contact"] = contacts.setdefault(loc.contact_id, Contact(self.account, loc.contact_id)) - loc["message"] = Message(self.account, loc.msg_id) - locations.append(loc) + location = AttrDict(loc) + location["chat"] = self + location["contact"] = contacts.setdefault(location.contact_id, Contact(self.account, location.contact_id)) + location["message"] = Message(self.account, location.msg_id) + locations.append(location) return locations From ade3d0d4eb9fc890604a8c7a01406e5eb92398a1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 12:55:43 +0000 Subject: [PATCH 02/34] Add more documentation comments --- src/accounts.rs | 3 +++ src/chat.rs | 3 +++ src/configure.rs | 4 ++++ src/imap.rs | 5 +++++ src/imap/select_folder.rs | 2 ++ src/qr.rs | 4 ++++ src/tools.rs | 2 ++ 7 files changed, 23 insertions(+) diff --git a/src/accounts.rs b/src/accounts.rs index 8ec3a34c8..9f013bfe6 100644 --- a/src/accounts.rs +++ b/src/accounts.rs @@ -476,10 +476,13 @@ impl Config { struct AccountConfig { /// Unique id. pub id: u32, + /// Root directory for all data for this account. /// /// The path is relative to the account manager directory. pub dir: std::path::PathBuf, + + /// Universally unique account identifier. pub uuid: Uuid, } diff --git a/src/chat.rs b/src/chat.rs index 522f55aa7..26e7a7db2 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1881,7 +1881,10 @@ pub(crate) async fn update_special_chat_names(context: &Context) -> Result<()> { /// [`Deref`]: std::ops::Deref #[derive(Debug)] pub(crate) struct ChatIdBlocked { + /// Chat ID. pub id: ChatId, + + /// Whether the chat is blocked, unblocked or a contact request. pub blocked: Blocked, } diff --git a/src/configure.rs b/src/configure.rs index 565169470..48b91a3f7 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -646,10 +646,14 @@ async fn try_smtp_one_param( } } +/// Failure to connect and login with email client configuration. #[derive(Debug, thiserror::Error)] #[error("Trying {config}…\nError: {msg}")] pub struct ConfigurationError { + /// Tried configuration description. config: String, + + /// Error message. msg: String, } diff --git a/src/imap.rs b/src/imap.rs index d7fcf1397..9889df69b 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -116,6 +116,8 @@ impl async_imap::Authenticator for OAuth2 { #[derive(Debug, Display, PartialEq, Eq, Clone, Copy)] pub enum FolderMeaning { Unknown, + + /// Spam folder. Spam, Inbox, Mvbox, @@ -149,8 +151,11 @@ impl FolderMeaning { #[derive(Debug)] struct ImapConfig { + /// Email address. pub addr: String, pub lp: ServerLoginParam, + + /// SOCKS 5 configuration. pub socks5_config: Option, pub strict_tls: bool, } diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index f209edff4..2d6dfa9a6 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -1,3 +1,5 @@ +//! # IMAP folder selection module. + use anyhow::Context as _; use super::session::Session as ImapSession; diff --git a/src/qr.rs b/src/qr.rs index 982777b4a..b9ffbbad5 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -473,11 +473,15 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result { #[derive(Debug, Deserialize)] struct CreateAccountSuccessResponse { + /// Email address. email: String, + + /// Password. password: String, } #[derive(Debug, Deserialize)] struct CreateAccountErrorResponse { + /// Reason for the failure to create account returned by the server. reason: String, } diff --git a/src/tools.rs b/src/tools.rs index 8703a124f..7e8d7e922 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -592,6 +592,8 @@ pub(crate) fn improve_single_line_input(input: &str) -> String { } pub(crate) trait IsNoneOrEmpty { + /// Returns true if an Option does not contain a string + /// or contains an empty string. fn is_none_or_empty(&self) -> bool; } impl IsNoneOrEmpty for Option From 2cc85e663775c97c9ab42c4c9e2ae6121f1c65d6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 14:48:25 +0000 Subject: [PATCH 03/34] Hide some constants from public crate interface --- src/constants.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 4c19abba6..9af48cc0b 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -190,11 +190,11 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4; pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL; /// How many existing messages shall be fetched after configuration. -pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100; +pub(crate) const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100; // max. width/height of an avatar -pub const BALANCED_AVATAR_SIZE: u32 = 256; -pub const WORSE_AVATAR_SIZE: u32 = 128; +pub(crate) const BALANCED_AVATAR_SIZE: u32 = 256; +pub(crate) const WORSE_AVATAR_SIZE: u32 = 128; // max. width/height of images pub const BALANCED_IMAGE_SIZE: u32 = 1280; From 0d7a3fe5528b5bc75110fb30397b412a8d63e5cb Mon Sep 17 00:00:00 2001 From: gerryfrancis <51095533+gerryfrancis@users.noreply.github.com> Date: Thu, 23 Feb 2023 19:05:20 +0100 Subject: [PATCH 04/34] Fix typo in CHANGELOG.md (#4086) Fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce2da427e..42b69dbc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -437,7 +437,7 @@ - Auto accept contact requests if `Config::Bot` is set for a client #3567 - Don't prepend the subject to chat messages in mailinglists - fix `set_core_version.py` script to also update version in `deltachat-jsonrpc/typescript/package.json` #3585 -- Reject webxcd-updates from contacts who are not group members #3568 +- Reject webxdc-updates from contacts who are not group members #3568 ## 1.93.0 From 76514178b6ea781fc0fdd68f57c6ac6ff0d0ab64 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 18:45:55 +0000 Subject: [PATCH 05/34] ci: do not run CI multiple times on the same branch Especially for PRs, if the branch is updated multiple times, only the last commit should be tested. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 449161d7f..fe1c4f8cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,11 @@ name: Rust CI +# Cancel previously started workflow runs +# when the branch is updated. +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + on: pull_request: push: From 059c673d00f547b969f442b57a587f88ea77dcd7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 18:50:44 +0000 Subject: [PATCH 06/34] ci: add workflow to the group Otherwise it cancels other workflows. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe1c4f8cc..392da23d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: Rust CI # Cancel previously started workflow runs # when the branch is updated. concurrency: - group: ${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: From b5187661ee46b8fe15953806e06e0e2e442826e4 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 01:13:40 +0000 Subject: [PATCH 07/34] ci: cancel running node.js tests when the branch is updated Do not waste CI time running the tests for previous commit. --- .github/workflows/node-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 7836f9b33..165880c68 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,4 +1,11 @@ name: "node.js tests" + +# Cancel previously started workflow runs +# when the branch is updated. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: push: From bc9e347a80646bdf630f8d57d602492a9331dc13 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 01:14:00 +0000 Subject: [PATCH 08/34] ci: there are no staging and trying branches --- .github/workflows/node-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 165880c68..62ab1d0fd 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -11,8 +11,6 @@ on: push: branches: - master - - staging - - trying jobs: tests: From e9668b3cfa1938fa09117c9d5756556a4ad44642 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 10:11:13 +0000 Subject: [PATCH 09/34] Cache INSERT statement in receive_imf It reduces iteration time by ~8% in message reception benchmarks ran by `cargo criterion --bench receive_emails`. --- src/receive_imf.rs | 91 +++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 2540d7442..b86fbed7c 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1149,7 +1149,10 @@ async fn add_parts( // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && msg.is_empty()); - let row_id = context.sql.insert( + let row_id = context + .sql + .call(|conn| { + let mut stmt = conn.prepare_cached( r#" INSERT INTO msgs ( @@ -1179,47 +1182,51 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to, mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer, ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info -"#, - paramsv![ - replace_msg_id, - rfc724_mid, - if trash { DC_CHAT_ID_TRASH } else { chat_id }, - if trash { ContactId::UNDEFINED } else { from_id }, - if trash { ContactId::UNDEFINED } else { to_id }, - sort_timestamp, - sent_timestamp, - rcvd_timestamp, - typ, - state, - is_dc_message, - if trash { "" } else { msg }, - if trash { "" } else { &subject }, - // txt_raw might contain invalid utf8 - if trash { "" } else { &txt_raw }, - if trash { - "".to_string() - } else { - param.to_string() - }, - part.bytes as isize, - if (save_mime_headers || mime_modified) && !trash { - mime_headers.clone() - } else { - Vec::new() - }, - mime_in_reply_to, - mime_references, - mime_modified, - part.error.as_deref().unwrap_or_default(), - ephemeral_timer, - ephemeral_timestamp, - if is_partial_download.is_some() { - DownloadState::Available - } else { - DownloadState::Done - }, - mime_parser.hop_info - ]).await?; +"#)?; + stmt.execute(params![ + replace_msg_id, + rfc724_mid, + if trash { DC_CHAT_ID_TRASH } else { chat_id }, + if trash { ContactId::UNDEFINED } else { from_id }, + if trash { ContactId::UNDEFINED } else { to_id }, + sort_timestamp, + sent_timestamp, + rcvd_timestamp, + typ, + state, + is_dc_message, + if trash { "" } else { msg }, + if trash { "" } else { &subject }, + // txt_raw might contain invalid utf8 + if trash { "" } else { &txt_raw }, + if trash { + "".to_string() + } else { + param.to_string() + }, + part.bytes as isize, + if (save_mime_headers || mime_modified) && !trash { + mime_headers.clone() + } else { + Vec::new() + }, + mime_in_reply_to, + mime_references, + mime_modified, + part.error.as_deref().unwrap_or_default(), + ephemeral_timer, + ephemeral_timestamp, + if is_partial_download.is_some() { + DownloadState::Available + } else { + DownloadState::Done + }, + mime_parser.hop_info + ])?; + let row_id = conn.last_insert_rowid(); + Ok(row_id) + }) + .await?; // We only replace placeholder with a first part, // afterwards insert additional parts. From a82b09bfc23fb1afd5bfb7f8e6641d34adb89903 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 19:21:14 +0000 Subject: [PATCH 10/34] Move TLS support to net::tls module --- src/imap/client.rs | 16 +++++---------- src/login_param.rs | 33 ------------------------------ src/net.rs | 1 + src/net/tls.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++++ src/smtp.rs | 17 ++++++---------- 5 files changed, 62 insertions(+), 55 deletions(-) create mode 100644 src/net/tls.rs diff --git a/src/imap/client.rs b/src/imap/client.rs index 5e8d0895d..265a9287a 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -11,9 +11,9 @@ use tokio::io::BufWriter; use super::capabilities::Capabilities; use super::session::Session; use crate::context::Context; -use crate::login_param::build_tls; use crate::net::connect_tcp; use crate::net::session::SessionStream; +use crate::net::tls::wrap_tls; use crate::socks::Socks5Config; /// IMAP write and read timeout. @@ -95,8 +95,7 @@ impl Client { strict_tls: bool, ) -> Result { let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(hostname, tcp_stream).await?; + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -142,9 +141,7 @@ impl Client { .context("STARTTLS command failed")?; let tcp_stream = client.into_inner(); - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, tcp_stream) + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream) .await .context("STARTTLS upgrade failed")?; @@ -165,8 +162,7 @@ impl Client { let socks5_stream = socks5_config .connect(context, domain, port, IMAP_TIMEOUT, strict_tls) .await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(domain, socks5_stream).await?; + let tls_stream = wrap_tls(strict_tls, domain, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -221,9 +217,7 @@ impl Client { .context("STARTTLS command failed")?; let socks5_stream = client.into_inner(); - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, socks5_stream) + let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream) .await .context("STARTTLS upgrade failed")?; let buffered_stream = BufWriter::new(tls_stream); diff --git a/src/login_param.rs b/src/login_param.rs index ee674c1fc..e01e43a0f 100644 --- a/src/login_param.rs +++ b/src/login_param.rs @@ -3,8 +3,6 @@ use std::fmt; use anyhow::{ensure, Result}; -use async_native_tls::Certificate; -use once_cell::sync::Lazy; use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2}; use crate::provider::{get_provider_by_id, Provider}; @@ -306,28 +304,6 @@ fn unset_empty(s: &str) -> &str { } } -// this certificate is missing on older android devices (eg. lg with android6 from 2017) -// certificate downloaded from https://letsencrypt.org/certificates/ -static LETSENCRYPT_ROOT: Lazy = Lazy::new(|| { - Certificate::from_der(include_bytes!( - "../assets/root-certificates/letsencrypt/isrgrootx1.der" - )) - .unwrap() -}); - -pub fn build_tls(strict_tls: bool) -> async_native_tls::TlsConnector { - let tls_builder = - async_native_tls::TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone()); - - if strict_tls { - tls_builder - } else { - tls_builder - .danger_accept_invalid_hostnames(true) - .danger_accept_invalid_certs(true) - } -} - #[cfg(test)] mod tests { use super::*; @@ -378,13 +354,4 @@ mod tests { assert_eq!(param, loaded); Ok(()) } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_build_tls() -> Result<()> { - // we are using some additional root certificates. - // make sure, they do not break construction of TlsConnector - let _ = build_tls(true); - let _ = build_tls(false); - Ok(()) - } } diff --git a/src/net.rs b/src/net.rs index 2fb9cb6b6..2d36e90aa 100644 --- a/src/net.rs +++ b/src/net.rs @@ -13,6 +13,7 @@ use crate::context::Context; use crate::tools::time; pub(crate) mod session; +pub(crate) mod tls; async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result { let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) diff --git a/src/net/tls.rs b/src/net/tls.rs new file mode 100644 index 000000000..980504416 --- /dev/null +++ b/src/net/tls.rs @@ -0,0 +1,50 @@ +//! TLS support. + +use anyhow::Result; +use async_native_tls::{Certificate, TlsConnector, TlsStream}; +use once_cell::sync::Lazy; +use tokio::io::{AsyncRead, AsyncWrite}; + +// this certificate is missing on older android devices (eg. lg with android6 from 2017) +// certificate downloaded from https://letsencrypt.org/certificates/ +static LETSENCRYPT_ROOT: Lazy = Lazy::new(|| { + Certificate::from_der(include_bytes!( + "../../assets/root-certificates/letsencrypt/isrgrootx1.der" + )) + .unwrap() +}); + +pub fn build_tls(strict_tls: bool) -> TlsConnector { + let tls_builder = TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone()); + + if strict_tls { + tls_builder + } else { + tls_builder + .danger_accept_invalid_hostnames(true) + .danger_accept_invalid_certs(true) + } +} + +pub async fn wrap_tls( + strict_tls: bool, + hostname: &str, + stream: T, +) -> Result> { + let tls = build_tls(strict_tls); + let tls_stream = tls.connect(hostname, stream).await?; + Ok(tls_stream) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_tls() { + // we are using some additional root certificates. + // make sure, they do not break construction of TlsConnector + let _ = build_tls(true); + let _ = build_tls(false); + } +} diff --git a/src/smtp.rs b/src/smtp.rs index cda3aff05..ca0e69ff8 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -13,12 +13,13 @@ use tokio::task; use crate::config::Config; use crate::contact::{Contact, ContactId}; use crate::events::EventType; -use crate::login_param::{build_tls, CertificateChecks, LoginParam, ServerLoginParam}; +use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam}; use crate::message::Message; use crate::message::{self, MsgId}; use crate::mimefactory::MimeFactory; use crate::net::connect_tcp; use crate::net::session::SessionStream; +use crate::net::tls::wrap_tls; use crate::oauth2::get_oauth2_access_token; use crate::provider::Socket; use crate::socks::Socks5Config; @@ -119,8 +120,7 @@ impl Smtp { let socks5_stream = socks5_config .connect(context, hostname, port, SMTP_TIMEOUT, strict_tls) .await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(hostname, socks5_stream).await?; + let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let client = smtp::SmtpClient::new().smtp_utf8(true); @@ -144,9 +144,7 @@ impl Smtp { let client = smtp::SmtpClient::new().smtp_utf8(true); let transport = SmtpTransport::new(client, socks5_stream).await?; let tcp_stream = transport.starttls().await?; - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, tcp_stream) + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream) .await .context("STARTTLS upgrade failed")?; let buffered_stream = BufWriter::new(tls_stream); @@ -181,8 +179,7 @@ impl Smtp { strict_tls: bool, ) -> Result>> { let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(hostname, tcp_stream).await?; + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let client = smtp::SmtpClient::new().smtp_utf8(true); @@ -203,9 +200,7 @@ impl Smtp { let client = smtp::SmtpClient::new().smtp_utf8(true); let transport = SmtpTransport::new(client, tcp_stream).await?; let tcp_stream = transport.starttls().await?; - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, tcp_stream) + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream) .await .context("STARTTLS upgrade failed")?; let buffered_stream = BufWriter::new(tls_stream); From d0a7e5f1b79ad5c0152e2ee2d1d11100a2838fc1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 14:32:22 +0000 Subject: [PATCH 11/34] Update timestamps in parameters with transactions This avoids accidentally reverting the changes to the `param` column done by another thread. --- CHANGELOG.md | 1 + src/update_helper.rs | 55 +++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b69dbc2..0213cb803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted. Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972 - Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 +- Update timestamps in `param` columns with transactions. #4083 ### API-Changes diff --git a/src/update_helper.rs b/src/update_helper.rs index 9136b23df..ed140dfdd 100644 --- a/src/update_helper.rs +++ b/src/update_helper.rs @@ -2,8 +2,8 @@ use anyhow::Result; -use crate::chat::{Chat, ChatId}; -use crate::contact::{Contact, ContactId}; +use crate::chat::ChatId; +use crate::contact::ContactId; use crate::context::Context; use crate::param::{Param, Params}; @@ -17,12 +17,26 @@ impl Context { scope: Param, new_timestamp: i64, ) -> Result { - let mut contact = Contact::load_from_db(self, contact_id).await?; - if contact.param.update_timestamp(scope, new_timestamp)? { - contact.update_param(self).await?; - return Ok(true); - } - Ok(false) + self.sql + .transaction(|transaction| { + let mut param: Params = transaction.query_row( + "SELECT param FROM contacts WHERE id=?", + [contact_id], + |row| { + let param: String = row.get(0)?; + Ok(param.parse().unwrap_or_default()) + }, + )?; + let update = param.update_timestamp(scope, new_timestamp)?; + if update { + transaction.execute( + "UPDATE contacts SET param=? WHERE id=?", + params![param.to_string(), contact_id], + )?; + } + Ok(update) + }) + .await } } @@ -35,12 +49,24 @@ impl ChatId { scope: Param, new_timestamp: i64, ) -> Result { - let mut chat = Chat::load_from_db(context, *self).await?; - if chat.param.update_timestamp(scope, new_timestamp)? { - chat.update_param(context).await?; - return Ok(true); - } - Ok(false) + context + .sql + .transaction(|transaction| { + let mut param: Params = + transaction.query_row("SELECT param FROM chats WHERE id=?", [self], |row| { + let param: String = row.get(0)?; + Ok(param.parse().unwrap_or_default()) + })?; + let update = param.update_timestamp(scope, new_timestamp)?; + if update { + transaction.execute( + "UPDATE chats SET param=? WHERE id=?", + params![param.to_string(), self], + )?; + } + Ok(update) + }) + .await } } @@ -60,6 +86,7 @@ impl Params { #[cfg(test)] mod tests { use super::*; + use crate::chat::Chat; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; use crate::tools::time; From 7af1a194915b385b3c434e9ff2596ccffee81fe6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 18:43:10 +0000 Subject: [PATCH 12/34] Do not build more than one deltachat-rpc-server at a time --- .github/workflows/deltachat-rpc-server.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 0cf842f16..72c3dc919 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -2,6 +2,10 @@ name: Build deltachat-rpc-server binaries +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: workflow_dispatch: From a2e088b415422bbddd6f09ec77abd5c238b2dd06 Mon Sep 17 00:00:00 2001 From: adbenitez Date: Wed, 22 Feb 2023 20:52:29 -0500 Subject: [PATCH 13/34] Compile deltachat-rpc-server for Android Co-Authored-By: link2xt --- .github/workflows/deltachat-rpc-server.yml | 44 ++++++++++++++++++++++ scripts/android-rpc-server.sh | 44 ++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100755 scripts/android-rpc-server.sh diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 72c3dc919..f073474f6 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -51,6 +51,50 @@ jobs: path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server if-no-files-found: error + build_android: + name: Cross-compile deltachat-rpc-server for Android (armeabi-v7a, arm64-v8a, x86 and x86_64) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - uses: nttld/setup-ndk@v1 + id: setup-ndk + with: + ndk-version: r21d + + - name: Build + env: + ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }} + run: sh scripts/android-rpc-server.sh + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-armv7 + path: target/armv7-linux-androideabi/release/deltachat-rpc-server + if-no-files-found: error + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-aarch64 + path: target/aarch64-linux-android/release/deltachat-rpc-server + if-no-files-found: error + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-i686 + path: target/i686-linux-android/release/deltachat-rpc-server + if-no-files-found: error + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-x86_64 + path: target/x86_64-linux-android/release/deltachat-rpc-server + if-no-files-found: error + build_windows: name: Build deltachat-rpc-server for Windows strategy: diff --git a/scripts/android-rpc-server.sh b/scripts/android-rpc-server.sh new file mode 100755 index 000000000..94963cbcd --- /dev/null +++ b/scripts/android-rpc-server.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Build deltachat-rpc-server for Android. + +set -e + +test -n "$ANDROID_NDK_ROOT" || exit 1 + +RUSTUP_TOOLCHAIN="1.64.0" +rustup install "$RUSTUP_TOOLCHAIN" +rustup target add armv7-linux-androideabi aarch64-linux-android i686-linux-android x86_64-linux-android --toolchain "$RUSTUP_TOOLCHAIN" + +KERNEL="$(uname -s | tr '[:upper:]' '[:lower:]')" +ARCH="$(uname -m)" +NDK_HOST_TAG="$KERNEL-$ARCH" +TOOLCHAIN="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$NDK_HOST_TAG" +export PATH="$PATH:$TOOLCHAIN/bin/" + +PACKAGE="deltachat-rpc-server" + +export CARGO_PROFILE_RELEASE_LTO=on + +CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="$TOOLCHAIN/bin/armv7a-linux-androideabi16-clang" \ + CFLAGS=-D__ANDROID_API__=16 \ + TARGET_CC=armv7a-linux-androideabi16-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target armv7-linux-androideabi -p $PACKAGE + +CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/aarch64-linux-android21-clang" \ + CFLAGS=-D__ANDROID_API__=21 \ + TARGET_CC=aarch64-linux-android21-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target aarch64-linux-android -p $PACKAGE + +CARGO_TARGET_I686_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/i686-linux-android16-clang" \ + CFLAGS=-D__ANDROID_API__=16 \ + TARGET_CC=i686-linux-android16-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target i686-linux-android -p $PACKAGE + +CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/x86_64-linux-android21-clang" \ + CFLAGS=-D__ANDROID_API__=21 \ + TARGET_CC=x86_64-linux-android21-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target x86_64-linux-android -p $PACKAGE From 08c9deee042063e6fef3e1afb182f172787fa8d7 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 24 Feb 2023 15:37:03 +0100 Subject: [PATCH 14/34] add test for Context::update_contacts_timestamp(), esp. the condition and the update was not covered before --- src/receive_imf/tests.rs | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index 50aa285ce..2fec63df4 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -2133,6 +2133,76 @@ Original signature updated", Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_ignore_old_status_updates() -> Result<()> { + let t = TestContext::new_alice().await; + let bob_id = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("bob@example.net")?, + Origin::AddressBook, + ) + .await? + .0; + + receive_imf( + &t, + b"From: Bob +To: Alice +Message-ID: <2@example.org> +Date: Wed, 22 Feb 2023 20:00:00 +0000 + +body + +-- +sig wednesday", + false, + ) + .await?; + let chat_id = t.get_last_msg().await.chat_id; + let bob = Contact::load_from_db(&t, bob_id).await?; + assert_eq!(bob.get_status(), "sig wednesday"); + assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 1); + + receive_imf( + &t, + b"From: Bob +To: Alice +Message-ID: <1@example.org> +Date: Tue, 21 Feb 2023 20:00:00 +0000 + +body + +-- +sig tuesday", + false, + ) + .await?; + let bob = Contact::load_from_db(&t, bob_id).await?; + assert_eq!(bob.get_status(), "sig wednesday"); + assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 2); + + receive_imf( + &t, + b"From: Bob +To: Alice +Message-ID: <3@example.org> +Date: Thu, 23 Feb 2023 20:00:00 +0000 + +body + +-- +sig thursday", + false, + ) + .await?; + let bob = Contact::load_from_db(&t, bob_id).await?; + assert_eq!(bob.get_status(), "sig thursday"); + assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 3); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_chat_assignment_private_classical_reply() { for outgoing_is_classical in &[true, false] { From e3eb35e160188af96c2c9b305e2ec48a3dac8938 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 20 Feb 2023 22:18:16 +0100 Subject: [PATCH 15/34] deltachat-jsonrpc: bump yerpc version in ts client --- deltachat-jsonrpc/typescript/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index aded96fcc..41da2e146 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -3,7 +3,7 @@ "dependencies": { "@deltachat/tiny-emitter": "3.0.0", "isomorphic-ws": "^4.0.1", - "yerpc": "^0.3.3" + "yerpc": "^0.4.2" }, "devDependencies": { "@types/chai": "^4.2.21", From 75670134d879399b31e9345d79f16f944dfbfebc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 22 Feb 2023 19:16:15 +0100 Subject: [PATCH 16/34] chore(jsonrpc): bump yerpc version --- deltachat-jsonrpc/typescript/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 41da2e146..3015eda6f 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -3,7 +3,7 @@ "dependencies": { "@deltachat/tiny-emitter": "3.0.0", "isomorphic-ws": "^4.0.1", - "yerpc": "^0.4.2" + "yerpc": "^0.4.3" }, "devDependencies": { "@types/chai": "^4.2.21", From c8ce4ce5aaac33cb4ab8acae4ead3e671b317b44 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 13:32:05 +0000 Subject: [PATCH 17/34] Switch to "vX.Y.Z" tagging scheme Previously we used tags like "1.109.0" and "py-1.109.0". Since Python bindings are always released at the same time as the core, it is easier to use a single tag for them. "v" prefix is added to make matching by "v*" easier in CI and scripts. --- CHANGELOG.md | 1 + python/pyproject.toml | 4 ++-- scripts/concourse/docs_wheels.yml | 2 +- scripts/set_core_version.py | 6 ++---- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0213cb803..890baf64d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055 - Updated provider database. - Disable DKIM-Checks again #4076 +- Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/python/pyproject.toml b/python/pyproject.toml index 227fb73d2..ce8dfaf5b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -44,8 +44,8 @@ deltachat = [ [tool.setuptools_scm] root = ".." -tag_regex = '^(?Ppy-)?(?P[^\+]+)(?P.*)?$' -git_describe_command = "git describe --dirty --tags --long --match py-*.*" +tag_regex = '^(?Pv)?(?P[^\+]+)(?P.*)?$' +git_describe_command = "git describe --dirty --tags --long --match v*.*" [tool.black] line-length = 120 diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml index bd2074d4b..efd57c561 100644 --- a/scripts/concourse/docs_wheels.yml +++ b/scripts/concourse/docs_wheels.yml @@ -12,7 +12,7 @@ resources: source: branch: master uri: https://github.com/deltachat/deltachat-core-rust.git - tag_filter: "py-*" + tag_filter: "v*" jobs: - name: doxygen diff --git a/scripts/set_core_version.py b/scripts/set_core_version.py index f3f29bbac..84444f8ef 100755 --- a/scripts/set_core_version.py +++ b/scripts/set_core_version.py @@ -115,10 +115,8 @@ def main(): print("after commit, on master make sure to: ") print("") - print(f" git tag -a {newversion}") - print(f" git push origin {newversion}") - print(f" git tag -a py-{newversion}") - print(f" git push origin py-{newversion}") + print(f" git tag -a v{newversion}") + print(f" git push origin v{newversion}") print("") From 45817fcacdd560db95c27e94c79f0d37a9eac535 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 16:50:19 +0000 Subject: [PATCH 18/34] Release 1.110.0 --- CHANGELOG.md | 2 +- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 8 ++++---- deltachat-repl/Cargo.toml | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- package.json | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 890baf64d..9fd88c9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 1.110.0 ### Changes - use transaction in `Contact::add_or_lookup()` #4059 diff --git a/Cargo.lock b/Cargo.lock index 7e0219a99..1a6765405 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,7 +834,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deltachat" -version = "1.109.0" +version = "1.110.0" dependencies = [ "ansi_term", "anyhow", @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.109.0" +version = "1.110.0" dependencies = [ "anyhow", "async-channel", @@ -927,7 +927,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.109.0" +version = "1.110.0" dependencies = [ "ansi_term", "anyhow", @@ -942,7 +942,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.109.0" +version = "1.110.0" dependencies = [ "anyhow", "deltachat-jsonrpc", @@ -965,7 +965,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.109.0" +version = "1.110.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index 867a10b72..4378fae09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.109.0" +version = "1.110.0" edition = "2021" license = "MPL-2.0" rust-version = "1.63" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 2bbec1c1f..6043a6752 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.109.0" +version = "1.110.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index e2958a4b2..7b9a26719 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.109.0" +version = "1.110.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 3015eda6f..795f5eaf7 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -26,8 +26,8 @@ }, "exports": { ".": { - "require": "./dist/deltachat.cjs", - "import": "./dist/deltachat.js" + "import": "./dist/deltachat.js", + "require": "./dist/deltachat.cjs" } }, "license": "MPL-2.0", @@ -36,8 +36,8 @@ "scripts": { "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs", "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js", - "build:tsc": "tsc", "build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs", + "build:tsc": "tsc", "docs": "typedoc --out docs deltachat.ts", "example": "run-s build example:build example:start", "example:build": "esbuild --bundle dist/example/example.js --outfile=dist/example.bundle.js", @@ -55,5 +55,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.109.0" + "version": "1.110.0" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index 59e3a3774..bab15c6d7 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.109.0" +version = "1.110.0" edition = "2021" [dependencies] diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index a332e21b5..aa984ffc3 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.109.0" +version = "1.110.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 9045c36e2..d6e2b3344 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.109.0" + "version": "1.110.0" } \ No newline at end of file From aeadbb35f3cecb49f065b6701c9ebfe9905da817 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 17:46:35 +0000 Subject: [PATCH 19/34] Remove empty API-Changes section --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd88c9c6..291a5f67a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,6 @@ - Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 - Update timestamps in `param` columns with transactions. #4083 -### API-Changes - ## 1.109.0 From b6336ce7e92c8de2a18f2ae3755103e8b15b2705 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 17:46:45 +0000 Subject: [PATCH 20/34] Add Unreleased changelog section --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 291a5f67a..3adf1b5a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Changes + +### Fixes + +### API-Changes + + ## 1.110.0 ### Changes From 064f806d905da44ce09996d28394995dbc698efb Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 16:48:45 +0000 Subject: [PATCH 21/34] Remove UpdateRecentQuota job --- src/context.rs | 5 +++++ src/job.rs | 48 +++++++++++------------------------------------- src/quota.rs | 25 +++++++++++++------------ src/scheduler.rs | 8 ++++++++ src/sql.rs | 2 +- 5 files changed, 38 insertions(+), 50 deletions(-) diff --git a/src/context.rs b/src/context.rs index 04687ad0c..21276f47e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; @@ -206,6 +207,9 @@ pub struct InnerContext { /// Set to `None` if quota was never tried to load. pub(crate) quota: RwLock>, + /// Set to true if quota update is requested. + pub(crate) quota_update_request: AtomicBool, + /// Server ID response if ID capability is supported /// and the server returned non-NIL on the inbox connection. /// @@ -365,6 +369,7 @@ impl Context { scheduler: RwLock::new(None), ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow to send 6 messages immediately, no more than once every 10 seconds. quota: RwLock::new(None), + quota_update_request: AtomicBool::new(false), server_id: RwLock::new(None), creation_time: std::time::SystemTime::now(), last_full_folder_scan: Mutex::new(None), diff --git a/src/job.rs b/src/job.rs index 3279843a9..cd54262ea 100644 --- a/src/job.rs +++ b/src/job.rs @@ -58,9 +58,6 @@ macro_rules! job_try { )] #[repr(u32)] pub enum Action { - // this is user initiated so it should have a fairly high priority - UpdateRecentQuota = 140, - // This job will download partially downloaded messages completely // and is added when download_full() is called. // Most messages are downloaded automatically on fetch @@ -202,17 +199,6 @@ pub async fn kill_action(context: &Context, action: Action) -> Result<()> { Ok(()) } -pub async fn action_exists(context: &Context, action: Action) -> Result { - let exists = context - .sql - .exists( - "SELECT COUNT(*) FROM jobs WHERE action=?;", - paramsv![action], - ) - .await?; - Ok(exists) -} - pub(crate) enum Connection<'a> { Inbox(&'a mut Imap), } @@ -240,7 +226,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ if tries < JOB_RETRIES { info!(context, "increase job {} tries to {}", job, tries); job.tries = tries; - let time_offset = get_backoff_time_offset(tries, job.action); + let time_offset = get_backoff_time_offset(tries); job.desired_timestamp = time() + time_offset; info!( context, @@ -289,10 +275,6 @@ async fn perform_job_action( let try_res = match job.action { Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await, - Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await { - Ok(status) => status, - Err(err) => Status::Finished(Err(err)), - }, Action::DownloadMsg => job.download_msg(context, connection.inbox()).await, }; @@ -301,24 +283,16 @@ async fn perform_job_action( try_res } -fn get_backoff_time_offset(tries: u32, action: Action) -> i64 { - match action { - // Just try every 10s to update the quota - // If all retries are exhausted, a new job will be created when the quota information is needed - Action::UpdateRecentQuota => 10, - - _ => { - // Exponential backoff - let n = 2_i32.pow(tries - 1) * 60; - let mut rng = thread_rng(); - let r: i32 = rng.gen(); - let mut seconds = r % (n + 1); - if seconds < 1 { - seconds = 1; - } - i64::from(seconds) - } +fn get_backoff_time_offset(tries: u32) -> i64 { + // Exponential backoff + let n = 2_i32.pow(tries - 1) * 60; + let mut rng = thread_rng(); + let r: i32 = rng.gen(); + let mut seconds = r % (n + 1); + if seconds < 1 { + seconds = 1; } + i64::from(seconds) } pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { @@ -339,7 +313,7 @@ pub async fn add(context: &Context, job: Job) -> Result<()> { if delay_seconds == 0 { match action { - Action::ResyncFolders | Action::UpdateRecentQuota | Action::DownloadMsg => { + Action::ResyncFolders | Action::DownloadMsg => { info!(context, "interrupt: imap"); context.interrupt_inbox(InterruptInfo::new(false)).await; } diff --git a/src/quota.rs b/src/quota.rs index 43cdd3465..90eab5222 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -1,6 +1,7 @@ //! # Support for IMAP QUOTA extension. use std::collections::BTreeMap; +use std::sync::atomic::Ordering; use anyhow::{anyhow, Context as _, Result}; use async_imap::types::{Quota, QuotaResource}; @@ -11,11 +12,10 @@ use crate::context::Context; use crate::imap::scan_folders::get_watched_folders; use crate::imap::session::Session as ImapSession; use crate::imap::Imap; -use crate::job::{Action, Status}; use crate::message::{Message, Viewtype}; -use crate::param::Params; +use crate::scheduler::InterruptInfo; use crate::tools::time; -use crate::{job, stock_str, EventType}; +use crate::{stock_str, EventType}; /// warn about a nearly full mailbox after this usage percentage is reached. /// quota icon is "yellow". @@ -112,12 +112,10 @@ pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> b impl Context { // Adds a job to update `quota.recent` pub(crate) async fn schedule_quota_update(&self) -> Result<()> { - if !job::action_exists(self, Action::UpdateRecentQuota).await? { - job::add( - self, - job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0), - ) - .await?; + let requested = self.quota_update_request.swap(true, Ordering::Relaxed); + if !requested { + // Quota update was not requested before. + self.interrupt_inbox(InterruptInfo::new(false)).await; } Ok(()) } @@ -132,10 +130,10 @@ impl Context { /// and new space is allocated as needed. /// /// Called in response to `Action::UpdateRecentQuota`. - pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result { + pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<()> { if let Err(err) = imap.prepare(self).await { warn!(self, "could not connect: {:#}", err); - return Ok(Status::RetryNow); + return Ok(()); } let session = imap.session.as_mut().context("no session")?; @@ -166,13 +164,16 @@ impl Context { } } + // Clear the request to update quota. + self.quota_update_request.store(false, Ordering::Relaxed); + *self.quota.write().await = Some(QuotaInfo { recent: quota, modified: time(), }); self.emit_event(EventType::ConnectivityChanged); - Ok(Status::Finished(Ok(()))) + Ok(()) } } diff --git a/src/scheduler.rs b/src/scheduler.rs index df9312c4e..4ce26c45d 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,4 +1,5 @@ use std::iter::{self, once}; +use std::sync::atomic::Ordering; use anyhow::{bail, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; @@ -128,6 +129,13 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne info = Default::default(); } None => { + let requested = ctx.quota_update_request.swap(false, Ordering::Relaxed); + if requested { + if let Err(err) = ctx.update_recent_quota(&mut connection).await { + warn!(ctx, "Failed to update quota: {:#}.", err); + } + } + maybe_add_time_based_warnings(&ctx).await; match ctx.get_config_i64(Config::LastHousekeeping).await { diff --git a/src/sql.rs b/src/sql.rs index 163d4e989..07884f875 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -983,7 +983,7 @@ mod tests { assert_eq!(avatar_bytes, &tokio::fs::read(&a).await.unwrap()[..]); t.sql.close().await; - housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed + housekeeping(&t).await.unwrap(); // housekeeping should emit warnings but not fail t.sql.open(&t, "".to_string()).await.unwrap(); let a = t.get_config(Config::Selfavatar).await.unwrap().unwrap(); From c02c5ffe2c45fa58a7a86c0352599744f6736f13 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 22:30:48 +0000 Subject: [PATCH 22/34] Format docs_wheels.yml with `prettier` --- scripts/concourse/docs_wheels.yml | 694 +++++++++++++++--------------- 1 file changed, 347 insertions(+), 347 deletions(-) diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml index efd57c561..3f461ad20 100644 --- a/scripts/concourse/docs_wheels.yml +++ b/scripts/concourse/docs_wheels.yml @@ -1,370 +1,370 @@ resources: -- name: deltachat-core-rust - type: git - icon: github - source: - branch: master - uri: https://github.com/deltachat/deltachat-core-rust.git + - name: deltachat-core-rust + type: git + icon: github + source: + branch: master + uri: https://github.com/deltachat/deltachat-core-rust.git -- name: deltachat-core-rust-release - type: git - icon: github - source: - branch: master - uri: https://github.com/deltachat/deltachat-core-rust.git - tag_filter: "v*" + - name: deltachat-core-rust-release + type: git + icon: github + source: + branch: master + uri: https://github.com/deltachat/deltachat-core-rust.git + tag_filter: "v*" jobs: -- name: doxygen - plan: - - get: deltachat-core-rust - trigger: true + - name: doxygen + plan: + - get: deltachat-core-rust + trigger: true - # Build Doxygen documentation - - task: build-doxygen - config: - inputs: - - name: deltachat-core-rust - outputs: - - name: c-docs - image_resource: - source: - repository: alpine - type: registry-image - platform: linux - run: - path: sh - args: - - -ec - - | - apk add --no-cache doxygen git - cd deltachat-core-rust - scripts/run-doxygen.sh - cd .. - cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/ + # Build Doxygen documentation + - task: build-doxygen + config: + inputs: + - name: deltachat-core-rust + outputs: + - name: c-docs + image_resource: + source: + repository: alpine + type: registry-image + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache doxygen git + cd deltachat-core-rust + scripts/run-doxygen.sh + cd .. + cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/ - - task: upload-c-docs - config: - inputs: - - name: c-docs - image_resource: - type: registry-image - source: - repository: alpine - platform: linux - run: - path: sh - args: - - -ec - - | - apk add --no-cache rsync openssh-client - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master + - task: upload-c-docs + config: + inputs: + - name: c-docs + image_resource: + type: registry-image + source: + repository: alpine + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache rsync openssh-client + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master -- name: python-x86_64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-x86_64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-docs - path: ./python/doc/_build/ - # Binary wheels - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-docs + path: ./python/doc/_build/ + # Binary wheels + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload python docs to py.delta.chat - - task: upload-py-docs - config: - inputs: - - name: py-docs - image_resource: - type: registry-image - source: - repository: alpine - platform: linux - run: - path: sh - args: - - -ec - - | - apk add --no-cache rsync openssh-client - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master + # Upload python docs to py.delta.chat + - task: upload-py-docs + config: + inputs: + - name: py-docs + image_resource: + type: registry-image + source: + repository: alpine + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache rsync openssh-client + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master - # Upload x86_64 wheels and source packages - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*manylinux201* + # Upload x86_64 wheels and source packages + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*manylinux201* -- name: python-aarch64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-aarch64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload aarch64 wheels - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*manylinux201* + # Upload aarch64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*manylinux201* -- name: python-musl-x86_64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-musl-x86_64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload musl x86_64 wheels - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*musllinux_1_1_x86_64* + # Upload musl x86_64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*musllinux_1_1_x86_64* -- name: python-musl-aarch64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-musl-aarch64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload musl aarch64 wheels - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*musllinux_1_1_aarch64* + # Upload musl aarch64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*musllinux_1_1_aarch64* From 0890b669fad62d7fa05f0eba338b3f577a8164f1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 23:35:29 +0000 Subject: [PATCH 23/34] Move RFC URLs in standards.md to the end Also update Autodiscover URL --- standards.md | 66 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/standards.md b/standards.md index 916038bca..546c22780 100644 --- a/standards.md +++ b/standards.md @@ -3,26 +3,54 @@ Some of the standards Delta Chat is based on: Tasks | Standards ----------------------------------|--------------------------------------------- -Transport | IMAP v4 ([RFC 3501](https://tools.ietf.org/html/rfc3501)), SMTP ([RFC 5321](https://tools.ietf.org/html/rfc5321)) and Internet Message Format (IMF, [RFC 5322](https://tools.ietf.org/html/rfc5322)) -Proxy | SOCKS5 ([RFC 1928](https://tools.ietf.org/html/rfc1928)) -Embedded media | MIME Document Series ([RFC 2045](https://tools.ietf.org/html/rfc2045), [RFC 2046](https://tools.ietf.org/html/rfc2046)), Content-Disposition Header ([RFC 2183](https://tools.ietf.org/html/rfc2183)), Multipart/Related ([RFC 2387](https://tools.ietf.org/html/rfc2387)) -Text and Quote encoding | Fixed, Flowed ([RFC 3676](https://tools.ietf.org/html/rfc3676)) -Reactions | Reaction: Indicating Summary Reaction to a Message [RFC 9078](https://datatracker.ietf.org/doc/rfc9078/) -Filename encoding | Encoded Words ([RFC 2047](https://tools.ietf.org/html/rfc2047)), Encoded Word Extensions ([RFC 2231](https://tools.ietf.org/html/rfc2231)) -Identify server folders | IMAP LIST Extension ([RFC 6154](https://tools.ietf.org/html/rfc6154)) -Push | IMAP IDLE ([RFC 2177](https://tools.ietf.org/html/rfc2177)) -Quota | IMAP QUOTA extension ([RFC 2087](https://tools.ietf.org/html/rfc2087)) -Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162](https://tools.ietf.org/html/rfc7162)) -Client/server identification | IMAP ID extension ([RFC 2971](https://datatracker.ietf.org/doc/html/rfc2971)) -Authorization | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749)) -End-to-end encryption | [Autocrypt Level 1](https://autocrypt.org/level1.html), OpenPGP ([RFC 4880](https://tools.ietf.org/html/rfc4880)), Security Multiparts for MIME ([RFC 1847](https://tools.ietf.org/html/rfc1847)) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html) +-------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Transport | IMAP v4 ([RFC 3501][]), SMTP ([RFC 5321][]) and Internet Message Format (IMF, [RFC 5322][]) +Proxy | SOCKS5 ([RFC 1928][]) +Embedded media | MIME Document Series ([RFC 2045][], [RFC 2046][]), Content-Disposition Header ([RFC 2183][]), Multipart/Related ([RFC 2387][]) +Text and Quote encoding | Fixed, Flowed ([RFC 3676][]) +Reactions | Reaction: Indicating Summary Reaction to a Message ([RFC 9078][]) +Filename encoding | Encoded Words ([RFC 2047][]), Encoded Word Extensions ([RFC 2231][]) +Identify server folders | IMAP LIST Extension ([RFC 6154][]) +Push | IMAP IDLE ([RFC 2177][]) +Quota | IMAP QUOTA extension ([RFC 2087][]) +Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162][]) +Client/server identification | IMAP ID extension ([RFC 2971][]) +Authorization | OAuth2 ([RFC 6749][]) +End-to-end encryption | [Autocrypt Level 1][], OpenPGP ([RFC 4880][]), Security Multiparts for MIME ([RFC 1847][]) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html) Header encryption | [Protected Headers for Cryptographic E-mail](https://datatracker.ietf.org/doc/draft-autocrypt-lamps-protected-headers/) -Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx) +Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover][] Messenger functions | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-mail-specification) -Detect mailing list | List-Id ([RFC 2919](https://tools.ietf.org/html/rfc2919)) and Precedence ([RFC 3834](https://tools.ietf.org/html/rfc3834)) -User and chat colors | [XEP-0392](https://xmpp.org/extensions/xep-0392.html): Consistent Color Generation -Send and receive system messages | Multipart/Report Media Type ([RFC 6522](https://tools.ietf.org/html/rfc6522)) -Return receipts | Message Disposition Notification (MDN, [RFC 8098](https://tools.ietf.org/html/rfc8098), [RFC 3503](https://tools.ietf.org/html/rfc3503)) using the Chat-Disposition-Notification-To header +Detect mailing list | List-Id ([RFC 2919][]) and Precedence ([RFC 3834][]) +User and chat colors | [XEP-0392][]: Consistent Color Generation +Send and receive system messages | Multipart/Report Media Type ([RFC 6522][]) +Return receipts | Message Disposition Notification (MDN, [RFC 8098][], [RFC 3503][]) using the Chat-Disposition-Notification-To header Locations | KML ([Open Geospatial Consortium](http://www.opengeospatial.org/standards/kml/), [Google Dev](https://developers.google.com/kml/)) +[Autocrypt Level 1]: https://autocrypt.org/level1.html +[Autodiscover]: https://learn.microsoft.com/en-us/exchange/autodiscover-service-for-exchange-2013 +[XEP-0392]: https://xmpp.org/extensions/xep-0392.html +[RFC 1847]: https://tools.ietf.org/html/rfc1847 +[RFC 1928]: https://tools.ietf.org/html/rfc1928 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2087]: https://tools.ietf.org/html/rfc2087 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 2183]: https://tools.ietf.org/html/rfc2183 +[RFC 2231]: https://tools.ietf.org/html/rfc2231 +[RFC 2387]: https://tools.ietf.org/html/rfc2387 +[RFC 2919]: https://tools.ietf.org/html/rfc2919 +[RFC 2971]: https://tools.ietf.org/html/rfc2971 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 3503]: https://tools.ietf.org/html/rfc3503 +[RFC 3676]: https://tools.ietf.org/html/rfc3676 +[RFC 3834]: https://tools.ietf.org/html/rfc3834 +[RFC 4880]: https://tools.ietf.org/html/rfc4880 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 5322]: https://tools.ietf.org/html/rfc5322 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 6749]: https://tools.ietf.org/html/rfc6749 +[RFC 7162]: https://tools.ietf.org/html/rfc7162 +[RFC 8098]: https://tools.ietf.org/html/rfc8098 +[RFC 9078]: https://tools.ietf.org/html/rfc9078 From 8f0d07b93c694b3f729f611d8f68a479a320545f Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 01:14:14 +0000 Subject: [PATCH 24/34] Make smeared timestamp creation non-async Using atomic operations instead, so create_smeared_timestamp() can be used in sync functions, such as SQL transactions. --- CHANGELOG.md | 1 + src/chat.rs | 17 ++-- src/context.rs | 5 +- src/ephemeral.rs | 2 +- src/lib.rs | 1 + src/message.rs | 9 +-- src/mimefactory.rs | 2 +- src/receive_imf.rs | 4 +- src/timesmearing.rs | 193 ++++++++++++++++++++++++++++++++++++++++++++ src/tools.rs | 85 +++---------------- src/webxdc.rs | 2 +- 11 files changed, 222 insertions(+), 99 deletions(-) create mode 100644 src/timesmearing.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3adf1b5a7..b3aa335a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Changes +- Make smeared timestamp generation non-async. #4075 ### Fixes diff --git a/src/chat.rs b/src/chat.rs index 26e7a7db2..837b88164 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -276,7 +276,7 @@ impl ChatId { grpname, grpid, create_blocked, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), create_protected, param.unwrap_or_default(), ], @@ -482,7 +482,7 @@ impl ChatId { self, &msg_text, cmd, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), None, None, None, @@ -1956,7 +1956,6 @@ impl ChatIdBlocked { _ => (), } - let created_timestamp = create_smeared_timestamp(context).await; let chat_id = context .sql .transaction(move |transaction| { @@ -1969,7 +1968,7 @@ impl ChatIdBlocked { chat_name, params.to_string(), create_blocked as u8, - created_timestamp, + create_smeared_timestamp(context) ], )?; let chat_id = ChatId::new( @@ -2117,7 +2116,7 @@ async fn prepare_msg_common( context, msg, update_msg_id, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), ) .await?; msg.chat_id = chat_id; @@ -2842,7 +2841,7 @@ pub async fn create_group_chat( Chattype::Group, chat_name, grpid, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), ], ) .await?; @@ -2900,7 +2899,7 @@ pub async fn create_broadcast_list(context: &Context) -> Result { Chattype::Broadcast, chat_name, grpid, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), ], ) .await?; @@ -3361,7 +3360,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) if let Some(reason) = chat.why_cant_send(context).await? { bail!("cannot send to {}: {}", chat_id, reason); } - curr_timestamp = create_smeared_timestamps(context, msg_ids.len()).await; + curr_timestamp = create_smeared_timestamps(context, msg_ids.len()); let ids = context .sql .query_map( @@ -3563,7 +3562,7 @@ pub async fn add_device_msg_with_importance( msg.try_calc_and_set_dimensions(context).await.ok(); prepare_msg_blob(context, msg).await?; - let timestamp_sent = create_smeared_timestamp(context).await; + let timestamp_sent = create_smeared_timestamp(context); // makes sure, the added message is the last one, // even if the date is wrong (useful esp. when warning about bad dates) diff --git a/src/context.rs b/src/context.rs index 21276f47e..eebb5541c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -27,6 +27,7 @@ use crate::quota::QuotaInfo; use crate::scheduler::Scheduler; use crate::sql::Sql; use crate::stock_str::StockStrings; +use crate::timesmearing::SmearedTimestamp; use crate::tools::{duration_to_str, time}; /// Builder for the [`Context`]. @@ -189,7 +190,7 @@ pub struct InnerContext { /// Blob directory path pub(crate) blobdir: PathBuf, pub(crate) sql: Sql, - pub(crate) last_smeared_timestamp: RwLock, + pub(crate) smeared_timestamp: SmearedTimestamp, running_state: RwLock, /// Mutex to avoid generating the key for the user more than once. pub(crate) generating_key_mutex: Mutex<()>, @@ -360,7 +361,7 @@ impl Context { blobdir, running_state: RwLock::new(Default::default()), sql: Sql::new(dbfile), - last_smeared_timestamp: RwLock::new(0), + smeared_timestamp: SmearedTimestamp::new(), generating_key_mutex: Mutex::new(()), oauth2_mutex: Mutex::new(()), wrong_pw_warning_mutex: Mutex::new(()), diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 606c7270f..e881bd082 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -650,7 +650,7 @@ mod tests { use crate::download::DownloadState; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; - use crate::tools::MAX_SECONDS_TO_LEND_FROM_FUTURE; + use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE; use crate::{ chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus}, tools::IsNoneOrEmpty, diff --git a/src/lib.rs b/src/lib.rs index 9feb6329b..c224ab09d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,6 +92,7 @@ mod smtp; mod socks; pub mod stock_str; mod sync; +mod timesmearing; mod token; mod update_helper; pub mod webxdc; diff --git a/src/message.rs b/src/message.rs index a7b2e27e3..8e1528a55 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1771,13 +1771,8 @@ async fn ndn_maybe_add_info_msg( // Tell the user which of the recipients failed if we know that (because in // a group, this might otherwise be unclear) let text = stock_str::failed_sending_to(context, contact.get_display_name()).await; - chat::add_info_msg( - context, - chat_id, - &text, - create_smeared_timestamp(context).await, - ) - .await?; + chat::add_info_msg(context, chat_id, &text, create_smeared_timestamp(context)) + .await?; context.emit_event(EventType::ChatModified(chat_id)); } } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 43978c112..d98edabbe 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -250,7 +250,7 @@ impl<'a> MimeFactory<'a> { .get_config(Config::Selfstatus) .await? .unwrap_or_default(); - let timestamp = create_smeared_timestamp(context).await; + let timestamp = create_smeared_timestamp(context); let res = MimeFactory::<'a> { from_addr, diff --git a/src/receive_imf.rs b/src/receive_imf.rs index b86fbed7c..e8200ad62 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -203,7 +203,7 @@ pub(crate) async fn receive_imf_inner( ) .await?; - let rcvd_timestamp = smeared_time(context).await; + let rcvd_timestamp = smeared_time(context); // Sender timestamp is allowed to be a bit in the future due to // unsynchronized clocks, but not too much. @@ -1380,7 +1380,7 @@ async fn calc_sort_timestamp( } } - Ok(min(sort_timestamp, smeared_time(context).await)) + Ok(min(sort_timestamp, smeared_time(context))) } async fn lookup_chat_by_reply( diff --git a/src/timesmearing.rs b/src/timesmearing.rs new file mode 100644 index 000000000..d6b0d5be0 --- /dev/null +++ b/src/timesmearing.rs @@ -0,0 +1,193 @@ +//! # Time smearing. +//! +//! As e-mails typically only use a second-based-resolution for timestamps, +//! the order of two mails sent withing one second is unclear. +//! This is bad e.g. when forwarding some messages from a chat - +//! these messages will appear at the recipient easily out of order. +//! +//! We work around this issue by not sending out two mails with the same timestamp. +//! For this purpose, in short, we track the last timestamp used in `last_smeared_timestamp` +//! when another timestamp is needed in the same second, we use `last_smeared_timestamp+1` +//! after some moments without messages sent out, +//! `last_smeared_timestamp` is again in sync with the normal time. +//! +//! However, we do not do all this for the far future, +//! but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE` + +use std::cmp::{max, min}; +use std::sync::atomic::{AtomicI64, Ordering}; + +pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5; + +/// Smeared timestamp generator. +#[derive(Debug)] +pub struct SmearedTimestamp { + /// Next timestamp available for allocation. + smeared_timestamp: AtomicI64, +} + +impl SmearedTimestamp { + /// Creates a new smeared timestamp generator. + pub fn new() -> Self { + Self { + smeared_timestamp: AtomicI64::new(0), + } + } + + /// Allocates `count` unique timestamps. + /// + /// Returns the first allocated timestamp. + pub fn create_n(&self, now: i64, count: i64) -> i64 { + let mut prev = self.smeared_timestamp.load(Ordering::Relaxed); + loop { + // Advance the timestamp if it is in the past, + // but keep `count - 1` timestamps from the past if possible. + let t = max(prev, now - count + 1); + + // Rewind the time back if there is no room + // to allocate `count` timestamps without going too far into the future. + // Not going too far into the future + // is more important than generating unique timestamps. + let first = min(t, now + MAX_SECONDS_TO_LEND_FROM_FUTURE - count + 1); + + // Allocate `count` timestamps by advancing the current timestamp. + let next = first + count; + + if let Err(x) = self.smeared_timestamp.compare_exchange_weak( + prev, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + prev = x; + } else { + return first; + } + } + } + + /// Creates a single timestamp. + pub fn create(&self, now: i64) -> i64 { + self.create_n(now, 1) + } + + /// Returns the current smeared timestamp. + pub fn current(&self) -> i64 { + self.smeared_timestamp.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use std::time::SystemTime; + + use super::*; + use crate::test_utils::TestContext; + use crate::tools::{create_smeared_timestamp, create_smeared_timestamps, smeared_time, time}; + + #[test] + fn test_smeared_timestamp() { + let smeared_timestamp = SmearedTimestamp::new(); + let now = time(); + + assert_eq!(smeared_timestamp.current(), 0); + + for i in 0..MAX_SECONDS_TO_LEND_FROM_FUTURE { + assert_eq!(smeared_timestamp.create(now), now + i); + } + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + + // System time rewinds back by 1000 seconds. + let now = now - 1000; + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + assert_eq!( + smeared_timestamp.create(now + 1), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + 1 + ); + assert_eq!(smeared_timestamp.create(now + 100), now + 100); + assert_eq!(smeared_timestamp.create(now + 100), now + 101); + assert_eq!(smeared_timestamp.create(now + 100), now + 102); + } + + #[test] + fn test_create_n_smeared_timestamps() { + let smeared_timestamp = SmearedTimestamp::new(); + let now = time(); + + // Create a single timestamp to initialize the generator. + assert_eq!(smeared_timestamp.create(now), now); + + // Wait a minute. + let now = now + 60; + + // Simulate forwarding 7 messages. + let forwarded_messages = 7; + + // We have not sent anything for a minute, + // so we can take the current timestamp and take 6 timestamps from the past. + assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 6); + + assert_eq!(smeared_timestamp.current(), now + 1); + + // Wait 4 seconds. + // Now we have 3 free timestamps in the past. + let now = now + 4; + + assert_eq!(smeared_timestamp.current(), now - 3); + + // Forward another 7 messages. + // We can only lend 3 timestamps from the past. + assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 3); + + // We had to borrow 3 timestamps from the future + // because there were not enough timestamps in the past. + assert_eq!(smeared_timestamp.current(), now + 4); + + // Forward another 7 messages. + // We cannot use more than 5 timestamps from the future, + // so we use 5 timestamps from the future, + // the current timestamp and one timestamp from the past. + assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_create_smeared_timestamp() { + let t = TestContext::new().await; + assert_ne!(create_smeared_timestamp(&t), create_smeared_timestamp(&t)); + assert!( + create_smeared_timestamp(&t) + >= SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() as i64 + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_create_smeared_timestamps() { + let t = TestContext::new().await; + let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1; + let start = create_smeared_timestamps(&t, count as usize); + let next = smeared_time(&t); + assert!((start + count - 1) < next); + + let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30; + let start = create_smeared_timestamps(&t, count as usize); + let next = smeared_time(&t); + assert!((start + count - 1) < next); + } +} diff --git a/src/tools.rs b/src/tools.rs index 7e8d7e922..c64e8d00a 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -3,7 +3,6 @@ #![allow(missing_docs)] -use core::cmp::{max, min}; use std::borrow::Cow; use std::fmt; use std::io::Cursor; @@ -140,63 +139,27 @@ pub(crate) fn gm2local_offset() -> i64 { i64::from(lt.offset().local_minus_utc()) } -// timesmearing -// - as e-mails typically only use a second-based-resolution for timestamps, -// the order of two mails sent withing one second is unclear. -// this is bad eg. when forwarding some messages from a chat - -// these messages will appear at the recipient easily out of order. -// - we work around this issue by not sending out two mails with the same timestamp. -// - for this purpose, in short, we track the last timestamp used in `last_smeared_timestamp` -// when another timestamp is needed in the same second, we use `last_smeared_timestamp+1` -// - after some moments without messages sent out, -// `last_smeared_timestamp` is again in sync with the normal time. -// - however, we do not do all this for the far future, -// but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE` -pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5; - /// Returns the current smeared timestamp, /// /// The returned timestamp MUST NOT be sent out. -pub(crate) async fn smeared_time(context: &Context) -> i64 { - let mut now = time(); - let ts = *context.last_smeared_timestamp.read().await; - if ts >= now { - now = ts + 1; - } - - now +pub(crate) fn smeared_time(context: &Context) -> i64 { + let now = time(); + let ts = context.smeared_timestamp.current(); + std::cmp::max(ts, now) } /// Returns a timestamp that is guaranteed to be unique. -pub(crate) async fn create_smeared_timestamp(context: &Context) -> i64 { +pub(crate) fn create_smeared_timestamp(context: &Context) -> i64 { let now = time(); - let mut ret = now; - - let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await; - if ret <= *last_smeared_timestamp { - ret = *last_smeared_timestamp + 1; - if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE { - ret = now + MAX_SECONDS_TO_LEND_FROM_FUTURE - } - } - - *last_smeared_timestamp = ret; - ret + context.smeared_timestamp.create(now) } // creates `count` timestamps that are guaranteed to be unique. -// the frist created timestamps is returned directly, +// the first created timestamps is returned directly, // get the other timestamps just by adding 1..count-1 -pub(crate) async fn create_smeared_timestamps(context: &Context, count: usize) -> i64 { +pub(crate) fn create_smeared_timestamps(context: &Context, count: usize) -> i64 { let now = time(); - let count = count as i64; - let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count; - - let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await; - start = max(*last_smeared_timestamp + 1, start); - - *last_smeared_timestamp = start + count - 1; - start + context.smeared_timestamp.create_n(now, count as i64) } // if the system time is not plausible, once a day, add a device message. @@ -1071,36 +1034,6 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; assert!(!file_exist!(context, &fn0)); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_create_smeared_timestamp() { - let t = TestContext::new().await; - assert_ne!( - create_smeared_timestamp(&t).await, - create_smeared_timestamp(&t).await - ); - assert!( - create_smeared_timestamp(&t).await - >= SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() as i64 - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_create_smeared_timestamps() { - let t = TestContext::new().await; - let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1; - let start = create_smeared_timestamps(&t, count as usize).await; - let next = smeared_time(&t).await; - assert!((start + count - 1) < next); - - let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30; - let start = create_smeared_timestamps(&t, count as usize).await; - let next = smeared_time(&t).await; - assert!((start + count - 1) < next); - } - #[test] fn test_duration_to_str() { assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s"); diff --git a/src/webxdc.rs b/src/webxdc.rs index f4cf660a6..de786ea99 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -408,7 +408,7 @@ impl Context { .create_status_update_record( &mut instance, update_str, - create_smeared_timestamp(self).await, + create_smeared_timestamp(self), send_now, ContactId::SELF, ) From 3fc67de35ed56049498b2d263663438a79564082 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:33:21 +0000 Subject: [PATCH 25/34] Stop saving and loading jobs.param column --- src/download.rs | 7 +------ src/job.rs | 23 ++++++----------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/download.rs b/src/download.rs index 5b544fe31..2fed19841 100644 --- a/src/download.rs +++ b/src/download.rs @@ -13,7 +13,6 @@ use crate::imap::{Imap, ImapActionResult}; use crate::job::{self, Action, Job, Status}; use crate::message::{Message, MsgId, Viewtype}; use crate::mimeparser::{MimeMessage, Part}; -use crate::param::Params; use crate::tools::time; use crate::{job_try, stock_str, EventType}; @@ -86,11 +85,7 @@ impl MsgId { DownloadState::Available | DownloadState::Failure => { self.update_download_state(context, DownloadState::InProgress) .await?; - job::add( - context, - Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0), - ) - .await?; + job::add(context, Job::new(Action::DownloadMsg, self.to_u32(), 0)).await?; } } Ok(()) diff --git a/src/job.rs b/src/job.rs index cd54262ea..9a16421f7 100644 --- a/src/job.rs +++ b/src/job.rs @@ -13,7 +13,6 @@ use rand::{thread_rng, Rng}; use crate::context::Context; use crate::imap::{get_folder_meaning, FolderMeaning, Imap}; -use crate::param::Params; use crate::scheduler::InterruptInfo; use crate::tools::time; @@ -77,7 +76,6 @@ pub struct Job { pub desired_timestamp: i64, pub added_timestamp: i64, pub tries: u32, - pub param: Params, } impl fmt::Display for Job { @@ -87,7 +85,7 @@ impl fmt::Display for Job { } impl Job { - pub fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self { + pub fn new(action: Action, foreign_id: u32, delay_seconds: i64) -> Self { let timestamp = time(); Self { @@ -97,7 +95,6 @@ impl Job { desired_timestamp: timestamp + delay_seconds, added_timestamp: timestamp, tries: 0, - param, } } @@ -127,23 +124,21 @@ impl Job { context .sql .execute( - "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;", + "UPDATE jobs SET desired_timestamp=?, tries=? WHERE id=?;", paramsv![ self.desired_timestamp, i64::from(self.tries), - self.param.to_string(), self.job_id as i32, ], ) .await?; } else { context.sql.execute( - "INSERT INTO jobs (added_timestamp, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?);", + "INSERT INTO jobs (added_timestamp, action, foreign_id, desired_timestamp) VALUES (?,?,?,?);", paramsv![ self.added_timestamp, self.action, self.foreign_id, - self.param.to_string(), self.desired_timestamp ] ).await?; @@ -297,11 +292,7 @@ fn get_backoff_time_offset(tries: u32) -> i64 { pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { kill_action(context, Action::ResyncFolders).await?; - add( - context, - Job::new(Action::ResyncFolders, 0, Params::new(), 0), - ) - .await?; + add(context, Job::new(Action::ResyncFolders, 0, 0)).await?; Ok(()) } @@ -370,7 +361,6 @@ LIMIT 1; desired_timestamp: row.get("desired_timestamp")?, added_timestamp: row.get("added_timestamp")?, tries: row.get("tries")?, - param: row.get::<_, String>("param")?.parse().unwrap_or_default(), }; Ok(job) @@ -410,8 +400,8 @@ mod tests { .sql .execute( "INSERT INTO jobs - (added_timestamp, action, foreign_id, param, desired_timestamp) - VALUES (?, ?, ?, ?, ?);", + (added_timestamp, action, foreign_id, desired_timestamp) + VALUES (?, ?, ?, ?);", paramsv![ now, if valid { @@ -420,7 +410,6 @@ mod tests { -1 }, foreign_id, - Params::new().to_string(), now ], ) From 7f2ccfb168b0b27733be37cbd0649bf7fc679d77 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:33:47 +0000 Subject: [PATCH 26/34] job: remove match with a single branch --- src/job.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/job.rs b/src/job.rs index 9a16421f7..7333ce498 100644 --- a/src/job.rs +++ b/src/job.rs @@ -303,12 +303,8 @@ pub async fn add(context: &Context, job: Job) -> Result<()> { job.save(context).await.context("failed to save job")?; if delay_seconds == 0 { - match action { - Action::ResyncFolders | Action::DownloadMsg => { - info!(context, "interrupt: imap"); - context.interrupt_inbox(InterruptInfo::new(false)).await; - } - } + info!(context, "interrupt: imap"); + context.interrupt_inbox(InterruptInfo::new(false)).await; } Ok(()) } From 10e8bcb73ea43b69d56bd22c14d874b575f412f6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:35:08 +0000 Subject: [PATCH 27/34] job: remove unused variable --- src/job.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index 7333ce498..df808f458 100644 --- a/src/job.rs +++ b/src/job.rs @@ -298,7 +298,6 @@ pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { /// Adds a job to the database, scheduling it. pub async fn add(context: &Context, job: Job) -> Result<()> { - let action = job.action; let delay_seconds = job.delay_seconds(); job.save(context).await.context("failed to save job")?; From 9f804c379c5818639d7105a1716487080841ef58 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:42:14 +0000 Subject: [PATCH 28/34] Remove always zero argument to Job::new() --- src/download.rs | 2 +- src/job.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/download.rs b/src/download.rs index 2fed19841..fd0efc9c3 100644 --- a/src/download.rs +++ b/src/download.rs @@ -85,7 +85,7 @@ impl MsgId { DownloadState::Available | DownloadState::Failure => { self.update_download_state(context, DownloadState::InProgress) .await?; - job::add(context, Job::new(Action::DownloadMsg, self.to_u32(), 0)).await?; + job::add(context, Job::new(Action::DownloadMsg, self.to_u32())).await?; } } Ok(()) diff --git a/src/job.rs b/src/job.rs index df808f458..4ba203823 100644 --- a/src/job.rs +++ b/src/job.rs @@ -85,14 +85,14 @@ impl fmt::Display for Job { } impl Job { - pub fn new(action: Action, foreign_id: u32, delay_seconds: i64) -> Self { + pub fn new(action: Action, foreign_id: u32) -> Self { let timestamp = time(); Self { job_id: 0, action, foreign_id, - desired_timestamp: timestamp + delay_seconds, + desired_timestamp: timestamp, added_timestamp: timestamp, tries: 0, } @@ -292,7 +292,7 @@ fn get_backoff_time_offset(tries: u32) -> i64 { pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { kill_action(context, Action::ResyncFolders).await?; - add(context, Job::new(Action::ResyncFolders, 0, 0)).await?; + add(context, Job::new(Action::ResyncFolders, 0)).await?; Ok(()) } From 1450bf5483305894063345673b9c3202d54c4bc3 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:45:22 +0000 Subject: [PATCH 29/34] Remove Job::delay_seconds() It always returned 0 for added jobs. --- src/job.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/job.rs b/src/job.rs index 4ba203823..ff37ea543 100644 --- a/src/job.rs +++ b/src/job.rs @@ -98,10 +98,6 @@ impl Job { } } - pub fn delay_seconds(&self) -> i64 { - self.desired_timestamp - self.added_timestamp - } - /// Deletes the job from the database. async fn delete(self, context: &Context) -> Result<()> { if self.job_id != 0 { @@ -298,13 +294,10 @@ pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { /// Adds a job to the database, scheduling it. pub async fn add(context: &Context, job: Job) -> Result<()> { - let delay_seconds = job.delay_seconds(); job.save(context).await.context("failed to save job")?; - if delay_seconds == 0 { - info!(context, "interrupt: imap"); - context.interrupt_inbox(InterruptInfo::new(false)).await; - } + info!(context, "interrupt: imap"); + context.interrupt_inbox(InterruptInfo::new(false)).await; Ok(()) } From 992a6cbfc2c395cf8d636ce91c582d1659a5ed94 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 20:51:38 +0000 Subject: [PATCH 30/34] mimeparser: wrap try_decrypt() into block_in_place() try_decrypt() is a CPU-bound task. When called from async function, it should be wrapped in tokio::task::spawn_blocking(). Using tokio::task::spawn_blocking() is difficult here because of &mail, &private_keyring and &public_keyring borrows, so we should at least use tokio::task::block_in_place() to avoid blocking the executor. --- CHANGELOG.md | 1 + src/mimeparser.rs | 39 ++++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3aa335a8..39bb421be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Make smeared timestamp generation non-async. #4075 ### Fixes +- Do not block async task executor while decrypting the messages. #4079 ### API-Changes diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 0b246a257..075fd192f 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -256,26 +256,27 @@ impl MimeMessage { hop_info += &decryption_info.dkim_results.to_string(); let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref()); - let (mail, mut signatures, encrypted) = - match try_decrypt(context, &mail, &private_keyring, &public_keyring) { - Ok(Some((raw, signatures))) => { - mail_raw = raw; - let decrypted_mail = mailparse::parse_mail(&mail_raw)?; - if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { - info!( - context, - "decrypted message mime-body:\n{}", - String::from_utf8_lossy(&mail_raw), - ); - } - (Ok(decrypted_mail), signatures, true) + let (mail, mut signatures, encrypted) = match tokio::task::block_in_place(|| { + try_decrypt(context, &mail, &private_keyring, &public_keyring) + }) { + Ok(Some((raw, signatures))) => { + mail_raw = raw; + let decrypted_mail = mailparse::parse_mail(&mail_raw)?; + if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { + info!( + context, + "decrypted message mime-body:\n{}", + String::from_utf8_lossy(&mail_raw), + ); } - Ok(None) => (Ok(mail), HashSet::new(), false), - Err(err) => { - warn!(context, "decryption failed: {:#}", err); - (Err(err), HashSet::new(), false) - } - }; + (Ok(decrypted_mail), signatures, true) + } + Ok(None) => (Ok(mail), HashSet::new(), false), + Err(err) => { + warn!(context, "decryption failed: {:#}", err); + (Err(err), HashSet::new(), false) + } + }; let mail = mail.as_ref().map(|mail| { let (content, signatures_detached) = validate_detached_signature(mail, &public_keyring) .unwrap_or((mail, Default::default())); From 89696582ada7f1ef76aec88a4aef7a14ec476e3d Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 18 Feb 2023 22:39:58 -0300 Subject: [PATCH 31/34] mimeparser: Handle headers from the signed part of unencrypted signed message This makes DC compatible with "multipart/signed" messages thus allowing switching to them someday from the current "multipart/mixed" unencrypted message format. --- CHANGELOG.md | 1 + src/mimeparser.rs | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39bb421be..3203bd410 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Updated provider database. - Disable DKIM-Checks again #4076 - Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089 +- mimeparser: handle headers from the signed part of unencrypted signed message #4013 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 075fd192f..50b71b092 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -224,8 +224,32 @@ impl MimeMessage { // Parse hidden headers. let mimetype = mail.ctype.mimetype.parse::()?; + let (part, mimetype) = + if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "signed" { + if let Some(part) = mail.subparts.first() { + // We don't remove "subject" from `headers` because currently just signed + // messages are shown as unencrypted anyway. + + MimeMessage::merge_headers( + context, + &mut headers, + &mut recipients, + &mut from, + &mut list_post, + &mut chat_disposition_notification_to, + &part.headers, + ); + (part, part.ctype.mimetype.parse::()?) + } else { + // If it's a partially fetched message, there are no subparts. + (&mail, mimetype) + } + } else { + // Currently we do not sign unencrypted messages by default. + (&mail, mimetype) + }; if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" { - if let Some(part) = mail.subparts.first() { + if let Some(part) = part.subparts.first() { for field in &part.headers { let key = field.get_key().to_lowercase(); From d1923d68a5b02e297df1c12ce87237f0e409272d Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 18 Feb 2023 23:08:10 -0300 Subject: [PATCH 32/34] Add a config option to sign all messages with Autocrypt header (#3986) Although it does a little for security, it will help to protect from unwanted server-side modifications and bugs. And now we have a time to test "multipart/signed" messages compatibility with other MUAs. --- src/config.rs | 3 ++ src/context.rs | 6 +++ src/e2ee.rs | 13 +++++ src/mimefactory.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++- src/pgp.rs | 14 ++++++ 5 files changed, 154 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 8c1357a8e..fc950ba44 100644 --- a/src/config.rs +++ b/src/config.rs @@ -300,6 +300,9 @@ pub enum Config { /// See `crate::authres::update_authservid_candidates`. AuthservIdCandidates, + /// Make all outgoing messages with Autocrypt header "multipart/signed". + SignUnencrypted, + /// Let the core save all events to the database. /// This value is used internally to remember the MsgId of the logging xdc #[strum(props(default = "0"))] diff --git a/src/context.rs b/src/context.rs index eebb5541c..1d8dbe847 100644 --- a/src/context.rs +++ b/src/context.rs @@ -763,6 +763,12 @@ impl Context { .await? .unwrap_or_default(), ); + res.insert( + "sign_unencrypted", + self.get_config_int(Config::SignUnencrypted) + .await? + .to_string(), + ); res.insert( "debug_logging", diff --git a/src/e2ee.rs b/src/e2ee.rs index bbd8fd895..b870f89fe 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -124,6 +124,19 @@ impl EncryptHelper { Ok(ctext) } + + /// Signs the passed-in `mail` using the private key from `context`. + /// Returns the payload and the signature. + pub async fn sign( + self, + context: &Context, + mail: lettre_email::PartBuilder, + ) -> Result<(lettre_email::MimeMessage, String)> { + let sign_key = SignedSecretKey::load_self(context).await?; + let mime_message = mail.build(); + let signature = pgp::pk_calc_signature(mime_message.as_string().as_bytes(), &sign_key)?; + Ok((mime_message, signature)) + } } /// Ensures a private key exists for the configured user. diff --git a/src/mimefactory.rs b/src/mimefactory.rs index d98edabbe..530bfafe6 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -779,10 +779,36 @@ impl<'a> MimeFactory<'a> { }; // Store protected headers in the outer message. - headers + let message = headers .protected .into_iter() - .fold(message, |message, header| message.header(header)) + .fold(message, |message, header| message.header(header)); + + if self.should_skip_autocrypt() + || !context.get_config_bool(Config::SignUnencrypted).await? + { + message + } else { + let (payload, signature) = encrypt_helper.sign(context, message).await?; + PartBuilder::new() + .header(( + "Content-Type".to_string(), + "multipart/signed; protocol=\"application/pgp-signature\"".to_string(), + )) + .child(payload) + .child( + PartBuilder::new() + .content_type( + &"application/pgp-signature; name=\"signature.asc\"" + .parse::() + .unwrap(), + ) + .header(("Content-Description", "OpenPGP digital signature")) + .header(("Content-Disposition", "attachment; filename=\"signature\";")) + .body(signature) + .build(), + ) + } }; // Store the unprotected headers on the outer message. @@ -2140,6 +2166,96 @@ mod tests { Ok(()) } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_selfavatar_unencrypted_signed() { + // create chat with bob, set selfavatar + let t = TestContext::new_alice().await; + t.set_config(Config::SignUnencrypted, Some("1")) + .await + .unwrap(); + let chat = t.create_chat_with_contact("bob", "bob@example.org").await; + + let file = t.dir.path().join("avatar.png"); + let bytes = include_bytes!("../test-data/image/avatar64x64.png"); + tokio::fs::write(&file, bytes).await.unwrap(); + t.set_config(Config::Selfavatar, Some(file.to_str().unwrap())) + .await + .unwrap(); + + // send message to bob: that should get multipart/mixed because of the avatar moved to inner header; + // make sure, `Subject:` stays in the outer header (imf header) + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("this is the text!".to_string())); + + let sent_msg = t.send_msg(chat.id, &mut msg).await; + let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n"); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("multipart/signed").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 0); + assert_eq!(part.match_indices("Autocrypt:").count(), 1); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("multipart/mixed").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 1); + assert_eq!(part.match_indices("Autocrypt:").count(), 0); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("text/plain").count(), 1); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 0); + + let body = payload.next().unwrap(); + assert_eq!(body.match_indices("this is the text!").count(), 1); + + let bob = TestContext::new_bob().await; + bob.recv_msg(&sent_msg).await; + let alice_id = Contact::lookup_id_by_addr(&bob.ctx, "alice@example.org", Origin::Unknown) + .await + .unwrap() + .unwrap(); + let alice_contact = Contact::load_from_db(&bob.ctx, alice_id).await.unwrap(); + assert!(alice_contact + .get_profile_image(&bob.ctx) + .await + .unwrap() + .is_some()); + + // if another message is sent, that one must not contain the avatar + // and no artificial multipart/mixed nesting + let sent_msg = t.send_msg(chat.id, &mut msg).await; + let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n"); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("multipart/signed").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 0); + assert_eq!(part.match_indices("Autocrypt:").count(), 1); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("text/plain").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 1); + assert_eq!(part.match_indices("Autocrypt:").count(), 0); + assert_eq!(part.match_indices("multipart/mixed").count(), 0); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let body = payload.next().unwrap(); + assert_eq!(body.match_indices("this is the text!").count(), 1); + assert_eq!(body.match_indices("text/plain").count(), 0); + assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0); + assert_eq!(body.match_indices("Subject:").count(), 0); + + bob.recv_msg(&sent_msg).await; + let alice_contact = Contact::load_from_db(&bob.ctx, alice_id).await.unwrap(); + assert!(alice_contact + .get_profile_image(&bob.ctx) + .await + .unwrap() + .is_some()); + } + /// Test that removed member address does not go into the `To:` field. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_remove_member_bcc() -> Result<()> { diff --git a/src/pgp.rs b/src/pgp.rs index 6bc4bface..9d4ae46ea 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -262,6 +262,20 @@ pub async fn pk_encrypt( .await? } +/// Signs `plain` text using `private_key_for_signing`. +pub fn pk_calc_signature( + plain: &[u8], + private_key_for_signing: &SignedSecretKey, +) -> Result { + let msg = Message::new_literal_bytes("", plain).sign( + private_key_for_signing, + || "".into(), + Default::default(), + )?; + let signature = msg.into_signature().to_armored_string(None)?; + Ok(signature) +} + /// Decrypts the message with keys from the private key keyring. /// /// Receiver private keys are provided in From f8e86f4503311ed6fe8b35dadd0f2c466ae1c613 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 25 Feb 2023 13:04:06 +0100 Subject: [PATCH 33/34] make slowest test faster the slowest test was `test_modify_chat_disordered`; due to several sleep(), it lasts at least 11 seconds. however, many of the sleep() are not needed and were added just to get things done that time. this pr removes 7 superfluous sleeps, making the test finish in about 3 seconds, so that this test is no longer the bottleneck when running `cargo test` on fast machines. (this is not only useful to safe some seconds - but also to notice degradations as of https://github.com/deltachat/deltachat-core-rust/issues/4051#issuecomment-1436995166 ) --- src/chat.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 837b88164..803b4fc85 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -4090,7 +4090,6 @@ mod tests { send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?; add_contact_to_chat(&alice, alice_chat_id, bob_id).await?; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; let add1 = alice.pop_sent_msg().await; add_contact_to_chat(&alice, alice_chat_id, claire_id).await?; @@ -4109,29 +4108,18 @@ mod tests { remove_contact_from_chat(&alice, alice_chat_id, daisy_id).await?; let remove2 = alice.pop_sent_msg().await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2); // Bob receives the add and deletion messages out of order let bob = TestContext::new_bob().await; bob.recv_msg(&add1).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - bob.recv_msg(&add3).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - let bob_chat_id = bob.recv_msg(&add2).await.chat_id; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 4); bob.recv_msg(&remove2).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - bob.recv_msg(&remove1).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 2); Ok(()) From eace8c5feeba08593df3490c11bd9085bfee7877 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 15:01:53 +0000 Subject: [PATCH 34/34] Note that 1.108.0 re-enables SMTP pipelining --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3203bd410..ba24f2c2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ - Prefer TLS over STARTTLS during autoconfiguration #4021 - Use SOCKS5 configuration for HTTP requests #4017 - Show non-deltachat emails by default for new installations #4019 +- Re-enabled SMTP pipelining after disabling it in #4006 ### Fixes - Fix Securejoin for multiple devices on a joining side #3982