From 5dc20a5b98e45c8776c32452b4a1386b4c356f46 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 18:56:37 +0000 Subject: [PATCH 01/57] ci: link deltachat-rpc-server statically on Linux --- .github/workflows/deltachat-rpc-server.yml | 34 +++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 9f8f83982..5477afa4f 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -6,17 +6,37 @@ on: workflow_dispatch: jobs: - build_server: - name: Build deltachat-rpc-server + # Build a version statically linked against musl libc + # to avoid problems with glibc version incompatibility. + build_static_linux: + name: Build deltachat-rpc-server for Linux + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Setup rust target + run: rustup target add x86_64-unknown-linux-musl + + - name: Install musl-gcc + run: sudo apt install musl-tools + + - name: Build + env: + RUSTFLAGS: "-C link-arg=-s" + run: cargo build --release --target x86_64-unknown-linux-musl -p deltachat-rpc-server --features vendored + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server + path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server + if-no-files-found: error + + build_windows: + name: Build deltachat-rpc-server for Windows strategy: fail-fast: false matrix: include: - - os: ubuntu-22.04 - artifact: gnu-linux-x86_64 - path: deltachat-rpc-server - target: x86_64-unknown-linux-gnu - - os: windows-latest artifact: win32.exe path: deltachat-rpc-server.exe From cff4a9dce3f9d882587772d938177f865b94d0b0 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 19:50:42 +0000 Subject: [PATCH 02/57] ci: cross-compile deltachat-rpc-server for aarch64 Linux --- .github/workflows/deltachat-rpc-server.yml | 16 ++++++++++++++++ scripts/aarch64-unknown-linux-musl.sh | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100755 scripts/aarch64-unknown-linux-musl.sh diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 5477afa4f..e2fe23c97 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -31,6 +31,22 @@ jobs: path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server if-no-files-found: error + build_aarch64_linux: + name: Cross-compile deltachat-rpc-server for aarch64 Linux + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - name: Build + run: sh scripts/aarch64-unknown-linux-musl.sh + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server + path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server + if-no-files-found: error + build_windows: name: Build deltachat-rpc-server for Windows strategy: diff --git a/scripts/aarch64-unknown-linux-musl.sh b/scripts/aarch64-unknown-linux-musl.sh new file mode 100755 index 000000000..4e5ed6f09 --- /dev/null +++ b/scripts/aarch64-unknown-linux-musl.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -x +set -e + +# Download Zig +rm -fr zig-linux-x86_64-0.10.1 zig-linux-x86_64-0.10.1.tar.xz +wget https://ziglang.org/download/0.10.1/zig-linux-x86_64-0.10.1.tar.xz +tar xf zig-linux-x86_64-0.10.1.tar.xz +export PATH="$PATH:$PWD/zig-linux-x86_64-0.10.1" + +cargo install cargo-zigbuild + +rustup target add aarch64-unknown-linux-musl + +cargo zigbuild --release --target aarch64-unknown-linux-musl -p deltachat-rpc-server --features vendored From b0f77f7a332389d1903000659e49258a1074f8d4 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 20:13:27 +0000 Subject: [PATCH 03/57] ci: add arch prefix to deltachat-rpc-server binary names --- .github/workflows/deltachat-rpc-server.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index e2fe23c97..0cf842f16 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -27,7 +27,7 @@ jobs: - name: Upload binary uses: actions/upload-artifact@v3 with: - name: deltachat-rpc-server + name: deltachat-rpc-server-x86_64 path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server if-no-files-found: error @@ -43,7 +43,7 @@ jobs: - name: Upload binary uses: actions/upload-artifact@v3 with: - name: deltachat-rpc-server + name: deltachat-rpc-server-aarch64 path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server if-no-files-found: error From 880ee63fed36aaa8c34e3f9b2ae45cdbc420c13e Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 20:39:46 +0000 Subject: [PATCH 04/57] Document aarch64-unknown-linux-musl.sh --- scripts/README.md | 2 ++ scripts/aarch64-unknown-linux-musl.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/README.md b/scripts/README.md index 9523e257c..af6a6addd 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -20,6 +20,8 @@ and an own build machine. - `run_all.sh` builds Python wheels +- `aarch64-unknown-linux-musl.sh` cross-compiles static `deltachat-rpc-server` for aarch64 + ## Triggering runs on the build machine locally (fast!) There is experimental support for triggering a remote Python or Rust test run diff --git a/scripts/aarch64-unknown-linux-musl.sh b/scripts/aarch64-unknown-linux-musl.sh index 4e5ed6f09..272574fce 100755 --- a/scripts/aarch64-unknown-linux-musl.sh +++ b/scripts/aarch64-unknown-linux-musl.sh @@ -1,4 +1,6 @@ #!/bin/sh +# +# Build statically linked deltachat-rpc-server for aarch64-unknown-linux-musl. set -x set -e From ee81d61988b15a91b185a15096cd3fa1ff655987 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 01:31:09 +0000 Subject: [PATCH 05/57] Fix ruff warnings --- .../src/deltachat_rpc_client/chat.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index 7dac7d2f4..6a3e70971 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -182,19 +182,23 @@ class Chat: """Add contacts to this group.""" for cnt in contact: if isinstance(cnt, str): - cnt = (await self.account.create_contact(cnt)).id + contact_id = (await self.account.create_contact(cnt)).id elif not isinstance(cnt, int): - cnt = cnt.id - await self._rpc.add_contact_to_chat(self.account.id, self.id, cnt) + contact_id = cnt.id + else: + contact_id = cnt + await self._rpc.add_contact_to_chat(self.account.id, self.id, contact_id) async def remove_contact(self, *contact: Union[int, str, Contact]) -> None: """Remove members from this group.""" for cnt in contact: if isinstance(cnt, str): - cnt = (await self.account.create_contact(cnt)).id + contact_id = (await self.account.create_contact(cnt)).id elif not isinstance(cnt, int): - cnt = cnt.id - await self._rpc.remove_contact_from_chat(self.account.id, self.id, cnt) + contact_id = cnt.id + else: + contact_id = cnt + await self._rpc.remove_contact_from_chat(self.account.id, self.id, contact_id) async def get_contacts(self) -> List[Contact]: """Get the contacts belonging to this chat. @@ -230,9 +234,9 @@ class Chat: locations = [] contacts: Dict[int, Contact] = {} for loc in result: - loc = AttrDict(loc) - loc["chat"] = self - loc["contact"] = contacts.setdefault(loc.contact_id, Contact(self.account, loc.contact_id)) - loc["message"] = Message(self.account, loc.msg_id) - locations.append(loc) + location = AttrDict(loc) + location["chat"] = self + location["contact"] = contacts.setdefault(location.contact_id, Contact(self.account, location.contact_id)) + location["message"] = Message(self.account, location.msg_id) + locations.append(location) return locations From ade3d0d4eb9fc890604a8c7a01406e5eb92398a1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 12:55:43 +0000 Subject: [PATCH 06/57] Add more documentation comments --- src/accounts.rs | 3 +++ src/chat.rs | 3 +++ src/configure.rs | 4 ++++ src/imap.rs | 5 +++++ src/imap/select_folder.rs | 2 ++ src/qr.rs | 4 ++++ src/tools.rs | 2 ++ 7 files changed, 23 insertions(+) diff --git a/src/accounts.rs b/src/accounts.rs index 8ec3a34c8..9f013bfe6 100644 --- a/src/accounts.rs +++ b/src/accounts.rs @@ -476,10 +476,13 @@ impl Config { struct AccountConfig { /// Unique id. pub id: u32, + /// Root directory for all data for this account. /// /// The path is relative to the account manager directory. pub dir: std::path::PathBuf, + + /// Universally unique account identifier. pub uuid: Uuid, } diff --git a/src/chat.rs b/src/chat.rs index 522f55aa7..26e7a7db2 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1881,7 +1881,10 @@ pub(crate) async fn update_special_chat_names(context: &Context) -> Result<()> { /// [`Deref`]: std::ops::Deref #[derive(Debug)] pub(crate) struct ChatIdBlocked { + /// Chat ID. pub id: ChatId, + + /// Whether the chat is blocked, unblocked or a contact request. pub blocked: Blocked, } diff --git a/src/configure.rs b/src/configure.rs index 565169470..48b91a3f7 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -646,10 +646,14 @@ async fn try_smtp_one_param( } } +/// Failure to connect and login with email client configuration. #[derive(Debug, thiserror::Error)] #[error("Trying {config}…\nError: {msg}")] pub struct ConfigurationError { + /// Tried configuration description. config: String, + + /// Error message. msg: String, } diff --git a/src/imap.rs b/src/imap.rs index d7fcf1397..9889df69b 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -116,6 +116,8 @@ impl async_imap::Authenticator for OAuth2 { #[derive(Debug, Display, PartialEq, Eq, Clone, Copy)] pub enum FolderMeaning { Unknown, + + /// Spam folder. Spam, Inbox, Mvbox, @@ -149,8 +151,11 @@ impl FolderMeaning { #[derive(Debug)] struct ImapConfig { + /// Email address. pub addr: String, pub lp: ServerLoginParam, + + /// SOCKS 5 configuration. pub socks5_config: Option, pub strict_tls: bool, } diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index f209edff4..2d6dfa9a6 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -1,3 +1,5 @@ +//! # IMAP folder selection module. + use anyhow::Context as _; use super::session::Session as ImapSession; diff --git a/src/qr.rs b/src/qr.rs index 982777b4a..b9ffbbad5 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -473,11 +473,15 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result { #[derive(Debug, Deserialize)] struct CreateAccountSuccessResponse { + /// Email address. email: String, + + /// Password. password: String, } #[derive(Debug, Deserialize)] struct CreateAccountErrorResponse { + /// Reason for the failure to create account returned by the server. reason: String, } diff --git a/src/tools.rs b/src/tools.rs index 8703a124f..7e8d7e922 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -592,6 +592,8 @@ pub(crate) fn improve_single_line_input(input: &str) -> String { } pub(crate) trait IsNoneOrEmpty { + /// Returns true if an Option does not contain a string + /// or contains an empty string. fn is_none_or_empty(&self) -> bool; } impl IsNoneOrEmpty for Option From 2cc85e663775c97c9ab42c4c9e2ae6121f1c65d6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 14:48:25 +0000 Subject: [PATCH 07/57] Hide some constants from public crate interface --- src/constants.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 4c19abba6..9af48cc0b 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -190,11 +190,11 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4; pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL; /// How many existing messages shall be fetched after configuration. -pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100; +pub(crate) const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100; // max. width/height of an avatar -pub const BALANCED_AVATAR_SIZE: u32 = 256; -pub const WORSE_AVATAR_SIZE: u32 = 128; +pub(crate) const BALANCED_AVATAR_SIZE: u32 = 256; +pub(crate) const WORSE_AVATAR_SIZE: u32 = 128; // max. width/height of images pub const BALANCED_IMAGE_SIZE: u32 = 1280; From 0d7a3fe5528b5bc75110fb30397b412a8d63e5cb Mon Sep 17 00:00:00 2001 From: gerryfrancis <51095533+gerryfrancis@users.noreply.github.com> Date: Thu, 23 Feb 2023 19:05:20 +0100 Subject: [PATCH 08/57] Fix typo in CHANGELOG.md (#4086) Fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce2da427e..42b69dbc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -437,7 +437,7 @@ - Auto accept contact requests if `Config::Bot` is set for a client #3567 - Don't prepend the subject to chat messages in mailinglists - fix `set_core_version.py` script to also update version in `deltachat-jsonrpc/typescript/package.json` #3585 -- Reject webxcd-updates from contacts who are not group members #3568 +- Reject webxdc-updates from contacts who are not group members #3568 ## 1.93.0 From 76514178b6ea781fc0fdd68f57c6ac6ff0d0ab64 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 18:45:55 +0000 Subject: [PATCH 09/57] ci: do not run CI multiple times on the same branch Especially for PRs, if the branch is updated multiple times, only the last commit should be tested. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 449161d7f..fe1c4f8cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,11 @@ name: Rust CI +# Cancel previously started workflow runs +# when the branch is updated. +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + on: pull_request: push: From 059c673d00f547b969f442b57a587f88ea77dcd7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 18:50:44 +0000 Subject: [PATCH 10/57] ci: add workflow to the group Otherwise it cancels other workflows. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe1c4f8cc..392da23d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: Rust CI # Cancel previously started workflow runs # when the branch is updated. concurrency: - group: ${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: From b5187661ee46b8fe15953806e06e0e2e442826e4 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 01:13:40 +0000 Subject: [PATCH 11/57] ci: cancel running node.js tests when the branch is updated Do not waste CI time running the tests for previous commit. --- .github/workflows/node-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 7836f9b33..165880c68 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,4 +1,11 @@ name: "node.js tests" + +# Cancel previously started workflow runs +# when the branch is updated. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: push: From bc9e347a80646bdf630f8d57d602492a9331dc13 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 01:14:00 +0000 Subject: [PATCH 12/57] ci: there are no staging and trying branches --- .github/workflows/node-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 165880c68..62ab1d0fd 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -11,8 +11,6 @@ on: push: branches: - master - - staging - - trying jobs: tests: From e9668b3cfa1938fa09117c9d5756556a4ad44642 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 10:11:13 +0000 Subject: [PATCH 13/57] Cache INSERT statement in receive_imf It reduces iteration time by ~8% in message reception benchmarks ran by `cargo criterion --bench receive_emails`. --- src/receive_imf.rs | 91 +++++++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 2540d7442..b86fbed7c 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1149,7 +1149,10 @@ async fn add_parts( // also change `MsgId::trash()` and `delete_expired_messages()` let trash = chat_id.is_trash() || (is_location_kml && msg.is_empty()); - let row_id = context.sql.insert( + let row_id = context + .sql + .call(|conn| { + let mut stmt = conn.prepare_cached( r#" INSERT INTO msgs ( @@ -1179,47 +1182,51 @@ SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_in_reply_to=excluded.mime_in_reply_to, mime_references=excluded.mime_references, mime_modified=excluded.mime_modified, error=excluded.error, ephemeral_timer=excluded.ephemeral_timer, ephemeral_timestamp=excluded.ephemeral_timestamp, download_state=excluded.download_state, hop_info=excluded.hop_info -"#, - paramsv![ - replace_msg_id, - rfc724_mid, - if trash { DC_CHAT_ID_TRASH } else { chat_id }, - if trash { ContactId::UNDEFINED } else { from_id }, - if trash { ContactId::UNDEFINED } else { to_id }, - sort_timestamp, - sent_timestamp, - rcvd_timestamp, - typ, - state, - is_dc_message, - if trash { "" } else { msg }, - if trash { "" } else { &subject }, - // txt_raw might contain invalid utf8 - if trash { "" } else { &txt_raw }, - if trash { - "".to_string() - } else { - param.to_string() - }, - part.bytes as isize, - if (save_mime_headers || mime_modified) && !trash { - mime_headers.clone() - } else { - Vec::new() - }, - mime_in_reply_to, - mime_references, - mime_modified, - part.error.as_deref().unwrap_or_default(), - ephemeral_timer, - ephemeral_timestamp, - if is_partial_download.is_some() { - DownloadState::Available - } else { - DownloadState::Done - }, - mime_parser.hop_info - ]).await?; +"#)?; + stmt.execute(params![ + replace_msg_id, + rfc724_mid, + if trash { DC_CHAT_ID_TRASH } else { chat_id }, + if trash { ContactId::UNDEFINED } else { from_id }, + if trash { ContactId::UNDEFINED } else { to_id }, + sort_timestamp, + sent_timestamp, + rcvd_timestamp, + typ, + state, + is_dc_message, + if trash { "" } else { msg }, + if trash { "" } else { &subject }, + // txt_raw might contain invalid utf8 + if trash { "" } else { &txt_raw }, + if trash { + "".to_string() + } else { + param.to_string() + }, + part.bytes as isize, + if (save_mime_headers || mime_modified) && !trash { + mime_headers.clone() + } else { + Vec::new() + }, + mime_in_reply_to, + mime_references, + mime_modified, + part.error.as_deref().unwrap_or_default(), + ephemeral_timer, + ephemeral_timestamp, + if is_partial_download.is_some() { + DownloadState::Available + } else { + DownloadState::Done + }, + mime_parser.hop_info + ])?; + let row_id = conn.last_insert_rowid(); + Ok(row_id) + }) + .await?; // We only replace placeholder with a first part, // afterwards insert additional parts. From a82b09bfc23fb1afd5bfb7f8e6641d34adb89903 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 19:21:14 +0000 Subject: [PATCH 14/57] Move TLS support to net::tls module --- src/imap/client.rs | 16 +++++---------- src/login_param.rs | 33 ------------------------------ src/net.rs | 1 + src/net/tls.rs | 50 ++++++++++++++++++++++++++++++++++++++++++++++ src/smtp.rs | 17 ++++++---------- 5 files changed, 62 insertions(+), 55 deletions(-) create mode 100644 src/net/tls.rs diff --git a/src/imap/client.rs b/src/imap/client.rs index 5e8d0895d..265a9287a 100644 --- a/src/imap/client.rs +++ b/src/imap/client.rs @@ -11,9 +11,9 @@ use tokio::io::BufWriter; use super::capabilities::Capabilities; use super::session::Session; use crate::context::Context; -use crate::login_param::build_tls; use crate::net::connect_tcp; use crate::net::session::SessionStream; +use crate::net::tls::wrap_tls; use crate::socks::Socks5Config; /// IMAP write and read timeout. @@ -95,8 +95,7 @@ impl Client { strict_tls: bool, ) -> Result { let tcp_stream = connect_tcp(context, hostname, port, IMAP_TIMEOUT, strict_tls).await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(hostname, tcp_stream).await?; + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -142,9 +141,7 @@ impl Client { .context("STARTTLS command failed")?; let tcp_stream = client.into_inner(); - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, tcp_stream) + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream) .await .context("STARTTLS upgrade failed")?; @@ -165,8 +162,7 @@ impl Client { let socks5_stream = socks5_config .connect(context, domain, port, IMAP_TIMEOUT, strict_tls) .await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(domain, socks5_stream).await?; + let tls_stream = wrap_tls(strict_tls, domain, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let mut client = ImapClient::new(session_stream); @@ -221,9 +217,7 @@ impl Client { .context("STARTTLS command failed")?; let socks5_stream = client.into_inner(); - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, socks5_stream) + let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream) .await .context("STARTTLS upgrade failed")?; let buffered_stream = BufWriter::new(tls_stream); diff --git a/src/login_param.rs b/src/login_param.rs index ee674c1fc..e01e43a0f 100644 --- a/src/login_param.rs +++ b/src/login_param.rs @@ -3,8 +3,6 @@ use std::fmt; use anyhow::{ensure, Result}; -use async_native_tls::Certificate; -use once_cell::sync::Lazy; use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2}; use crate::provider::{get_provider_by_id, Provider}; @@ -306,28 +304,6 @@ fn unset_empty(s: &str) -> &str { } } -// this certificate is missing on older android devices (eg. lg with android6 from 2017) -// certificate downloaded from https://letsencrypt.org/certificates/ -static LETSENCRYPT_ROOT: Lazy = Lazy::new(|| { - Certificate::from_der(include_bytes!( - "../assets/root-certificates/letsencrypt/isrgrootx1.der" - )) - .unwrap() -}); - -pub fn build_tls(strict_tls: bool) -> async_native_tls::TlsConnector { - let tls_builder = - async_native_tls::TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone()); - - if strict_tls { - tls_builder - } else { - tls_builder - .danger_accept_invalid_hostnames(true) - .danger_accept_invalid_certs(true) - } -} - #[cfg(test)] mod tests { use super::*; @@ -378,13 +354,4 @@ mod tests { assert_eq!(param, loaded); Ok(()) } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_build_tls() -> Result<()> { - // we are using some additional root certificates. - // make sure, they do not break construction of TlsConnector - let _ = build_tls(true); - let _ = build_tls(false); - Ok(()) - } } diff --git a/src/net.rs b/src/net.rs index 2fb9cb6b6..2d36e90aa 100644 --- a/src/net.rs +++ b/src/net.rs @@ -13,6 +13,7 @@ use crate::context::Context; use crate::tools::time; pub(crate) mod session; +pub(crate) mod tls; async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result { let tcp_stream = timeout(timeout_val, TcpStream::connect(addr)) diff --git a/src/net/tls.rs b/src/net/tls.rs new file mode 100644 index 000000000..980504416 --- /dev/null +++ b/src/net/tls.rs @@ -0,0 +1,50 @@ +//! TLS support. + +use anyhow::Result; +use async_native_tls::{Certificate, TlsConnector, TlsStream}; +use once_cell::sync::Lazy; +use tokio::io::{AsyncRead, AsyncWrite}; + +// this certificate is missing on older android devices (eg. lg with android6 from 2017) +// certificate downloaded from https://letsencrypt.org/certificates/ +static LETSENCRYPT_ROOT: Lazy = Lazy::new(|| { + Certificate::from_der(include_bytes!( + "../../assets/root-certificates/letsencrypt/isrgrootx1.der" + )) + .unwrap() +}); + +pub fn build_tls(strict_tls: bool) -> TlsConnector { + let tls_builder = TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone()); + + if strict_tls { + tls_builder + } else { + tls_builder + .danger_accept_invalid_hostnames(true) + .danger_accept_invalid_certs(true) + } +} + +pub async fn wrap_tls( + strict_tls: bool, + hostname: &str, + stream: T, +) -> Result> { + let tls = build_tls(strict_tls); + let tls_stream = tls.connect(hostname, stream).await?; + Ok(tls_stream) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_tls() { + // we are using some additional root certificates. + // make sure, they do not break construction of TlsConnector + let _ = build_tls(true); + let _ = build_tls(false); + } +} diff --git a/src/smtp.rs b/src/smtp.rs index cda3aff05..ca0e69ff8 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -13,12 +13,13 @@ use tokio::task; use crate::config::Config; use crate::contact::{Contact, ContactId}; use crate::events::EventType; -use crate::login_param::{build_tls, CertificateChecks, LoginParam, ServerLoginParam}; +use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam}; use crate::message::Message; use crate::message::{self, MsgId}; use crate::mimefactory::MimeFactory; use crate::net::connect_tcp; use crate::net::session::SessionStream; +use crate::net::tls::wrap_tls; use crate::oauth2::get_oauth2_access_token; use crate::provider::Socket; use crate::socks::Socks5Config; @@ -119,8 +120,7 @@ impl Smtp { let socks5_stream = socks5_config .connect(context, hostname, port, SMTP_TIMEOUT, strict_tls) .await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(hostname, socks5_stream).await?; + let tls_stream = wrap_tls(strict_tls, hostname, socks5_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let client = smtp::SmtpClient::new().smtp_utf8(true); @@ -144,9 +144,7 @@ impl Smtp { let client = smtp::SmtpClient::new().smtp_utf8(true); let transport = SmtpTransport::new(client, socks5_stream).await?; let tcp_stream = transport.starttls().await?; - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, tcp_stream) + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream) .await .context("STARTTLS upgrade failed")?; let buffered_stream = BufWriter::new(tls_stream); @@ -181,8 +179,7 @@ impl Smtp { strict_tls: bool, ) -> Result>> { let tcp_stream = connect_tcp(context, hostname, port, SMTP_TIMEOUT, false).await?; - let tls = build_tls(strict_tls); - let tls_stream = tls.connect(hostname, tcp_stream).await?; + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream).await?; let buffered_stream = BufWriter::new(tls_stream); let session_stream: Box = Box::new(buffered_stream); let client = smtp::SmtpClient::new().smtp_utf8(true); @@ -203,9 +200,7 @@ impl Smtp { let client = smtp::SmtpClient::new().smtp_utf8(true); let transport = SmtpTransport::new(client, tcp_stream).await?; let tcp_stream = transport.starttls().await?; - let tls = build_tls(strict_tls); - let tls_stream = tls - .connect(hostname, tcp_stream) + let tls_stream = wrap_tls(strict_tls, hostname, tcp_stream) .await .context("STARTTLS upgrade failed")?; let buffered_stream = BufWriter::new(tls_stream); From d0a7e5f1b79ad5c0152e2ee2d1d11100a2838fc1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 14:32:22 +0000 Subject: [PATCH 15/57] Update timestamps in parameters with transactions This avoids accidentally reverting the changes to the `param` column done by another thread. --- CHANGELOG.md | 1 + src/update_helper.rs | 55 +++++++++++++++++++++++++++++++++----------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42b69dbc2..0213cb803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted. Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972 - Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 +- Update timestamps in `param` columns with transactions. #4083 ### API-Changes diff --git a/src/update_helper.rs b/src/update_helper.rs index 9136b23df..ed140dfdd 100644 --- a/src/update_helper.rs +++ b/src/update_helper.rs @@ -2,8 +2,8 @@ use anyhow::Result; -use crate::chat::{Chat, ChatId}; -use crate::contact::{Contact, ContactId}; +use crate::chat::ChatId; +use crate::contact::ContactId; use crate::context::Context; use crate::param::{Param, Params}; @@ -17,12 +17,26 @@ impl Context { scope: Param, new_timestamp: i64, ) -> Result { - let mut contact = Contact::load_from_db(self, contact_id).await?; - if contact.param.update_timestamp(scope, new_timestamp)? { - contact.update_param(self).await?; - return Ok(true); - } - Ok(false) + self.sql + .transaction(|transaction| { + let mut param: Params = transaction.query_row( + "SELECT param FROM contacts WHERE id=?", + [contact_id], + |row| { + let param: String = row.get(0)?; + Ok(param.parse().unwrap_or_default()) + }, + )?; + let update = param.update_timestamp(scope, new_timestamp)?; + if update { + transaction.execute( + "UPDATE contacts SET param=? WHERE id=?", + params![param.to_string(), contact_id], + )?; + } + Ok(update) + }) + .await } } @@ -35,12 +49,24 @@ impl ChatId { scope: Param, new_timestamp: i64, ) -> Result { - let mut chat = Chat::load_from_db(context, *self).await?; - if chat.param.update_timestamp(scope, new_timestamp)? { - chat.update_param(context).await?; - return Ok(true); - } - Ok(false) + context + .sql + .transaction(|transaction| { + let mut param: Params = + transaction.query_row("SELECT param FROM chats WHERE id=?", [self], |row| { + let param: String = row.get(0)?; + Ok(param.parse().unwrap_or_default()) + })?; + let update = param.update_timestamp(scope, new_timestamp)?; + if update { + transaction.execute( + "UPDATE chats SET param=? WHERE id=?", + params![param.to_string(), self], + )?; + } + Ok(update) + }) + .await } } @@ -60,6 +86,7 @@ impl Params { #[cfg(test)] mod tests { use super::*; + use crate::chat::Chat; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; use crate::tools::time; From 7af1a194915b385b3c434e9ff2596ccffee81fe6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 23 Feb 2023 18:43:10 +0000 Subject: [PATCH 16/57] Do not build more than one deltachat-rpc-server at a time --- .github/workflows/deltachat-rpc-server.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 0cf842f16..72c3dc919 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -2,6 +2,10 @@ name: Build deltachat-rpc-server binaries +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: workflow_dispatch: From a2e088b415422bbddd6f09ec77abd5c238b2dd06 Mon Sep 17 00:00:00 2001 From: adbenitez Date: Wed, 22 Feb 2023 20:52:29 -0500 Subject: [PATCH 17/57] Compile deltachat-rpc-server for Android Co-Authored-By: link2xt --- .github/workflows/deltachat-rpc-server.yml | 44 ++++++++++++++++++++++ scripts/android-rpc-server.sh | 44 ++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100755 scripts/android-rpc-server.sh diff --git a/.github/workflows/deltachat-rpc-server.yml b/.github/workflows/deltachat-rpc-server.yml index 72c3dc919..f073474f6 100644 --- a/.github/workflows/deltachat-rpc-server.yml +++ b/.github/workflows/deltachat-rpc-server.yml @@ -51,6 +51,50 @@ jobs: path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server if-no-files-found: error + build_android: + name: Cross-compile deltachat-rpc-server for Android (armeabi-v7a, arm64-v8a, x86 and x86_64) + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + + - uses: nttld/setup-ndk@v1 + id: setup-ndk + with: + ndk-version: r21d + + - name: Build + env: + ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }} + run: sh scripts/android-rpc-server.sh + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-armv7 + path: target/armv7-linux-androideabi/release/deltachat-rpc-server + if-no-files-found: error + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-aarch64 + path: target/aarch64-linux-android/release/deltachat-rpc-server + if-no-files-found: error + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-i686 + path: target/i686-linux-android/release/deltachat-rpc-server + if-no-files-found: error + + - name: Upload binary + uses: actions/upload-artifact@v3 + with: + name: deltachat-rpc-server-android-x86_64 + path: target/x86_64-linux-android/release/deltachat-rpc-server + if-no-files-found: error + build_windows: name: Build deltachat-rpc-server for Windows strategy: diff --git a/scripts/android-rpc-server.sh b/scripts/android-rpc-server.sh new file mode 100755 index 000000000..94963cbcd --- /dev/null +++ b/scripts/android-rpc-server.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Build deltachat-rpc-server for Android. + +set -e + +test -n "$ANDROID_NDK_ROOT" || exit 1 + +RUSTUP_TOOLCHAIN="1.64.0" +rustup install "$RUSTUP_TOOLCHAIN" +rustup target add armv7-linux-androideabi aarch64-linux-android i686-linux-android x86_64-linux-android --toolchain "$RUSTUP_TOOLCHAIN" + +KERNEL="$(uname -s | tr '[:upper:]' '[:lower:]')" +ARCH="$(uname -m)" +NDK_HOST_TAG="$KERNEL-$ARCH" +TOOLCHAIN="$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/$NDK_HOST_TAG" +export PATH="$PATH:$TOOLCHAIN/bin/" + +PACKAGE="deltachat-rpc-server" + +export CARGO_PROFILE_RELEASE_LTO=on + +CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER="$TOOLCHAIN/bin/armv7a-linux-androideabi16-clang" \ + CFLAGS=-D__ANDROID_API__=16 \ + TARGET_CC=armv7a-linux-androideabi16-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target armv7-linux-androideabi -p $PACKAGE + +CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/aarch64-linux-android21-clang" \ + CFLAGS=-D__ANDROID_API__=21 \ + TARGET_CC=aarch64-linux-android21-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target aarch64-linux-android -p $PACKAGE + +CARGO_TARGET_I686_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/i686-linux-android16-clang" \ + CFLAGS=-D__ANDROID_API__=16 \ + TARGET_CC=i686-linux-android16-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target i686-linux-android -p $PACKAGE + +CARGO_TARGET_X86_64_LINUX_ANDROID_LINKER="$TOOLCHAIN/bin/x86_64-linux-android21-clang" \ + CFLAGS=-D__ANDROID_API__=21 \ + TARGET_CC=x86_64-linux-android21-clang \ + TARGET_AR=llvm-ar \ + cargo "+$RUSTUP_TOOLCHAIN" rustc --release --target x86_64-linux-android -p $PACKAGE From 08c9deee042063e6fef3e1afb182f172787fa8d7 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 24 Feb 2023 15:37:03 +0100 Subject: [PATCH 18/57] add test for Context::update_contacts_timestamp(), esp. the condition and the update was not covered before --- src/receive_imf/tests.rs | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index 50aa285ce..2fec63df4 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -2133,6 +2133,76 @@ Original signature updated", Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_ignore_old_status_updates() -> Result<()> { + let t = TestContext::new_alice().await; + let bob_id = Contact::add_or_lookup( + &t, + "", + ContactAddress::new("bob@example.net")?, + Origin::AddressBook, + ) + .await? + .0; + + receive_imf( + &t, + b"From: Bob +To: Alice +Message-ID: <2@example.org> +Date: Wed, 22 Feb 2023 20:00:00 +0000 + +body + +-- +sig wednesday", + false, + ) + .await?; + let chat_id = t.get_last_msg().await.chat_id; + let bob = Contact::load_from_db(&t, bob_id).await?; + assert_eq!(bob.get_status(), "sig wednesday"); + assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 1); + + receive_imf( + &t, + b"From: Bob +To: Alice +Message-ID: <1@example.org> +Date: Tue, 21 Feb 2023 20:00:00 +0000 + +body + +-- +sig tuesday", + false, + ) + .await?; + let bob = Contact::load_from_db(&t, bob_id).await?; + assert_eq!(bob.get_status(), "sig wednesday"); + assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 2); + + receive_imf( + &t, + b"From: Bob +To: Alice +Message-ID: <3@example.org> +Date: Thu, 23 Feb 2023 20:00:00 +0000 + +body + +-- +sig thursday", + false, + ) + .await?; + let bob = Contact::load_from_db(&t, bob_id).await?; + assert_eq!(bob.get_status(), "sig thursday"); + assert_eq!(get_chat_msgs(&t, chat_id).await?.len(), 3); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_chat_assignment_private_classical_reply() { for outgoing_is_classical in &[true, false] { From e3eb35e160188af96c2c9b305e2ec48a3dac8938 Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Mon, 20 Feb 2023 22:18:16 +0100 Subject: [PATCH 19/57] deltachat-jsonrpc: bump yerpc version in ts client --- deltachat-jsonrpc/typescript/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index aded96fcc..41da2e146 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -3,7 +3,7 @@ "dependencies": { "@deltachat/tiny-emitter": "3.0.0", "isomorphic-ws": "^4.0.1", - "yerpc": "^0.3.3" + "yerpc": "^0.4.2" }, "devDependencies": { "@types/chai": "^4.2.21", From 75670134d879399b31e9345d79f16f944dfbfebc Mon Sep 17 00:00:00 2001 From: "Franz Heinzmann (Frando)" Date: Wed, 22 Feb 2023 19:16:15 +0100 Subject: [PATCH 20/57] chore(jsonrpc): bump yerpc version --- deltachat-jsonrpc/typescript/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 41da2e146..3015eda6f 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -3,7 +3,7 @@ "dependencies": { "@deltachat/tiny-emitter": "3.0.0", "isomorphic-ws": "^4.0.1", - "yerpc": "^0.4.2" + "yerpc": "^0.4.3" }, "devDependencies": { "@types/chai": "^4.2.21", From c8ce4ce5aaac33cb4ab8acae4ead3e671b317b44 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 13:32:05 +0000 Subject: [PATCH 21/57] Switch to "vX.Y.Z" tagging scheme Previously we used tags like "1.109.0" and "py-1.109.0". Since Python bindings are always released at the same time as the core, it is easier to use a single tag for them. "v" prefix is added to make matching by "v*" easier in CI and scripts. --- CHANGELOG.md | 1 + python/pyproject.toml | 4 ++-- scripts/concourse/docs_wheels.yml | 2 +- scripts/set_core_version.py | 6 ++---- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0213cb803..890baf64d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055 - Updated provider database. - Disable DKIM-Checks again #4076 +- Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/python/pyproject.toml b/python/pyproject.toml index 227fb73d2..ce8dfaf5b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -44,8 +44,8 @@ deltachat = [ [tool.setuptools_scm] root = ".." -tag_regex = '^(?Ppy-)?(?P[^\+]+)(?P.*)?$' -git_describe_command = "git describe --dirty --tags --long --match py-*.*" +tag_regex = '^(?Pv)?(?P[^\+]+)(?P.*)?$' +git_describe_command = "git describe --dirty --tags --long --match v*.*" [tool.black] line-length = 120 diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml index bd2074d4b..efd57c561 100644 --- a/scripts/concourse/docs_wheels.yml +++ b/scripts/concourse/docs_wheels.yml @@ -12,7 +12,7 @@ resources: source: branch: master uri: https://github.com/deltachat/deltachat-core-rust.git - tag_filter: "py-*" + tag_filter: "v*" jobs: - name: doxygen diff --git a/scripts/set_core_version.py b/scripts/set_core_version.py index f3f29bbac..84444f8ef 100755 --- a/scripts/set_core_version.py +++ b/scripts/set_core_version.py @@ -115,10 +115,8 @@ def main(): print("after commit, on master make sure to: ") print("") - print(f" git tag -a {newversion}") - print(f" git push origin {newversion}") - print(f" git tag -a py-{newversion}") - print(f" git push origin py-{newversion}") + print(f" git tag -a v{newversion}") + print(f" git push origin v{newversion}") print("") From 45817fcacdd560db95c27e94c79f0d37a9eac535 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 16:50:19 +0000 Subject: [PATCH 22/57] Release 1.110.0 --- CHANGELOG.md | 2 +- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 8 ++++---- deltachat-repl/Cargo.toml | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- package.json | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 890baf64d..9fd88c9c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 1.110.0 ### Changes - use transaction in `Contact::add_or_lookup()` #4059 diff --git a/Cargo.lock b/Cargo.lock index 7e0219a99..1a6765405 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,7 +834,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" [[package]] name = "deltachat" -version = "1.109.0" +version = "1.110.0" dependencies = [ "ansi_term", "anyhow", @@ -905,7 +905,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.109.0" +version = "1.110.0" dependencies = [ "anyhow", "async-channel", @@ -927,7 +927,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.109.0" +version = "1.110.0" dependencies = [ "ansi_term", "anyhow", @@ -942,7 +942,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.109.0" +version = "1.110.0" dependencies = [ "anyhow", "deltachat-jsonrpc", @@ -965,7 +965,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.109.0" +version = "1.110.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index 867a10b72..4378fae09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.109.0" +version = "1.110.0" edition = "2021" license = "MPL-2.0" rust-version = "1.63" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 2bbec1c1f..6043a6752 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.109.0" +version = "1.110.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index e2958a4b2..7b9a26719 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.109.0" +version = "1.110.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 3015eda6f..795f5eaf7 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -26,8 +26,8 @@ }, "exports": { ".": { - "require": "./dist/deltachat.cjs", - "import": "./dist/deltachat.js" + "import": "./dist/deltachat.js", + "require": "./dist/deltachat.cjs" } }, "license": "MPL-2.0", @@ -36,8 +36,8 @@ "scripts": { "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs", "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js", - "build:tsc": "tsc", "build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs", + "build:tsc": "tsc", "docs": "typedoc --out docs deltachat.ts", "example": "run-s build example:build example:start", "example:build": "esbuild --bundle dist/example/example.js --outfile=dist/example.bundle.js", @@ -55,5 +55,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.109.0" + "version": "1.110.0" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index 59e3a3774..bab15c6d7 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.109.0" +version = "1.110.0" edition = "2021" [dependencies] diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index a332e21b5..aa984ffc3 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.109.0" +version = "1.110.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 9045c36e2..d6e2b3344 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.109.0" + "version": "1.110.0" } \ No newline at end of file From aeadbb35f3cecb49f065b6701c9ebfe9905da817 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 17:46:35 +0000 Subject: [PATCH 23/57] Remove empty API-Changes section --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd88c9c6..291a5f67a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,6 @@ - Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 - Update timestamps in `param` columns with transactions. #4083 -### API-Changes - ## 1.109.0 From b6336ce7e92c8de2a18f2ae3755103e8b15b2705 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 17:46:45 +0000 Subject: [PATCH 24/57] Add Unreleased changelog section --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 291a5f67a..3adf1b5a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Changes + +### Fixes + +### API-Changes + + ## 1.110.0 ### Changes From 064f806d905da44ce09996d28394995dbc698efb Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 22 Feb 2023 16:48:45 +0000 Subject: [PATCH 25/57] Remove UpdateRecentQuota job --- src/context.rs | 5 +++++ src/job.rs | 48 +++++++++++------------------------------------- src/quota.rs | 25 +++++++++++++------------ src/scheduler.rs | 8 ++++++++ src/sql.rs | 2 +- 5 files changed, 38 insertions(+), 50 deletions(-) diff --git a/src/context.rs b/src/context.rs index 04687ad0c..21276f47e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; @@ -206,6 +207,9 @@ pub struct InnerContext { /// Set to `None` if quota was never tried to load. pub(crate) quota: RwLock>, + /// Set to true if quota update is requested. + pub(crate) quota_update_request: AtomicBool, + /// Server ID response if ID capability is supported /// and the server returned non-NIL on the inbox connection. /// @@ -365,6 +369,7 @@ impl Context { scheduler: RwLock::new(None), ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow to send 6 messages immediately, no more than once every 10 seconds. quota: RwLock::new(None), + quota_update_request: AtomicBool::new(false), server_id: RwLock::new(None), creation_time: std::time::SystemTime::now(), last_full_folder_scan: Mutex::new(None), diff --git a/src/job.rs b/src/job.rs index 3279843a9..cd54262ea 100644 --- a/src/job.rs +++ b/src/job.rs @@ -58,9 +58,6 @@ macro_rules! job_try { )] #[repr(u32)] pub enum Action { - // this is user initiated so it should have a fairly high priority - UpdateRecentQuota = 140, - // This job will download partially downloaded messages completely // and is added when download_full() is called. // Most messages are downloaded automatically on fetch @@ -202,17 +199,6 @@ pub async fn kill_action(context: &Context, action: Action) -> Result<()> { Ok(()) } -pub async fn action_exists(context: &Context, action: Action) -> Result { - let exists = context - .sql - .exists( - "SELECT COUNT(*) FROM jobs WHERE action=?;", - paramsv![action], - ) - .await?; - Ok(exists) -} - pub(crate) enum Connection<'a> { Inbox(&'a mut Imap), } @@ -240,7 +226,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ if tries < JOB_RETRIES { info!(context, "increase job {} tries to {}", job, tries); job.tries = tries; - let time_offset = get_backoff_time_offset(tries, job.action); + let time_offset = get_backoff_time_offset(tries); job.desired_timestamp = time() + time_offset; info!( context, @@ -289,10 +275,6 @@ async fn perform_job_action( let try_res = match job.action { Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await, - Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await { - Ok(status) => status, - Err(err) => Status::Finished(Err(err)), - }, Action::DownloadMsg => job.download_msg(context, connection.inbox()).await, }; @@ -301,24 +283,16 @@ async fn perform_job_action( try_res } -fn get_backoff_time_offset(tries: u32, action: Action) -> i64 { - match action { - // Just try every 10s to update the quota - // If all retries are exhausted, a new job will be created when the quota information is needed - Action::UpdateRecentQuota => 10, - - _ => { - // Exponential backoff - let n = 2_i32.pow(tries - 1) * 60; - let mut rng = thread_rng(); - let r: i32 = rng.gen(); - let mut seconds = r % (n + 1); - if seconds < 1 { - seconds = 1; - } - i64::from(seconds) - } +fn get_backoff_time_offset(tries: u32) -> i64 { + // Exponential backoff + let n = 2_i32.pow(tries - 1) * 60; + let mut rng = thread_rng(); + let r: i32 = rng.gen(); + let mut seconds = r % (n + 1); + if seconds < 1 { + seconds = 1; } + i64::from(seconds) } pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { @@ -339,7 +313,7 @@ pub async fn add(context: &Context, job: Job) -> Result<()> { if delay_seconds == 0 { match action { - Action::ResyncFolders | Action::UpdateRecentQuota | Action::DownloadMsg => { + Action::ResyncFolders | Action::DownloadMsg => { info!(context, "interrupt: imap"); context.interrupt_inbox(InterruptInfo::new(false)).await; } diff --git a/src/quota.rs b/src/quota.rs index 43cdd3465..90eab5222 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -1,6 +1,7 @@ //! # Support for IMAP QUOTA extension. use std::collections::BTreeMap; +use std::sync::atomic::Ordering; use anyhow::{anyhow, Context as _, Result}; use async_imap::types::{Quota, QuotaResource}; @@ -11,11 +12,10 @@ use crate::context::Context; use crate::imap::scan_folders::get_watched_folders; use crate::imap::session::Session as ImapSession; use crate::imap::Imap; -use crate::job::{Action, Status}; use crate::message::{Message, Viewtype}; -use crate::param::Params; +use crate::scheduler::InterruptInfo; use crate::tools::time; -use crate::{job, stock_str, EventType}; +use crate::{stock_str, EventType}; /// warn about a nearly full mailbox after this usage percentage is reached. /// quota icon is "yellow". @@ -112,12 +112,10 @@ pub fn needs_quota_warning(curr_percentage: u64, warned_at_percentage: u64) -> b impl Context { // Adds a job to update `quota.recent` pub(crate) async fn schedule_quota_update(&self) -> Result<()> { - if !job::action_exists(self, Action::UpdateRecentQuota).await? { - job::add( - self, - job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0), - ) - .await?; + let requested = self.quota_update_request.swap(true, Ordering::Relaxed); + if !requested { + // Quota update was not requested before. + self.interrupt_inbox(InterruptInfo::new(false)).await; } Ok(()) } @@ -132,10 +130,10 @@ impl Context { /// and new space is allocated as needed. /// /// Called in response to `Action::UpdateRecentQuota`. - pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result { + pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<()> { if let Err(err) = imap.prepare(self).await { warn!(self, "could not connect: {:#}", err); - return Ok(Status::RetryNow); + return Ok(()); } let session = imap.session.as_mut().context("no session")?; @@ -166,13 +164,16 @@ impl Context { } } + // Clear the request to update quota. + self.quota_update_request.store(false, Ordering::Relaxed); + *self.quota.write().await = Some(QuotaInfo { recent: quota, modified: time(), }); self.emit_event(EventType::ConnectivityChanged); - Ok(Status::Finished(Ok(()))) + Ok(()) } } diff --git a/src/scheduler.rs b/src/scheduler.rs index df9312c4e..4ce26c45d 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -1,4 +1,5 @@ use std::iter::{self, once}; +use std::sync::atomic::Ordering; use anyhow::{bail, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; @@ -128,6 +129,13 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne info = Default::default(); } None => { + let requested = ctx.quota_update_request.swap(false, Ordering::Relaxed); + if requested { + if let Err(err) = ctx.update_recent_quota(&mut connection).await { + warn!(ctx, "Failed to update quota: {:#}.", err); + } + } + maybe_add_time_based_warnings(&ctx).await; match ctx.get_config_i64(Config::LastHousekeeping).await { diff --git a/src/sql.rs b/src/sql.rs index 163d4e989..07884f875 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -983,7 +983,7 @@ mod tests { assert_eq!(avatar_bytes, &tokio::fs::read(&a).await.unwrap()[..]); t.sql.close().await; - housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed + housekeeping(&t).await.unwrap(); // housekeeping should emit warnings but not fail t.sql.open(&t, "".to_string()).await.unwrap(); let a = t.get_config(Config::Selfavatar).await.unwrap().unwrap(); From c02c5ffe2c45fa58a7a86c0352599744f6736f13 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 22:30:48 +0000 Subject: [PATCH 26/57] Format docs_wheels.yml with `prettier` --- scripts/concourse/docs_wheels.yml | 694 +++++++++++++++--------------- 1 file changed, 347 insertions(+), 347 deletions(-) diff --git a/scripts/concourse/docs_wheels.yml b/scripts/concourse/docs_wheels.yml index efd57c561..3f461ad20 100644 --- a/scripts/concourse/docs_wheels.yml +++ b/scripts/concourse/docs_wheels.yml @@ -1,370 +1,370 @@ resources: -- name: deltachat-core-rust - type: git - icon: github - source: - branch: master - uri: https://github.com/deltachat/deltachat-core-rust.git + - name: deltachat-core-rust + type: git + icon: github + source: + branch: master + uri: https://github.com/deltachat/deltachat-core-rust.git -- name: deltachat-core-rust-release - type: git - icon: github - source: - branch: master - uri: https://github.com/deltachat/deltachat-core-rust.git - tag_filter: "v*" + - name: deltachat-core-rust-release + type: git + icon: github + source: + branch: master + uri: https://github.com/deltachat/deltachat-core-rust.git + tag_filter: "v*" jobs: -- name: doxygen - plan: - - get: deltachat-core-rust - trigger: true + - name: doxygen + plan: + - get: deltachat-core-rust + trigger: true - # Build Doxygen documentation - - task: build-doxygen - config: - inputs: - - name: deltachat-core-rust - outputs: - - name: c-docs - image_resource: - source: - repository: alpine - type: registry-image - platform: linux - run: - path: sh - args: - - -ec - - | - apk add --no-cache doxygen git - cd deltachat-core-rust - scripts/run-doxygen.sh - cd .. - cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/ + # Build Doxygen documentation + - task: build-doxygen + config: + inputs: + - name: deltachat-core-rust + outputs: + - name: c-docs + image_resource: + source: + repository: alpine + type: registry-image + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache doxygen git + cd deltachat-core-rust + scripts/run-doxygen.sh + cd .. + cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/ - - task: upload-c-docs - config: - inputs: - - name: c-docs - image_resource: - type: registry-image - source: - repository: alpine - platform: linux - run: - path: sh - args: - - -ec - - | - apk add --no-cache rsync openssh-client - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master + - task: upload-c-docs + config: + inputs: + - name: c-docs + image_resource: + type: registry-image + source: + repository: alpine + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache rsync openssh-client + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master -- name: python-x86_64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-x86_64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_x86_64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-docs - path: ./python/doc/_build/ - # Binary wheels - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-docs + path: ./python/doc/_build/ + # Binary wheels + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload python docs to py.delta.chat - - task: upload-py-docs - config: - inputs: - - name: py-docs - image_resource: - type: registry-image - source: - repository: alpine - platform: linux - run: - path: sh - args: - - -ec - - | - apk add --no-cache rsync openssh-client - mkdir -p ~/.ssh - chmod 700 ~/.ssh - echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master + # Upload python docs to py.delta.chat + - task: upload-py-docs + config: + inputs: + - name: py-docs + image_resource: + type: registry-image + source: + repository: alpine + platform: linux + run: + path: sh + args: + - -ec + - | + apk add --no-cache rsync openssh-client + mkdir -p ~/.ssh + chmod 700 ~/.ssh + echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master - # Upload x86_64 wheels and source packages - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*manylinux201* + # Upload x86_64 wheels and source packages + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*manylinux201* -- name: python-aarch64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-aarch64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/manylinux2014_aarch64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload aarch64 wheels - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*manylinux201* + # Upload aarch64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*manylinux201* -- name: python-musl-x86_64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-musl-x86_64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_x86_64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload musl x86_64 wheels - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*musllinux_1_1_x86_64* + # Upload musl x86_64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*musllinux_1_1_x86_64* -- name: python-musl-aarch64 - plan: - - get: deltachat-core-rust - - get: deltachat-core-rust-release - trigger: true + - name: python-musl-aarch64 + plan: + - get: deltachat-core-rust + - get: deltachat-core-rust-release + trigger: true - # Build manylinux image with additional dependencies - - task: build-coredeps - privileged: true - config: - inputs: - # Building the latest, not tagged coredeps - - name: deltachat-core-rust - image_resource: - source: - repository: concourse/oci-build-task - type: registry-image - outputs: - - name: coredeps-image - path: image - params: - CONTEXT: deltachat-core-rust/scripts/coredeps - UNPACK_ROOTFS: "true" - BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64 - platform: linux - caches: - - path: cache - run: - path: build + # Build manylinux image with additional dependencies + - task: build-coredeps + privileged: true + config: + inputs: + # Building the latest, not tagged coredeps + - name: deltachat-core-rust + image_resource: + source: + repository: concourse/oci-build-task + type: registry-image + outputs: + - name: coredeps-image + path: image + params: + CONTEXT: deltachat-core-rust/scripts/coredeps + UNPACK_ROOTFS: "true" + BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64 + platform: linux + caches: + - path: cache + run: + path: build - # Use built image to build python wheels - - task: build-wheels - image: coredeps-image - config: - inputs: - - name: deltachat-core-rust-release - path: . - outputs: - - name: py-wheels - path: ./python/.docker-tox/wheelhouse/ - platform: linux - run: - path: bash - args: - - -exc - - | - scripts/run_all.sh + # Use built image to build python wheels + - task: build-wheels + image: coredeps-image + config: + inputs: + - name: deltachat-core-rust-release + path: . + outputs: + - name: py-wheels + path: ./python/.docker-tox/wheelhouse/ + platform: linux + run: + path: bash + args: + - -exc + - | + scripts/run_all.sh - # Upload musl aarch64 wheels - - task: upload-wheels - config: - inputs: - - name: py-wheels - image_resource: - type: registry-image - source: - repository: debian - platform: linux - run: - path: sh - args: - - -ec - - | - apt-get update -y - apt-get install -y --no-install-recommends python3-pip python3-setuptools - pip3 install devpi - devpi use https://m.devpi.net/dc/master - devpi login ((devpi.login)) --password ((devpi.password)) - devpi upload py-wheels/*musllinux_1_1_aarch64* + # Upload musl aarch64 wheels + - task: upload-wheels + config: + inputs: + - name: py-wheels + image_resource: + type: registry-image + source: + repository: debian + platform: linux + run: + path: sh + args: + - -ec + - | + apt-get update -y + apt-get install -y --no-install-recommends python3-pip python3-setuptools + pip3 install devpi + devpi use https://m.devpi.net/dc/master + devpi login ((devpi.login)) --password ((devpi.password)) + devpi upload py-wheels/*musllinux_1_1_aarch64* From 0890b669fad62d7fa05f0eba338b3f577a8164f1 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 24 Feb 2023 23:35:29 +0000 Subject: [PATCH 27/57] Move RFC URLs in standards.md to the end Also update Autodiscover URL --- standards.md | 66 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/standards.md b/standards.md index 916038bca..546c22780 100644 --- a/standards.md +++ b/standards.md @@ -3,26 +3,54 @@ Some of the standards Delta Chat is based on: Tasks | Standards ----------------------------------|--------------------------------------------- -Transport | IMAP v4 ([RFC 3501](https://tools.ietf.org/html/rfc3501)), SMTP ([RFC 5321](https://tools.ietf.org/html/rfc5321)) and Internet Message Format (IMF, [RFC 5322](https://tools.ietf.org/html/rfc5322)) -Proxy | SOCKS5 ([RFC 1928](https://tools.ietf.org/html/rfc1928)) -Embedded media | MIME Document Series ([RFC 2045](https://tools.ietf.org/html/rfc2045), [RFC 2046](https://tools.ietf.org/html/rfc2046)), Content-Disposition Header ([RFC 2183](https://tools.ietf.org/html/rfc2183)), Multipart/Related ([RFC 2387](https://tools.ietf.org/html/rfc2387)) -Text and Quote encoding | Fixed, Flowed ([RFC 3676](https://tools.ietf.org/html/rfc3676)) -Reactions | Reaction: Indicating Summary Reaction to a Message [RFC 9078](https://datatracker.ietf.org/doc/rfc9078/) -Filename encoding | Encoded Words ([RFC 2047](https://tools.ietf.org/html/rfc2047)), Encoded Word Extensions ([RFC 2231](https://tools.ietf.org/html/rfc2231)) -Identify server folders | IMAP LIST Extension ([RFC 6154](https://tools.ietf.org/html/rfc6154)) -Push | IMAP IDLE ([RFC 2177](https://tools.ietf.org/html/rfc2177)) -Quota | IMAP QUOTA extension ([RFC 2087](https://tools.ietf.org/html/rfc2087)) -Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162](https://tools.ietf.org/html/rfc7162)) -Client/server identification | IMAP ID extension ([RFC 2971](https://datatracker.ietf.org/doc/html/rfc2971)) -Authorization | OAuth2 ([RFC 6749](https://tools.ietf.org/html/rfc6749)) -End-to-end encryption | [Autocrypt Level 1](https://autocrypt.org/level1.html), OpenPGP ([RFC 4880](https://tools.ietf.org/html/rfc4880)), Security Multiparts for MIME ([RFC 1847](https://tools.ietf.org/html/rfc1847)) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html) +-------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Transport | IMAP v4 ([RFC 3501][]), SMTP ([RFC 5321][]) and Internet Message Format (IMF, [RFC 5322][]) +Proxy | SOCKS5 ([RFC 1928][]) +Embedded media | MIME Document Series ([RFC 2045][], [RFC 2046][]), Content-Disposition Header ([RFC 2183][]), Multipart/Related ([RFC 2387][]) +Text and Quote encoding | Fixed, Flowed ([RFC 3676][]) +Reactions | Reaction: Indicating Summary Reaction to a Message ([RFC 9078][]) +Filename encoding | Encoded Words ([RFC 2047][]), Encoded Word Extensions ([RFC 2231][]) +Identify server folders | IMAP LIST Extension ([RFC 6154][]) +Push | IMAP IDLE ([RFC 2177][]) +Quota | IMAP QUOTA extension ([RFC 2087][]) +Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162][]) +Client/server identification | IMAP ID extension ([RFC 2971][]) +Authorization | OAuth2 ([RFC 6749][]) +End-to-end encryption | [Autocrypt Level 1][], OpenPGP ([RFC 4880][]), Security Multiparts for MIME ([RFC 1847][]) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html) Header encryption | [Protected Headers for Cryptographic E-mail](https://datatracker.ietf.org/doc/draft-autocrypt-lamps-protected-headers/) -Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover](https://technet.microsoft.com/library/bb124251(v=exchg.150).aspx) +Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover][] Messenger functions | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-mail-specification) -Detect mailing list | List-Id ([RFC 2919](https://tools.ietf.org/html/rfc2919)) and Precedence ([RFC 3834](https://tools.ietf.org/html/rfc3834)) -User and chat colors | [XEP-0392](https://xmpp.org/extensions/xep-0392.html): Consistent Color Generation -Send and receive system messages | Multipart/Report Media Type ([RFC 6522](https://tools.ietf.org/html/rfc6522)) -Return receipts | Message Disposition Notification (MDN, [RFC 8098](https://tools.ietf.org/html/rfc8098), [RFC 3503](https://tools.ietf.org/html/rfc3503)) using the Chat-Disposition-Notification-To header +Detect mailing list | List-Id ([RFC 2919][]) and Precedence ([RFC 3834][]) +User and chat colors | [XEP-0392][]: Consistent Color Generation +Send and receive system messages | Multipart/Report Media Type ([RFC 6522][]) +Return receipts | Message Disposition Notification (MDN, [RFC 8098][], [RFC 3503][]) using the Chat-Disposition-Notification-To header Locations | KML ([Open Geospatial Consortium](http://www.opengeospatial.org/standards/kml/), [Google Dev](https://developers.google.com/kml/)) +[Autocrypt Level 1]: https://autocrypt.org/level1.html +[Autodiscover]: https://learn.microsoft.com/en-us/exchange/autodiscover-service-for-exchange-2013 +[XEP-0392]: https://xmpp.org/extensions/xep-0392.html +[RFC 1847]: https://tools.ietf.org/html/rfc1847 +[RFC 1928]: https://tools.ietf.org/html/rfc1928 +[RFC 2045]: https://tools.ietf.org/html/rfc2045 +[RFC 2046]: https://tools.ietf.org/html/rfc2046 +[RFC 2047]: https://tools.ietf.org/html/rfc2047 +[RFC 2087]: https://tools.ietf.org/html/rfc2087 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 2183]: https://tools.ietf.org/html/rfc2183 +[RFC 2231]: https://tools.ietf.org/html/rfc2231 +[RFC 2387]: https://tools.ietf.org/html/rfc2387 +[RFC 2919]: https://tools.ietf.org/html/rfc2919 +[RFC 2971]: https://tools.ietf.org/html/rfc2971 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 3503]: https://tools.ietf.org/html/rfc3503 +[RFC 3676]: https://tools.ietf.org/html/rfc3676 +[RFC 3834]: https://tools.ietf.org/html/rfc3834 +[RFC 4880]: https://tools.ietf.org/html/rfc4880 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 5322]: https://tools.ietf.org/html/rfc5322 +[RFC 6154]: https://tools.ietf.org/html/rfc6154 +[RFC 6522]: https://tools.ietf.org/html/rfc6522 +[RFC 6749]: https://tools.ietf.org/html/rfc6749 +[RFC 7162]: https://tools.ietf.org/html/rfc7162 +[RFC 8098]: https://tools.ietf.org/html/rfc8098 +[RFC 9078]: https://tools.ietf.org/html/rfc9078 From 8f0d07b93c694b3f729f611d8f68a479a320545f Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 01:14:14 +0000 Subject: [PATCH 28/57] Make smeared timestamp creation non-async Using atomic operations instead, so create_smeared_timestamp() can be used in sync functions, such as SQL transactions. --- CHANGELOG.md | 1 + src/chat.rs | 17 ++-- src/context.rs | 5 +- src/ephemeral.rs | 2 +- src/lib.rs | 1 + src/message.rs | 9 +-- src/mimefactory.rs | 2 +- src/receive_imf.rs | 4 +- src/timesmearing.rs | 193 ++++++++++++++++++++++++++++++++++++++++++++ src/tools.rs | 85 +++---------------- src/webxdc.rs | 2 +- 11 files changed, 222 insertions(+), 99 deletions(-) create mode 100644 src/timesmearing.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3adf1b5a7..b3aa335a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Changes +- Make smeared timestamp generation non-async. #4075 ### Fixes diff --git a/src/chat.rs b/src/chat.rs index 26e7a7db2..837b88164 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -276,7 +276,7 @@ impl ChatId { grpname, grpid, create_blocked, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), create_protected, param.unwrap_or_default(), ], @@ -482,7 +482,7 @@ impl ChatId { self, &msg_text, cmd, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), None, None, None, @@ -1956,7 +1956,6 @@ impl ChatIdBlocked { _ => (), } - let created_timestamp = create_smeared_timestamp(context).await; let chat_id = context .sql .transaction(move |transaction| { @@ -1969,7 +1968,7 @@ impl ChatIdBlocked { chat_name, params.to_string(), create_blocked as u8, - created_timestamp, + create_smeared_timestamp(context) ], )?; let chat_id = ChatId::new( @@ -2117,7 +2116,7 @@ async fn prepare_msg_common( context, msg, update_msg_id, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), ) .await?; msg.chat_id = chat_id; @@ -2842,7 +2841,7 @@ pub async fn create_group_chat( Chattype::Group, chat_name, grpid, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), ], ) .await?; @@ -2900,7 +2899,7 @@ pub async fn create_broadcast_list(context: &Context) -> Result { Chattype::Broadcast, chat_name, grpid, - create_smeared_timestamp(context).await, + create_smeared_timestamp(context), ], ) .await?; @@ -3361,7 +3360,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) if let Some(reason) = chat.why_cant_send(context).await? { bail!("cannot send to {}: {}", chat_id, reason); } - curr_timestamp = create_smeared_timestamps(context, msg_ids.len()).await; + curr_timestamp = create_smeared_timestamps(context, msg_ids.len()); let ids = context .sql .query_map( @@ -3563,7 +3562,7 @@ pub async fn add_device_msg_with_importance( msg.try_calc_and_set_dimensions(context).await.ok(); prepare_msg_blob(context, msg).await?; - let timestamp_sent = create_smeared_timestamp(context).await; + let timestamp_sent = create_smeared_timestamp(context); // makes sure, the added message is the last one, // even if the date is wrong (useful esp. when warning about bad dates) diff --git a/src/context.rs b/src/context.rs index 21276f47e..eebb5541c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -27,6 +27,7 @@ use crate::quota::QuotaInfo; use crate::scheduler::Scheduler; use crate::sql::Sql; use crate::stock_str::StockStrings; +use crate::timesmearing::SmearedTimestamp; use crate::tools::{duration_to_str, time}; /// Builder for the [`Context`]. @@ -189,7 +190,7 @@ pub struct InnerContext { /// Blob directory path pub(crate) blobdir: PathBuf, pub(crate) sql: Sql, - pub(crate) last_smeared_timestamp: RwLock, + pub(crate) smeared_timestamp: SmearedTimestamp, running_state: RwLock, /// Mutex to avoid generating the key for the user more than once. pub(crate) generating_key_mutex: Mutex<()>, @@ -360,7 +361,7 @@ impl Context { blobdir, running_state: RwLock::new(Default::default()), sql: Sql::new(dbfile), - last_smeared_timestamp: RwLock::new(0), + smeared_timestamp: SmearedTimestamp::new(), generating_key_mutex: Mutex::new(()), oauth2_mutex: Mutex::new(()), wrong_pw_warning_mutex: Mutex::new(()), diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 606c7270f..e881bd082 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -650,7 +650,7 @@ mod tests { use crate::download::DownloadState; use crate::receive_imf::receive_imf; use crate::test_utils::TestContext; - use crate::tools::MAX_SECONDS_TO_LEND_FROM_FUTURE; + use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE; use crate::{ chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus}, tools::IsNoneOrEmpty, diff --git a/src/lib.rs b/src/lib.rs index 9feb6329b..c224ab09d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,6 +92,7 @@ mod smtp; mod socks; pub mod stock_str; mod sync; +mod timesmearing; mod token; mod update_helper; pub mod webxdc; diff --git a/src/message.rs b/src/message.rs index a7b2e27e3..8e1528a55 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1771,13 +1771,8 @@ async fn ndn_maybe_add_info_msg( // Tell the user which of the recipients failed if we know that (because in // a group, this might otherwise be unclear) let text = stock_str::failed_sending_to(context, contact.get_display_name()).await; - chat::add_info_msg( - context, - chat_id, - &text, - create_smeared_timestamp(context).await, - ) - .await?; + chat::add_info_msg(context, chat_id, &text, create_smeared_timestamp(context)) + .await?; context.emit_event(EventType::ChatModified(chat_id)); } } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 43978c112..d98edabbe 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -250,7 +250,7 @@ impl<'a> MimeFactory<'a> { .get_config(Config::Selfstatus) .await? .unwrap_or_default(); - let timestamp = create_smeared_timestamp(context).await; + let timestamp = create_smeared_timestamp(context); let res = MimeFactory::<'a> { from_addr, diff --git a/src/receive_imf.rs b/src/receive_imf.rs index b86fbed7c..e8200ad62 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -203,7 +203,7 @@ pub(crate) async fn receive_imf_inner( ) .await?; - let rcvd_timestamp = smeared_time(context).await; + let rcvd_timestamp = smeared_time(context); // Sender timestamp is allowed to be a bit in the future due to // unsynchronized clocks, but not too much. @@ -1380,7 +1380,7 @@ async fn calc_sort_timestamp( } } - Ok(min(sort_timestamp, smeared_time(context).await)) + Ok(min(sort_timestamp, smeared_time(context))) } async fn lookup_chat_by_reply( diff --git a/src/timesmearing.rs b/src/timesmearing.rs new file mode 100644 index 000000000..d6b0d5be0 --- /dev/null +++ b/src/timesmearing.rs @@ -0,0 +1,193 @@ +//! # Time smearing. +//! +//! As e-mails typically only use a second-based-resolution for timestamps, +//! the order of two mails sent withing one second is unclear. +//! This is bad e.g. when forwarding some messages from a chat - +//! these messages will appear at the recipient easily out of order. +//! +//! We work around this issue by not sending out two mails with the same timestamp. +//! For this purpose, in short, we track the last timestamp used in `last_smeared_timestamp` +//! when another timestamp is needed in the same second, we use `last_smeared_timestamp+1` +//! after some moments without messages sent out, +//! `last_smeared_timestamp` is again in sync with the normal time. +//! +//! However, we do not do all this for the far future, +//! but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE` + +use std::cmp::{max, min}; +use std::sync::atomic::{AtomicI64, Ordering}; + +pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5; + +/// Smeared timestamp generator. +#[derive(Debug)] +pub struct SmearedTimestamp { + /// Next timestamp available for allocation. + smeared_timestamp: AtomicI64, +} + +impl SmearedTimestamp { + /// Creates a new smeared timestamp generator. + pub fn new() -> Self { + Self { + smeared_timestamp: AtomicI64::new(0), + } + } + + /// Allocates `count` unique timestamps. + /// + /// Returns the first allocated timestamp. + pub fn create_n(&self, now: i64, count: i64) -> i64 { + let mut prev = self.smeared_timestamp.load(Ordering::Relaxed); + loop { + // Advance the timestamp if it is in the past, + // but keep `count - 1` timestamps from the past if possible. + let t = max(prev, now - count + 1); + + // Rewind the time back if there is no room + // to allocate `count` timestamps without going too far into the future. + // Not going too far into the future + // is more important than generating unique timestamps. + let first = min(t, now + MAX_SECONDS_TO_LEND_FROM_FUTURE - count + 1); + + // Allocate `count` timestamps by advancing the current timestamp. + let next = first + count; + + if let Err(x) = self.smeared_timestamp.compare_exchange_weak( + prev, + next, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + prev = x; + } else { + return first; + } + } + } + + /// Creates a single timestamp. + pub fn create(&self, now: i64) -> i64 { + self.create_n(now, 1) + } + + /// Returns the current smeared timestamp. + pub fn current(&self) -> i64 { + self.smeared_timestamp.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use std::time::SystemTime; + + use super::*; + use crate::test_utils::TestContext; + use crate::tools::{create_smeared_timestamp, create_smeared_timestamps, smeared_time, time}; + + #[test] + fn test_smeared_timestamp() { + let smeared_timestamp = SmearedTimestamp::new(); + let now = time(); + + assert_eq!(smeared_timestamp.current(), 0); + + for i in 0..MAX_SECONDS_TO_LEND_FROM_FUTURE { + assert_eq!(smeared_timestamp.create(now), now + i); + } + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + + // System time rewinds back by 1000 seconds. + let now = now - 1000; + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + assert_eq!( + smeared_timestamp.create(now), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + ); + assert_eq!( + smeared_timestamp.create(now + 1), + now + MAX_SECONDS_TO_LEND_FROM_FUTURE + 1 + ); + assert_eq!(smeared_timestamp.create(now + 100), now + 100); + assert_eq!(smeared_timestamp.create(now + 100), now + 101); + assert_eq!(smeared_timestamp.create(now + 100), now + 102); + } + + #[test] + fn test_create_n_smeared_timestamps() { + let smeared_timestamp = SmearedTimestamp::new(); + let now = time(); + + // Create a single timestamp to initialize the generator. + assert_eq!(smeared_timestamp.create(now), now); + + // Wait a minute. + let now = now + 60; + + // Simulate forwarding 7 messages. + let forwarded_messages = 7; + + // We have not sent anything for a minute, + // so we can take the current timestamp and take 6 timestamps from the past. + assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 6); + + assert_eq!(smeared_timestamp.current(), now + 1); + + // Wait 4 seconds. + // Now we have 3 free timestamps in the past. + let now = now + 4; + + assert_eq!(smeared_timestamp.current(), now - 3); + + // Forward another 7 messages. + // We can only lend 3 timestamps from the past. + assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 3); + + // We had to borrow 3 timestamps from the future + // because there were not enough timestamps in the past. + assert_eq!(smeared_timestamp.current(), now + 4); + + // Forward another 7 messages. + // We cannot use more than 5 timestamps from the future, + // so we use 5 timestamps from the future, + // the current timestamp and one timestamp from the past. + assert_eq!(smeared_timestamp.create_n(now, forwarded_messages), now - 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_create_smeared_timestamp() { + let t = TestContext::new().await; + assert_ne!(create_smeared_timestamp(&t), create_smeared_timestamp(&t)); + assert!( + create_smeared_timestamp(&t) + >= SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() as i64 + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_create_smeared_timestamps() { + let t = TestContext::new().await; + let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1; + let start = create_smeared_timestamps(&t, count as usize); + let next = smeared_time(&t); + assert!((start + count - 1) < next); + + let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30; + let start = create_smeared_timestamps(&t, count as usize); + let next = smeared_time(&t); + assert!((start + count - 1) < next); + } +} diff --git a/src/tools.rs b/src/tools.rs index 7e8d7e922..c64e8d00a 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -3,7 +3,6 @@ #![allow(missing_docs)] -use core::cmp::{max, min}; use std::borrow::Cow; use std::fmt; use std::io::Cursor; @@ -140,63 +139,27 @@ pub(crate) fn gm2local_offset() -> i64 { i64::from(lt.offset().local_minus_utc()) } -// timesmearing -// - as e-mails typically only use a second-based-resolution for timestamps, -// the order of two mails sent withing one second is unclear. -// this is bad eg. when forwarding some messages from a chat - -// these messages will appear at the recipient easily out of order. -// - we work around this issue by not sending out two mails with the same timestamp. -// - for this purpose, in short, we track the last timestamp used in `last_smeared_timestamp` -// when another timestamp is needed in the same second, we use `last_smeared_timestamp+1` -// - after some moments without messages sent out, -// `last_smeared_timestamp` is again in sync with the normal time. -// - however, we do not do all this for the far future, -// but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE` -pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5; - /// Returns the current smeared timestamp, /// /// The returned timestamp MUST NOT be sent out. -pub(crate) async fn smeared_time(context: &Context) -> i64 { - let mut now = time(); - let ts = *context.last_smeared_timestamp.read().await; - if ts >= now { - now = ts + 1; - } - - now +pub(crate) fn smeared_time(context: &Context) -> i64 { + let now = time(); + let ts = context.smeared_timestamp.current(); + std::cmp::max(ts, now) } /// Returns a timestamp that is guaranteed to be unique. -pub(crate) async fn create_smeared_timestamp(context: &Context) -> i64 { +pub(crate) fn create_smeared_timestamp(context: &Context) -> i64 { let now = time(); - let mut ret = now; - - let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await; - if ret <= *last_smeared_timestamp { - ret = *last_smeared_timestamp + 1; - if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE { - ret = now + MAX_SECONDS_TO_LEND_FROM_FUTURE - } - } - - *last_smeared_timestamp = ret; - ret + context.smeared_timestamp.create(now) } // creates `count` timestamps that are guaranteed to be unique. -// the frist created timestamps is returned directly, +// the first created timestamps is returned directly, // get the other timestamps just by adding 1..count-1 -pub(crate) async fn create_smeared_timestamps(context: &Context, count: usize) -> i64 { +pub(crate) fn create_smeared_timestamps(context: &Context, count: usize) -> i64 { let now = time(); - let count = count as i64; - let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count; - - let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await; - start = max(*last_smeared_timestamp + 1, start); - - *last_smeared_timestamp = start + count - 1; - start + context.smeared_timestamp.create_n(now, count as i64) } // if the system time is not plausible, once a day, add a device message. @@ -1071,36 +1034,6 @@ DKIM Results: Passed=true, Works=true, Allow_Keychange=true"; assert!(!file_exist!(context, &fn0)); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_create_smeared_timestamp() { - let t = TestContext::new().await; - assert_ne!( - create_smeared_timestamp(&t).await, - create_smeared_timestamp(&t).await - ); - assert!( - create_smeared_timestamp(&t).await - >= SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() as i64 - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_create_smeared_timestamps() { - let t = TestContext::new().await; - let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1; - let start = create_smeared_timestamps(&t, count as usize).await; - let next = smeared_time(&t).await; - assert!((start + count - 1) < next); - - let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30; - let start = create_smeared_timestamps(&t, count as usize).await; - let next = smeared_time(&t).await; - assert!((start + count - 1) < next); - } - #[test] fn test_duration_to_str() { assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s"); diff --git a/src/webxdc.rs b/src/webxdc.rs index f4cf660a6..de786ea99 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -408,7 +408,7 @@ impl Context { .create_status_update_record( &mut instance, update_str, - create_smeared_timestamp(self).await, + create_smeared_timestamp(self), send_now, ContactId::SELF, ) From 3fc67de35ed56049498b2d263663438a79564082 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:33:21 +0000 Subject: [PATCH 29/57] Stop saving and loading jobs.param column --- src/download.rs | 7 +------ src/job.rs | 23 ++++++----------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/src/download.rs b/src/download.rs index 5b544fe31..2fed19841 100644 --- a/src/download.rs +++ b/src/download.rs @@ -13,7 +13,6 @@ use crate::imap::{Imap, ImapActionResult}; use crate::job::{self, Action, Job, Status}; use crate::message::{Message, MsgId, Viewtype}; use crate::mimeparser::{MimeMessage, Part}; -use crate::param::Params; use crate::tools::time; use crate::{job_try, stock_str, EventType}; @@ -86,11 +85,7 @@ impl MsgId { DownloadState::Available | DownloadState::Failure => { self.update_download_state(context, DownloadState::InProgress) .await?; - job::add( - context, - Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0), - ) - .await?; + job::add(context, Job::new(Action::DownloadMsg, self.to_u32(), 0)).await?; } } Ok(()) diff --git a/src/job.rs b/src/job.rs index cd54262ea..9a16421f7 100644 --- a/src/job.rs +++ b/src/job.rs @@ -13,7 +13,6 @@ use rand::{thread_rng, Rng}; use crate::context::Context; use crate::imap::{get_folder_meaning, FolderMeaning, Imap}; -use crate::param::Params; use crate::scheduler::InterruptInfo; use crate::tools::time; @@ -77,7 +76,6 @@ pub struct Job { pub desired_timestamp: i64, pub added_timestamp: i64, pub tries: u32, - pub param: Params, } impl fmt::Display for Job { @@ -87,7 +85,7 @@ impl fmt::Display for Job { } impl Job { - pub fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self { + pub fn new(action: Action, foreign_id: u32, delay_seconds: i64) -> Self { let timestamp = time(); Self { @@ -97,7 +95,6 @@ impl Job { desired_timestamp: timestamp + delay_seconds, added_timestamp: timestamp, tries: 0, - param, } } @@ -127,23 +124,21 @@ impl Job { context .sql .execute( - "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;", + "UPDATE jobs SET desired_timestamp=?, tries=? WHERE id=?;", paramsv![ self.desired_timestamp, i64::from(self.tries), - self.param.to_string(), self.job_id as i32, ], ) .await?; } else { context.sql.execute( - "INSERT INTO jobs (added_timestamp, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?);", + "INSERT INTO jobs (added_timestamp, action, foreign_id, desired_timestamp) VALUES (?,?,?,?);", paramsv![ self.added_timestamp, self.action, self.foreign_id, - self.param.to_string(), self.desired_timestamp ] ).await?; @@ -297,11 +292,7 @@ fn get_backoff_time_offset(tries: u32) -> i64 { pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { kill_action(context, Action::ResyncFolders).await?; - add( - context, - Job::new(Action::ResyncFolders, 0, Params::new(), 0), - ) - .await?; + add(context, Job::new(Action::ResyncFolders, 0, 0)).await?; Ok(()) } @@ -370,7 +361,6 @@ LIMIT 1; desired_timestamp: row.get("desired_timestamp")?, added_timestamp: row.get("added_timestamp")?, tries: row.get("tries")?, - param: row.get::<_, String>("param")?.parse().unwrap_or_default(), }; Ok(job) @@ -410,8 +400,8 @@ mod tests { .sql .execute( "INSERT INTO jobs - (added_timestamp, action, foreign_id, param, desired_timestamp) - VALUES (?, ?, ?, ?, ?);", + (added_timestamp, action, foreign_id, desired_timestamp) + VALUES (?, ?, ?, ?);", paramsv![ now, if valid { @@ -420,7 +410,6 @@ mod tests { -1 }, foreign_id, - Params::new().to_string(), now ], ) From 7f2ccfb168b0b27733be37cbd0649bf7fc679d77 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:33:47 +0000 Subject: [PATCH 30/57] job: remove match with a single branch --- src/job.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/job.rs b/src/job.rs index 9a16421f7..7333ce498 100644 --- a/src/job.rs +++ b/src/job.rs @@ -303,12 +303,8 @@ pub async fn add(context: &Context, job: Job) -> Result<()> { job.save(context).await.context("failed to save job")?; if delay_seconds == 0 { - match action { - Action::ResyncFolders | Action::DownloadMsg => { - info!(context, "interrupt: imap"); - context.interrupt_inbox(InterruptInfo::new(false)).await; - } - } + info!(context, "interrupt: imap"); + context.interrupt_inbox(InterruptInfo::new(false)).await; } Ok(()) } From 10e8bcb73ea43b69d56bd22c14d874b575f412f6 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:35:08 +0000 Subject: [PATCH 31/57] job: remove unused variable --- src/job.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index 7333ce498..df808f458 100644 --- a/src/job.rs +++ b/src/job.rs @@ -298,7 +298,6 @@ pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { /// Adds a job to the database, scheduling it. pub async fn add(context: &Context, job: Job) -> Result<()> { - let action = job.action; let delay_seconds = job.delay_seconds(); job.save(context).await.context("failed to save job")?; From 9f804c379c5818639d7105a1716487080841ef58 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:42:14 +0000 Subject: [PATCH 32/57] Remove always zero argument to Job::new() --- src/download.rs | 2 +- src/job.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/download.rs b/src/download.rs index 2fed19841..fd0efc9c3 100644 --- a/src/download.rs +++ b/src/download.rs @@ -85,7 +85,7 @@ impl MsgId { DownloadState::Available | DownloadState::Failure => { self.update_download_state(context, DownloadState::InProgress) .await?; - job::add(context, Job::new(Action::DownloadMsg, self.to_u32(), 0)).await?; + job::add(context, Job::new(Action::DownloadMsg, self.to_u32())).await?; } } Ok(()) diff --git a/src/job.rs b/src/job.rs index df808f458..4ba203823 100644 --- a/src/job.rs +++ b/src/job.rs @@ -85,14 +85,14 @@ impl fmt::Display for Job { } impl Job { - pub fn new(action: Action, foreign_id: u32, delay_seconds: i64) -> Self { + pub fn new(action: Action, foreign_id: u32) -> Self { let timestamp = time(); Self { job_id: 0, action, foreign_id, - desired_timestamp: timestamp + delay_seconds, + desired_timestamp: timestamp, added_timestamp: timestamp, tries: 0, } @@ -292,7 +292,7 @@ fn get_backoff_time_offset(tries: u32) -> i64 { pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { kill_action(context, Action::ResyncFolders).await?; - add(context, Job::new(Action::ResyncFolders, 0, 0)).await?; + add(context, Job::new(Action::ResyncFolders, 0)).await?; Ok(()) } From 1450bf5483305894063345673b9c3202d54c4bc3 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 02:45:22 +0000 Subject: [PATCH 33/57] Remove Job::delay_seconds() It always returned 0 for added jobs. --- src/job.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/job.rs b/src/job.rs index 4ba203823..ff37ea543 100644 --- a/src/job.rs +++ b/src/job.rs @@ -98,10 +98,6 @@ impl Job { } } - pub fn delay_seconds(&self) -> i64 { - self.desired_timestamp - self.added_timestamp - } - /// Deletes the job from the database. async fn delete(self, context: &Context) -> Result<()> { if self.job_id != 0 { @@ -298,13 +294,10 @@ pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { /// Adds a job to the database, scheduling it. pub async fn add(context: &Context, job: Job) -> Result<()> { - let delay_seconds = job.delay_seconds(); job.save(context).await.context("failed to save job")?; - if delay_seconds == 0 { - info!(context, "interrupt: imap"); - context.interrupt_inbox(InterruptInfo::new(false)).await; - } + info!(context, "interrupt: imap"); + context.interrupt_inbox(InterruptInfo::new(false)).await; Ok(()) } From 992a6cbfc2c395cf8d636ce91c582d1659a5ed94 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 21 Feb 2023 20:51:38 +0000 Subject: [PATCH 34/57] mimeparser: wrap try_decrypt() into block_in_place() try_decrypt() is a CPU-bound task. When called from async function, it should be wrapped in tokio::task::spawn_blocking(). Using tokio::task::spawn_blocking() is difficult here because of &mail, &private_keyring and &public_keyring borrows, so we should at least use tokio::task::block_in_place() to avoid blocking the executor. --- CHANGELOG.md | 1 + src/mimeparser.rs | 39 ++++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3aa335a8..39bb421be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Make smeared timestamp generation non-async. #4075 ### Fixes +- Do not block async task executor while decrypting the messages. #4079 ### API-Changes diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 0b246a257..075fd192f 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -256,26 +256,27 @@ impl MimeMessage { hop_info += &decryption_info.dkim_results.to_string(); let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref()); - let (mail, mut signatures, encrypted) = - match try_decrypt(context, &mail, &private_keyring, &public_keyring) { - Ok(Some((raw, signatures))) => { - mail_raw = raw; - let decrypted_mail = mailparse::parse_mail(&mail_raw)?; - if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { - info!( - context, - "decrypted message mime-body:\n{}", - String::from_utf8_lossy(&mail_raw), - ); - } - (Ok(decrypted_mail), signatures, true) + let (mail, mut signatures, encrypted) = match tokio::task::block_in_place(|| { + try_decrypt(context, &mail, &private_keyring, &public_keyring) + }) { + Ok(Some((raw, signatures))) => { + mail_raw = raw; + let decrypted_mail = mailparse::parse_mail(&mail_raw)?; + if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { + info!( + context, + "decrypted message mime-body:\n{}", + String::from_utf8_lossy(&mail_raw), + ); } - Ok(None) => (Ok(mail), HashSet::new(), false), - Err(err) => { - warn!(context, "decryption failed: {:#}", err); - (Err(err), HashSet::new(), false) - } - }; + (Ok(decrypted_mail), signatures, true) + } + Ok(None) => (Ok(mail), HashSet::new(), false), + Err(err) => { + warn!(context, "decryption failed: {:#}", err); + (Err(err), HashSet::new(), false) + } + }; let mail = mail.as_ref().map(|mail| { let (content, signatures_detached) = validate_detached_signature(mail, &public_keyring) .unwrap_or((mail, Default::default())); From 89696582ada7f1ef76aec88a4aef7a14ec476e3d Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 18 Feb 2023 22:39:58 -0300 Subject: [PATCH 35/57] mimeparser: Handle headers from the signed part of unencrypted signed message This makes DC compatible with "multipart/signed" messages thus allowing switching to them someday from the current "multipart/mixed" unencrypted message format. --- CHANGELOG.md | 1 + src/mimeparser.rs | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39bb421be..3203bd410 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - Updated provider database. - Disable DKIM-Checks again #4076 - Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089 +- mimeparser: handle headers from the signed part of unencrypted signed message #4013 ### Fixes - Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063 diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 075fd192f..50b71b092 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -224,8 +224,32 @@ impl MimeMessage { // Parse hidden headers. let mimetype = mail.ctype.mimetype.parse::()?; + let (part, mimetype) = + if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "signed" { + if let Some(part) = mail.subparts.first() { + // We don't remove "subject" from `headers` because currently just signed + // messages are shown as unencrypted anyway. + + MimeMessage::merge_headers( + context, + &mut headers, + &mut recipients, + &mut from, + &mut list_post, + &mut chat_disposition_notification_to, + &part.headers, + ); + (part, part.ctype.mimetype.parse::()?) + } else { + // If it's a partially fetched message, there are no subparts. + (&mail, mimetype) + } + } else { + // Currently we do not sign unencrypted messages by default. + (&mail, mimetype) + }; if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" { - if let Some(part) = mail.subparts.first() { + if let Some(part) = part.subparts.first() { for field in &part.headers { let key = field.get_key().to_lowercase(); From d1923d68a5b02e297df1c12ce87237f0e409272d Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 18 Feb 2023 23:08:10 -0300 Subject: [PATCH 36/57] Add a config option to sign all messages with Autocrypt header (#3986) Although it does a little for security, it will help to protect from unwanted server-side modifications and bugs. And now we have a time to test "multipart/signed" messages compatibility with other MUAs. --- src/config.rs | 3 ++ src/context.rs | 6 +++ src/e2ee.rs | 13 +++++ src/mimefactory.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++- src/pgp.rs | 14 ++++++ 5 files changed, 154 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 8c1357a8e..fc950ba44 100644 --- a/src/config.rs +++ b/src/config.rs @@ -300,6 +300,9 @@ pub enum Config { /// See `crate::authres::update_authservid_candidates`. AuthservIdCandidates, + /// Make all outgoing messages with Autocrypt header "multipart/signed". + SignUnencrypted, + /// Let the core save all events to the database. /// This value is used internally to remember the MsgId of the logging xdc #[strum(props(default = "0"))] diff --git a/src/context.rs b/src/context.rs index eebb5541c..1d8dbe847 100644 --- a/src/context.rs +++ b/src/context.rs @@ -763,6 +763,12 @@ impl Context { .await? .unwrap_or_default(), ); + res.insert( + "sign_unencrypted", + self.get_config_int(Config::SignUnencrypted) + .await? + .to_string(), + ); res.insert( "debug_logging", diff --git a/src/e2ee.rs b/src/e2ee.rs index bbd8fd895..b870f89fe 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -124,6 +124,19 @@ impl EncryptHelper { Ok(ctext) } + + /// Signs the passed-in `mail` using the private key from `context`. + /// Returns the payload and the signature. + pub async fn sign( + self, + context: &Context, + mail: lettre_email::PartBuilder, + ) -> Result<(lettre_email::MimeMessage, String)> { + let sign_key = SignedSecretKey::load_self(context).await?; + let mime_message = mail.build(); + let signature = pgp::pk_calc_signature(mime_message.as_string().as_bytes(), &sign_key)?; + Ok((mime_message, signature)) + } } /// Ensures a private key exists for the configured user. diff --git a/src/mimefactory.rs b/src/mimefactory.rs index d98edabbe..530bfafe6 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -779,10 +779,36 @@ impl<'a> MimeFactory<'a> { }; // Store protected headers in the outer message. - headers + let message = headers .protected .into_iter() - .fold(message, |message, header| message.header(header)) + .fold(message, |message, header| message.header(header)); + + if self.should_skip_autocrypt() + || !context.get_config_bool(Config::SignUnencrypted).await? + { + message + } else { + let (payload, signature) = encrypt_helper.sign(context, message).await?; + PartBuilder::new() + .header(( + "Content-Type".to_string(), + "multipart/signed; protocol=\"application/pgp-signature\"".to_string(), + )) + .child(payload) + .child( + PartBuilder::new() + .content_type( + &"application/pgp-signature; name=\"signature.asc\"" + .parse::() + .unwrap(), + ) + .header(("Content-Description", "OpenPGP digital signature")) + .header(("Content-Disposition", "attachment; filename=\"signature\";")) + .body(signature) + .build(), + ) + } }; // Store the unprotected headers on the outer message. @@ -2140,6 +2166,96 @@ mod tests { Ok(()) } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_selfavatar_unencrypted_signed() { + // create chat with bob, set selfavatar + let t = TestContext::new_alice().await; + t.set_config(Config::SignUnencrypted, Some("1")) + .await + .unwrap(); + let chat = t.create_chat_with_contact("bob", "bob@example.org").await; + + let file = t.dir.path().join("avatar.png"); + let bytes = include_bytes!("../test-data/image/avatar64x64.png"); + tokio::fs::write(&file, bytes).await.unwrap(); + t.set_config(Config::Selfavatar, Some(file.to_str().unwrap())) + .await + .unwrap(); + + // send message to bob: that should get multipart/mixed because of the avatar moved to inner header; + // make sure, `Subject:` stays in the outer header (imf header) + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("this is the text!".to_string())); + + let sent_msg = t.send_msg(chat.id, &mut msg).await; + let mut payload = sent_msg.payload().splitn(4, "\r\n\r\n"); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("multipart/signed").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 0); + assert_eq!(part.match_indices("Autocrypt:").count(), 1); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("multipart/mixed").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 1); + assert_eq!(part.match_indices("Autocrypt:").count(), 0); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("text/plain").count(), 1); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 0); + + let body = payload.next().unwrap(); + assert_eq!(body.match_indices("this is the text!").count(), 1); + + let bob = TestContext::new_bob().await; + bob.recv_msg(&sent_msg).await; + let alice_id = Contact::lookup_id_by_addr(&bob.ctx, "alice@example.org", Origin::Unknown) + .await + .unwrap() + .unwrap(); + let alice_contact = Contact::load_from_db(&bob.ctx, alice_id).await.unwrap(); + assert!(alice_contact + .get_profile_image(&bob.ctx) + .await + .unwrap() + .is_some()); + + // if another message is sent, that one must not contain the avatar + // and no artificial multipart/mixed nesting + let sent_msg = t.send_msg(chat.id, &mut msg).await; + let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n"); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("multipart/signed").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 0); + assert_eq!(part.match_indices("Autocrypt:").count(), 1); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let part = payload.next().unwrap(); + assert_eq!(part.match_indices("text/plain").count(), 1); + assert_eq!(part.match_indices("Subject:").count(), 1); + assert_eq!(part.match_indices("Autocrypt:").count(), 0); + assert_eq!(part.match_indices("multipart/mixed").count(), 0); + assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); + + let body = payload.next().unwrap(); + assert_eq!(body.match_indices("this is the text!").count(), 1); + assert_eq!(body.match_indices("text/plain").count(), 0); + assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0); + assert_eq!(body.match_indices("Subject:").count(), 0); + + bob.recv_msg(&sent_msg).await; + let alice_contact = Contact::load_from_db(&bob.ctx, alice_id).await.unwrap(); + assert!(alice_contact + .get_profile_image(&bob.ctx) + .await + .unwrap() + .is_some()); + } + /// Test that removed member address does not go into the `To:` field. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_remove_member_bcc() -> Result<()> { diff --git a/src/pgp.rs b/src/pgp.rs index 6bc4bface..9d4ae46ea 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -262,6 +262,20 @@ pub async fn pk_encrypt( .await? } +/// Signs `plain` text using `private_key_for_signing`. +pub fn pk_calc_signature( + plain: &[u8], + private_key_for_signing: &SignedSecretKey, +) -> Result { + let msg = Message::new_literal_bytes("", plain).sign( + private_key_for_signing, + || "".into(), + Default::default(), + )?; + let signature = msg.into_signature().to_armored_string(None)?; + Ok(signature) +} + /// Decrypts the message with keys from the private key keyring. /// /// Receiver private keys are provided in From f8e86f4503311ed6fe8b35dadd0f2c466ae1c613 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 25 Feb 2023 13:04:06 +0100 Subject: [PATCH 37/57] make slowest test faster the slowest test was `test_modify_chat_disordered`; due to several sleep(), it lasts at least 11 seconds. however, many of the sleep() are not needed and were added just to get things done that time. this pr removes 7 superfluous sleeps, making the test finish in about 3 seconds, so that this test is no longer the bottleneck when running `cargo test` on fast machines. (this is not only useful to safe some seconds - but also to notice degradations as of https://github.com/deltachat/deltachat-core-rust/issues/4051#issuecomment-1436995166 ) --- src/chat.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 837b88164..803b4fc85 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -4090,7 +4090,6 @@ mod tests { send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?; add_contact_to_chat(&alice, alice_chat_id, bob_id).await?; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; let add1 = alice.pop_sent_msg().await; add_contact_to_chat(&alice, alice_chat_id, claire_id).await?; @@ -4109,29 +4108,18 @@ mod tests { remove_contact_from_chat(&alice, alice_chat_id, daisy_id).await?; let remove2 = alice.pop_sent_msg().await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; assert_eq!(get_chat_contacts(&alice, alice_chat_id).await?.len(), 2); // Bob receives the add and deletion messages out of order let bob = TestContext::new_bob().await; bob.recv_msg(&add1).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - bob.recv_msg(&add3).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - let bob_chat_id = bob.recv_msg(&add2).await.chat_id; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 4); bob.recv_msg(&remove2).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - bob.recv_msg(&remove1).await; - tokio::time::sleep(std::time::Duration::from_millis(1100)).await; - assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?.len(), 2); Ok(()) From eace8c5feeba08593df3490c11bd9085bfee7877 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 15:01:53 +0000 Subject: [PATCH 38/57] Note that 1.108.0 re-enables SMTP pipelining --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3203bd410..ba24f2c2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,7 @@ - Prefer TLS over STARTTLS during autoconfiguration #4021 - Use SOCKS5 configuration for HTTP requests #4017 - Show non-deltachat emails by default for new installations #4019 +- Re-enabled SMTP pipelining after disabling it in #4006 ### Fixes - Fix Securejoin for multiple devices on a joining side #3982 From 79026f93b60546ef5d4a2f49acc70aa9bfd990ac Mon Sep 17 00:00:00 2001 From: adbenitez Date: Sun, 26 Feb 2023 05:13:20 -0500 Subject: [PATCH 39/57] add more advanced API to send a message --- CHANGELOG.md | 1 + deltachat-jsonrpc/src/api/mod.rs | 49 +++++++++++++++++++ deltachat-jsonrpc/src/api/types/message.rs | 12 +++++ .../src/deltachat_rpc_client/chat.py | 16 +++++- 4 files changed, 76 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce2da427e..039ce1254 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067 ### API-Changes +- jsonrpc: add more advanced API to send a message. ## 1.109.0 diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 22f261ced..6064f476a 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -41,6 +41,7 @@ use types::account::Account; use types::chat::FullChat; use types::chat_list::ChatListEntry; use types::contact::ContactObject; +use types::message::DraftMessage; use types::message::MessageObject; use types::provider_info::ProviderInfo; use types::webxdc::WebxdcMessageInfo; @@ -1511,6 +1512,54 @@ impl CommandApi { Ok(message_id.to_u32()) } + async fn send_msg( + &self, + account_id: u32, + chat_id: u32, + draft: DraftMessage, + ) -> Result<(u32, MessageObject)> { + let ctx = self.get_context(account_id).await?; + let mut message = Message::new(if let Some(viewtype) = draft.viewtype { + viewtype.into() + } else if draft.file.is_some() { + Viewtype::File + } else { + Viewtype::Text + }); + if draft.text.is_some() { + message.set_text(draft.text); + } + if draft.html.is_some() { + message.set_html(draft.html); + } + if draft.override_sender_name.is_some() { + message.set_override_sender_name(draft.override_sender_name); + } + if let Some(file) = draft.file { + message.set_file(file, None); + } + if let Some((latitude, longitude)) = draft.location { + message.set_location(latitude, longitude); + } + if let Some(id) = draft.quoted_message_id { + message + .set_quote( + &ctx, + Some( + &Message::load_from_db(&ctx, MsgId::new(id)) + .await + .context("message to quote could not be loaded")?, + ), + ) + .await?; + } + let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message) + .await? + .to_u32(); + let message = MessageObject::from_message_id(&ctx, msg_id).await?; + Ok((msg_id, message)) + } + // --------------------------------------------- // functions for the composer // the composer is the message input field diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index 04a850715..f5185fdb6 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -502,3 +502,15 @@ impl From for JSONRPCMessageListItem { } } } + +#[derive(Deserialize, TypeDef)] +#[serde(rename_all = "camelCase")] +pub struct DraftMessage { + pub text: Option, + pub html: Option, + pub viewtype: Option, + pub file: Option, + pub location: Option<(f64, f64)>, + pub override_sender_name: Option, + pub quoted_message_id: Option, +} diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index 7dac7d2f4..91cb48e4e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union from ._utils import AttrDict -from .const import ChatVisibility +from .const import ChatVisibility, ViewType from .contact import Contact from .message import Message @@ -108,15 +108,27 @@ class Chat: async def send_message( self, text: Optional[str] = None, + html: Optional[str] = None, + viewtype: Optional[ViewType] = None, file: Optional[str] = None, location: Optional[Tuple[float, float]] = None, + override_sender_name: Optional[str] = None, quoted_msg: Optional[Union[int, Message]] = None, ) -> Message: """Send a message and return the resulting Message instance.""" if isinstance(quoted_msg, Message): quoted_msg = quoted_msg.id - msg_id, _ = await self._rpc.misc_send_msg(self.account.id, self.id, text, file, location, quoted_msg) + draft = { + "text": text, + "html": html, + "viewtype": viewtype, + "file": file, + "location": location, + "overrideSenderName": override_sender_name, + "quotedMsg": quoted_msg, + } + msg_id, _ = await self._rpc.send_msg(self.account.id, self.id, draft) return Message(self.account, msg_id) async def send_text(self, text: str) -> Message: From 247bf5865daf591e262d3f4106cf562ca41658cb Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 26 Feb 2023 12:23:57 +0100 Subject: [PATCH 40/57] add countermitm and openpgp4fpr to "standard used" noticed that this is missing while working on https://github.com/deltachat/invite --- standards.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/standards.md b/standards.md index 546c22780..145940cf0 100644 --- a/standards.md +++ b/standards.md @@ -17,6 +17,8 @@ Seen status synchronization | IMAP CONDSTORE extension ([RFC 7162][]) Client/server identification | IMAP ID extension ([RFC 2971][]) Authorization | OAuth2 ([RFC 6749][]) End-to-end encryption | [Autocrypt Level 1][], OpenPGP ([RFC 4880][]), Security Multiparts for MIME ([RFC 1847][]) and [“Mixed Up” Encryption repairing](https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html) +Detect/prevent active attacks | [countermitm][] protocols +Compare public keys | [openpgp4fpr][] URI Scheme Header encryption | [Protected Headers for Cryptographic E-mail](https://datatracker.ietf.org/doc/draft-autocrypt-lamps-protected-headers/) Configuration assistance | [Autoconfigure](https://web.archive.org/web/20210402044801/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration) and [Autodiscover][] Messenger functions | [Chat-over-Email](https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#chat-mail-specification) @@ -27,6 +29,8 @@ Return receipts | Message Disposition Notification (MDN, [RFC 8 Locations | KML ([Open Geospatial Consortium](http://www.opengeospatial.org/standards/kml/), [Google Dev](https://developers.google.com/kml/)) [Autocrypt Level 1]: https://autocrypt.org/level1.html +[countermitm]: https://countermitm.readthedocs.io/en/latest/ +[openpgp4fpr]: https://metacode.biz/openpgp/openpgp4fpr [Autodiscover]: https://learn.microsoft.com/en-us/exchange/autodiscover-service-for-exchange-2013 [XEP-0392]: https://xmpp.org/extensions/xep-0392.html [RFC 1847]: https://tools.ietf.org/html/rfc1847 From 8177070673e2a234cd46f739d4850824828d8ae5 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 22:46:19 +0000 Subject: [PATCH 41/57] Set minimum TLS version to 1.2 --- CHANGELOG.md | 1 + src/net/tls.rs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba24f2c2d..e2d91b675 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changes - Make smeared timestamp generation non-async. #4075 +- Set minimum TLS version to 1.2. #4096 ### Fixes - Do not block async task executor while decrypting the messages. #4079 diff --git a/src/net/tls.rs b/src/net/tls.rs index 980504416..7bb6badfe 100644 --- a/src/net/tls.rs +++ b/src/net/tls.rs @@ -1,7 +1,7 @@ //! TLS support. use anyhow::Result; -use async_native_tls::{Certificate, TlsConnector, TlsStream}; +use async_native_tls::{Certificate, Protocol, TlsConnector, TlsStream}; use once_cell::sync::Lazy; use tokio::io::{AsyncRead, AsyncWrite}; @@ -15,7 +15,9 @@ static LETSENCRYPT_ROOT: Lazy = Lazy::new(|| { }); pub fn build_tls(strict_tls: bool) -> TlsConnector { - let tls_builder = TlsConnector::new().add_root_certificate(LETSENCRYPT_ROOT.clone()); + let tls_builder = TlsConnector::new() + .min_protocol_version(Some(Protocol::Tlsv12)) + .add_root_certificate(LETSENCRYPT_ROOT.clone()); if strict_tls { tls_builder From 7e132b5383a067bb691f84c258708e6c5af5d478 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 20:40:56 +0000 Subject: [PATCH 42/57] Add cargo-deny config and CI --- .github/workflows/ci.yml | 9 +++++++++ CHANGELOG.md | 1 + deltachat-repl/Cargo.toml | 1 + deny.toml | 16 ++++++++++++++++ 4 files changed, 27 insertions(+) create mode 100644 deny.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 392da23d1..474176c03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,15 @@ jobs: - name: Run clippy run: scripts/clippy.sh + cargo_deny: + name: cargo deny + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + arguments: --all-features --workspace + docs: name: Rust doc comments runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index e2d91b675..b6b48021f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changes - Make smeared timestamp generation non-async. #4075 - Set minimum TLS version to 1.2. #4096 +- Run `cargo-deny` in CI. #4101 ### Fixes - Do not block async task executor while decrypting the messages. #4079 diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index bab15c6d7..ed209d9af 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,7 @@ [package] name = "deltachat-repl" version = "1.110.0" +license = "MPL-2.0" edition = "2021" [dependencies] diff --git a/deny.toml b/deny.toml new file mode 100644 index 000000000..a49d4bf51 --- /dev/null +++ b/deny.toml @@ -0,0 +1,16 @@ +[advisories] +ignore = [ + "RUSTSEC-2020-0071", +] + +[licenses] +allow = [ + "0BSD", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "CC0-1.0", + "MIT", + "BSL-1.0", # Boost Software License 1.0 + "Unicode-DFS-2016", +] From 8e9bb8b06e4d44512419482bbd055086cfcaa598 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 26 Feb 2023 13:55:09 +0000 Subject: [PATCH 43/57] Check provider database with CI scripts/update-provider-database.sh checks out the provider database and updates data.rs file, making it easier to update. CI uses this script to check that checked in data.rs corresponds to the provider database repository. --- .github/workflows/ci.yml | 8 +++++++ CHANGELOG.md | 1 + .../update-provider-database.py | 5 ++++- scripts/update-provider-database.sh | 22 +++++++++++++++++++ src/provider/data.rs | 2 +- 5 files changed, 36 insertions(+), 2 deletions(-) rename src/provider/update.py => scripts/update-provider-database.py (98%) create mode 100755 scripts/update-provider-database.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 474176c03..a5086b537 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,14 @@ jobs: with: arguments: --all-features --workspace + provider_database: + name: Check provider database + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check provider database + run: scripts/update-provider-database.sh + docs: name: Rust doc comments runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b48021f..c6044a1bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Make smeared timestamp generation non-async. #4075 - Set minimum TLS version to 1.2. #4096 - Run `cargo-deny` in CI. #4101 +- Check provider database with CI. #4099 ### Fixes - Do not block async task executor while decrypting the messages. #4079 diff --git a/src/provider/update.py b/scripts/update-provider-database.py similarity index 98% rename from src/provider/update.py rename to scripts/update-provider-database.py index 304b80e74..681872ff5 100755 --- a/src/provider/update.py +++ b/scripts/update-provider-database.py @@ -201,7 +201,10 @@ if __name__ == "__main__": out_all += out_ids; out_all += "].iter().copied().collect());\n\n" - now = datetime.datetime.utcnow() + if len(sys.argv) < 3: + now = datetime.datetime.utcnow() + else: + now = datetime.datetime.fromisoformat(sys.argv[2]) out_all += "pub static PROVIDER_UPDATED: Lazy = "\ "Lazy::new(|| chrono::NaiveDate::from_ymd_opt("+str(now.year)+", "+str(now.month)+", "+str(now.day)+").unwrap());\n" diff --git a/scripts/update-provider-database.sh b/scripts/update-provider-database.sh new file mode 100755 index 000000000..c38c1e91d --- /dev/null +++ b/scripts/update-provider-database.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Updates provider database. +# Returns 1 if the database is changed, 0 otherwise. +set -euo pipefail + +export TZ=UTC + +# Provider database revision. +REV=3c8f7e846c915a183dc44536fb5480d1f25d7c42 + +CORE_ROOT="$PWD" +TMP="$(mktemp -d)" +git clone --filter=blob:none https://github.com/deltachat/provider-db.git "$TMP" +cd "$TMP" +git checkout "$REV" +DATE=$(git show -s --format=%cs) +"$CORE_ROOT"/scripts/update-provider-database.py "$TMP/_providers" "$DATE" >"$CORE_ROOT/src/provider/data.rs" +rustfmt "$CORE_ROOT/src/provider/data.rs" +rm -fr "$TMP" + +cd "$CORE_ROOT" +test -z "$(git status --porcelain src/provider/data.rs)" diff --git a/src/provider/data.rs b/src/provider/data.rs index 73eb49634..f7736c6e9 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -1952,4 +1952,4 @@ pub(crate) static PROVIDER_IDS: Lazy> = }); pub static PROVIDER_UPDATED: Lazy = - Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 21).unwrap()); + Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 2, 20).unwrap()); From 0086232bbb216c6afb5b3933c425e6db441d7d44 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 26 Feb 2023 20:53:37 +0000 Subject: [PATCH 44/57] Rename update-provider-database.py into create-provider-data-rs.py --- .../{update-provider-database.py => create-provider-data-rs.py} | 0 scripts/update-provider-database.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{update-provider-database.py => create-provider-data-rs.py} (100%) diff --git a/scripts/update-provider-database.py b/scripts/create-provider-data-rs.py similarity index 100% rename from scripts/update-provider-database.py rename to scripts/create-provider-data-rs.py diff --git a/scripts/update-provider-database.sh b/scripts/update-provider-database.sh index c38c1e91d..69dbff8af 100755 --- a/scripts/update-provider-database.sh +++ b/scripts/update-provider-database.sh @@ -14,7 +14,7 @@ git clone --filter=blob:none https://github.com/deltachat/provider-db.git "$TMP" cd "$TMP" git checkout "$REV" DATE=$(git show -s --format=%cs) -"$CORE_ROOT"/scripts/update-provider-database.py "$TMP/_providers" "$DATE" >"$CORE_ROOT/src/provider/data.rs" +"$CORE_ROOT"/scripts/create-provider-data-rs.py "$TMP/_providers" "$DATE" >"$CORE_ROOT/src/provider/data.rs" rustfmt "$CORE_ROOT/src/provider/data.rs" rm -fr "$TMP" From a2f1df052b1a8f5ee85ce9c6d43065ba5ba045c4 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 26 Feb 2023 20:54:40 +0000 Subject: [PATCH 45/57] Autoformat create-provider-data-rs.py --- scripts/create-provider-data-rs.py | 87 +++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/scripts/create-provider-data-rs.py b/scripts/create-provider-data-rs.py index 681872ff5..ea5d291ee 100755 --- a/scripts/create-provider-data-rs.py +++ b/scripts/create-provider-data-rs.py @@ -11,15 +11,17 @@ out_domains = "" out_ids = "" domains_set = set() + def camel(name): words = name.split("_") return "".join(w.capitalize() for i, w in enumerate(words)) + def cleanstr(s): s = s.strip() s = s.replace("\n", " ") s = s.replace("\\", "\\\\") - s = s.replace("\"", "\\\"") + s = s.replace('"', '\\"') return s @@ -64,7 +66,13 @@ def process_config_defaults(data): config_defaults = data.get("config_defaults", "") for key in config_defaults: value = str(config_defaults[key]) - defaults += " ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n" + defaults += ( + " ConfigDefault { key: Config::" + + camel(key) + + ', value: "' + + value + + '" },\n' + ) defaults += " ])" return defaults @@ -88,11 +96,11 @@ def process_data(data, file): raise TypeError("domain used twice: " + domain) domains_set.add(domain) - domains += " (\"" + domain + "\", &*" + file2varname(file) + "),\n" + domains += ' ("' + domain + '", &*' + file2varname(file) + "),\n" comment += domain + ", " ids = "" - ids += " (\"" + file2id(file) + "\", &*" + file2varname(file) + "),\n" + ids += ' ("' + file2id(file) + '", &*' + file2varname(file) + "),\n" server = "" has_imap = False @@ -120,8 +128,19 @@ def process_data(data, file): if username_pattern != "EMAIL" and username_pattern != "EMAILLOCALPART": raise TypeError("bad username pattern") - server += (" Server { protocol: " + protocol.capitalize() + ", socket: " + socket.capitalize() + ", hostname: \"" - + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern.capitalize() + " },\n") + server += ( + " Server { protocol: " + + protocol.capitalize() + + ", socket: " + + socket.capitalize() + + ', hostname: "' + + hostname + + '", port: ' + + str(port) + + ", username_pattern: " + + username_pattern.capitalize() + + " },\n" + ) opt = process_opt(data) config_defaults = process_config_defaults(data) @@ -133,12 +152,16 @@ def process_data(data, file): before_login_hint = cleanstr(data.get("before_login_hint", "")) after_login_hint = cleanstr(data.get("after_login_hint", "")) if (not has_imap and not has_smtp) or (has_imap and has_smtp): - provider += "static " + file2varname(file) + ": Lazy = Lazy::new(|| Provider {\n" - provider += " id: \"" + file2id(file) + "\",\n" + provider += ( + "static " + + file2varname(file) + + ": Lazy = Lazy::new(|| Provider {\n" + ) + provider += ' id: "' + file2id(file) + '",\n' provider += " status: Status::" + status.capitalize() + ",\n" - provider += " before_login_hint: \"" + before_login_hint + "\",\n" - provider += " after_login_hint: \"" + after_login_hint + "\",\n" - provider += " overview_page: \"" + file2url(file) + "\",\n" + provider += ' before_login_hint: "' + before_login_hint + '",\n' + provider += ' after_login_hint: "' + after_login_hint + '",\n' + provider += ' overview_page: "' + file2url(file) + '",\n' provider += " server: vec![\n" + server + " ],\n" provider += " opt: " + opt + ",\n" provider += " config_defaults: " + config_defaults + ",\n" @@ -148,7 +171,9 @@ def process_data(data, file): raise TypeError("SMTP and IMAP must be specified together or left out both") if status != "OK" and before_login_hint == "": - raise TypeError("status PREPARATION or BROKEN requires before_login_hint: " + file) + raise TypeError( + "status PREPARATION or BROKEN requires before_login_hint: " + file + ) # finally, add the provider global out_all, out_domains, out_ids @@ -172,7 +197,7 @@ def process_file(file): def process_dir(dir): print("processing directory: {}".format(dir), file=sys.stderr) - files = sorted(f for f in dir.iterdir() if f.suffix == '.md') + files = sorted(f for f in dir.iterdir() if f.suffix == ".md") for f in files: process_file(f) @@ -181,31 +206,41 @@ if __name__ == "__main__": if len(sys.argv) < 2: raise SystemExit("usage: update.py DIR_WITH_MD_FILES > data.rs") - out_all = ("// file generated by src/provider/update.py\n\n" - "use crate::provider::Protocol::*;\n" - "use crate::provider::Socket::*;\n" - "use crate::provider::UsernamePattern::*;\n" - "use crate::provider::{\n" - " Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status,\n" - "};\n" - "use std::collections::HashMap;\n\n" - "use once_cell::sync::Lazy;\n\n") + out_all = ( + "// file generated by src/provider/update.py\n\n" + "use crate::provider::Protocol::*;\n" + "use crate::provider::Socket::*;\n" + "use crate::provider::UsernamePattern::*;\n" + "use crate::provider::{\n" + " Config, ConfigDefault, Oauth2Authorizer, Provider, ProviderOptions, Server, Status,\n" + "};\n" + "use std::collections::HashMap;\n\n" + "use once_cell::sync::Lazy;\n\n" + ) process_dir(Path(sys.argv[1])) out_all += "pub(crate) static PROVIDER_DATA: Lazy> = Lazy::new(|| [\n" - out_all += out_domains; + out_all += out_domains out_all += "].iter().copied().collect());\n\n" out_all += "pub(crate) static PROVIDER_IDS: Lazy> = Lazy::new(|| [\n" - out_all += out_ids; + out_all += out_ids out_all += "].iter().copied().collect());\n\n" if len(sys.argv) < 3: now = datetime.datetime.utcnow() else: now = datetime.datetime.fromisoformat(sys.argv[2]) - out_all += "pub static PROVIDER_UPDATED: Lazy = "\ - "Lazy::new(|| chrono::NaiveDate::from_ymd_opt("+str(now.year)+", "+str(now.month)+", "+str(now.day)+").unwrap());\n" + out_all += ( + "pub static PROVIDER_UPDATED: Lazy = " + "Lazy::new(|| chrono::NaiveDate::from_ymd_opt(" + + str(now.year) + + ", " + + str(now.month) + + ", " + + str(now.day) + + ").unwrap());\n" + ) print(out_all) From 50d83ff063214be11d5ddbfb744e55dd3f5a37cb Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 25 Feb 2023 11:00:58 +0000 Subject: [PATCH 46/57] Remove ResyncFolders job --- src/context.rs | 4 +++ src/imap.rs | 18 ++++++++++++++ src/job.rs | 65 +++++++----------------------------------------- src/scheduler.rs | 12 +++++++-- 4 files changed, 41 insertions(+), 58 deletions(-) diff --git a/src/context.rs b/src/context.rs index 1d8dbe847..0f6d721ea 100644 --- a/src/context.rs +++ b/src/context.rs @@ -211,6 +211,9 @@ pub struct InnerContext { /// Set to true if quota update is requested. pub(crate) quota_update_request: AtomicBool, + /// IMAP UID resync request. + pub(crate) resync_request: AtomicBool, + /// Server ID response if ID capability is supported /// and the server returned non-NIL on the inbox connection. /// @@ -371,6 +374,7 @@ impl Context { ratelimit: RwLock::new(Ratelimit::new(Duration::new(60, 0), 6.0)), // Allow to send 6 messages immediately, no more than once every 10 seconds. quota: RwLock::new(None), quota_update_request: AtomicBool::new(false), + resync_request: AtomicBool::new(false), server_id: RwLock::new(None), creation_time: std::time::SystemTime::now(), last_full_folder_scan: Mutex::new(None), diff --git a/src/imap.rs b/src/imap.rs index 9889df69b..f6d36182d 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -904,6 +904,24 @@ impl Imap { info!(context, "Done fetching existing messages."); Ok(()) } + + /// Synchronizes UIDs for all folders. + pub(crate) async fn resync_folders(&mut self, context: &Context) -> Result<()> { + self.prepare(context).await?; + + let all_folders = self + .list_folders(context) + .await + .context("listing folders for resync")?; + for folder in all_folders { + let folder_meaning = get_folder_meaning(&folder); + if folder_meaning != FolderMeaning::Virtual { + self.resync_folder_uids(context, folder.name(), folder_meaning) + .await?; + } + } + Ok(()) + } } impl Session { diff --git a/src/job.rs b/src/job.rs index ff37ea543..25f3814a2 100644 --- a/src/job.rs +++ b/src/job.rs @@ -6,13 +6,14 @@ #![allow(missing_docs)] use std::fmt; +use std::sync::atomic::Ordering; use anyhow::{Context as _, Result}; use deltachat_derive::{FromSql, ToSql}; use rand::{thread_rng, Rng}; use crate::context::Context; -use crate::imap::{get_folder_meaning, FolderMeaning, Imap}; +use crate::imap::Imap; use crate::scheduler::InterruptInfo; use crate::tools::time; @@ -24,7 +25,6 @@ const JOB_RETRIES: u32 = 17; pub enum Status { Finished(Result<()>), RetryNow, - RetryLater, } #[macro_export] @@ -62,10 +62,6 @@ pub enum Action { // Most messages are downloaded automatically on fetch // and do not go through this job. DownloadMsg = 250, - - // UID synchronization is high-priority to make sure correct UIDs - // are used by message moving/deletion. - ResyncFolders = 300, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -142,52 +138,6 @@ impl Job { Ok(()) } - /// Synchronizes UIDs for all folders. - async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status { - if let Err(err) = imap.prepare(context).await { - warn!(context, "could not connect: {:#}", err); - return Status::RetryLater; - } - - let all_folders = match imap.list_folders(context).await { - Ok(v) => v, - Err(e) => { - warn!(context, "Listing folders for resync failed: {:#}", e); - return Status::RetryLater; - } - }; - - let mut any_failed = false; - - for folder in all_folders { - let folder_meaning = get_folder_meaning(&folder); - if folder_meaning == FolderMeaning::Virtual { - continue; - } - if let Err(e) = imap - .resync_folder_uids(context, folder.name(), folder_meaning) - .await - { - warn!(context, "{:#}", e); - any_failed = true; - } - } - - if any_failed { - Status::RetryLater - } else { - Status::Finished(Ok(())) - } - } -} - -/// Delete all pending jobs with the given action. -pub async fn kill_action(context: &Context, action: Action) -> Result<()> { - context - .sql - .execute("DELETE FROM jobs WHERE action=?;", paramsv![action]) - .await?; - Ok(()) } pub(crate) enum Connection<'a> { @@ -211,7 +161,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ }; match try_res { - Status::RetryNow | Status::RetryLater => { + Status::RetryNow => { let tries = job.tries + 1; if tries < JOB_RETRIES { @@ -265,7 +215,6 @@ async fn perform_job_action( info!(context, "begin immediate try {} of job {}", tries, job); let try_res = match job.action { - Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await, Action::DownloadMsg => job.download_msg(context, connection.inbox()).await, }; @@ -287,8 +236,12 @@ fn get_backoff_time_offset(tries: u32) -> i64 { } pub(crate) async fn schedule_resync(context: &Context) -> Result<()> { - kill_action(context, Action::ResyncFolders).await?; - add(context, Job::new(Action::ResyncFolders, 0)).await?; + context.resync_request.store(true, Ordering::Relaxed); + context + .interrupt_inbox(InterruptInfo { + probe_network: false, + }) + .await; Ok(()) } diff --git a/src/scheduler.rs b/src/scheduler.rs index 4ce26c45d..13582d2ea 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -129,13 +129,21 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne info = Default::default(); } None => { - let requested = ctx.quota_update_request.swap(false, Ordering::Relaxed); - if requested { + let quota_requested = ctx.quota_update_request.swap(false, Ordering::Relaxed); + if quota_requested { if let Err(err) = ctx.update_recent_quota(&mut connection).await { warn!(ctx, "Failed to update quota: {:#}.", err); } } + let resync_requested = ctx.resync_request.swap(false, Ordering::Relaxed); + if resync_requested { + if let Err(err) = connection.resync_folders(&ctx).await { + warn!(ctx, "Failed to resync folders: {:#}.", err); + ctx.resync_request.store(true, Ordering::Relaxed); + } + } + maybe_add_time_based_warnings(&ctx).await; match ctx.get_config_i64(Config::LastHousekeeping).await { From f0a2ca78157d23f5fc7283ed13b438b66f5c7ccf Mon Sep 17 00:00:00 2001 From: adbenitez Date: Sun, 26 Feb 2023 22:50:24 -0500 Subject: [PATCH 47/57] send_msg(): return only msg_id --- deltachat-jsonrpc/src/api/mod.rs | 10 ++-------- deltachat-rpc-client/src/deltachat_rpc_client/chat.py | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 6064f476a..b9af596ff 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1512,12 +1512,7 @@ impl CommandApi { Ok(message_id.to_u32()) } - async fn send_msg( - &self, - account_id: u32, - chat_id: u32, - draft: DraftMessage, - ) -> Result<(u32, MessageObject)> { + async fn send_msg(&self, account_id: u32, chat_id: u32, draft: DraftMessage) -> Result { let ctx = self.get_context(account_id).await?; let mut message = Message::new(if let Some(viewtype) = draft.viewtype { viewtype.into() @@ -1556,8 +1551,7 @@ impl CommandApi { let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message) .await? .to_u32(); - let message = MessageObject::from_message_id(&ctx, msg_id).await?; - Ok((msg_id, message)) + Ok(msg_id) } // --------------------------------------------- diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py index aab84d7c7..2d33a300e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/chat.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/chat.py @@ -128,7 +128,7 @@ class Chat: "overrideSenderName": override_sender_name, "quotedMsg": quoted_msg, } - msg_id, _ = await self._rpc.send_msg(self.account.id, self.id, draft) + msg_id = await self._rpc.send_msg(self.account.id, self.id, draft) return Message(self.account, msg_id) async def send_text(self, text: str) -> Message: From 9f81299de05a1ca67b1882cbe26a9e091b3cfaeb Mon Sep 17 00:00:00 2001 From: adbenitez Date: Sun, 26 Feb 2023 23:34:15 -0500 Subject: [PATCH 48/57] rename DraftMessage to MessageData --- deltachat-jsonrpc/src/api/mod.rs | 26 +++++++++++----------- deltachat-jsonrpc/src/api/types/message.rs | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index b9af596ff..576b15246 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -41,7 +41,7 @@ use types::account::Account; use types::chat::FullChat; use types::chat_list::ChatListEntry; use types::contact::ContactObject; -use types::message::DraftMessage; +use types::message::MessageData; use types::message::MessageObject; use types::provider_info::ProviderInfo; use types::webxdc::WebxdcMessageInfo; @@ -1512,31 +1512,31 @@ impl CommandApi { Ok(message_id.to_u32()) } - async fn send_msg(&self, account_id: u32, chat_id: u32, draft: DraftMessage) -> Result { + async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result { let ctx = self.get_context(account_id).await?; - let mut message = Message::new(if let Some(viewtype) = draft.viewtype { + let mut message = Message::new(if let Some(viewtype) = data.viewtype { viewtype.into() - } else if draft.file.is_some() { + } else if data.file.is_some() { Viewtype::File } else { Viewtype::Text }); - if draft.text.is_some() { - message.set_text(draft.text); + if data.text.is_some() { + message.set_text(data.text); } - if draft.html.is_some() { - message.set_html(draft.html); + if data.html.is_some() { + message.set_html(data.html); } - if draft.override_sender_name.is_some() { - message.set_override_sender_name(draft.override_sender_name); + if data.override_sender_name.is_some() { + message.set_override_sender_name(data.override_sender_name); } - if let Some(file) = draft.file { + if let Some(file) = data.file { message.set_file(file, None); } - if let Some((latitude, longitude)) = draft.location { + if let Some((latitude, longitude)) = data.location { message.set_location(latitude, longitude); } - if let Some(id) = draft.quoted_message_id { + if let Some(id) = data.quoted_message_id { message .set_quote( &ctx, diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index f5185fdb6..44d2a1f05 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -505,7 +505,7 @@ impl From for JSONRPCMessageListItem { #[derive(Deserialize, TypeDef)] #[serde(rename_all = "camelCase")] -pub struct DraftMessage { +pub struct MessageData { pub text: Option, pub html: Option, pub viewtype: Option, From a6b2c25d42852b938843ea0c9d7488e4827c34a9 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 27 Feb 2023 08:26:34 +0000 Subject: [PATCH 49/57] Move changelog to the unreleased section and add PR number --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 837754018..c4ccc5956 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Do not block async task executor while decrypting the messages. #4079 ### API-Changes +- jsonrpc: add more advanced API to send a message. #4097 ## 1.110.0 @@ -36,7 +37,6 @@ - Update timestamps in `param` columns with transactions. #4083 ### API-Changes -- jsonrpc: add more advanced API to send a message. ## 1.109.0 From 7bdf79dee50304b3a2874e3846870587a29416ee Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 22 Jan 2023 15:10:25 +0000 Subject: [PATCH 50/57] python: do not shadow variables to fix pyright warnings --- python/src/deltachat/account.py | 6 +++--- python/src/deltachat/chat.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index e6749c7ba..b1b08b43b 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -284,9 +284,9 @@ class Account: :returns: :class:`deltachat.contact.Contact` instance. """ (name, addr) = self.get_contact_addr_and_name(obj, name) - name = as_dc_charpointer(name) - addr = as_dc_charpointer(addr) - contact_id = lib.dc_create_contact(self._dc_context, name, addr) + name_c = as_dc_charpointer(name) + addr_c = as_dc_charpointer(addr) + contact_id = lib.dc_create_contact(self._dc_context, name_c, addr_c) return Contact(self, contact_id) def get_contact(self, obj) -> Optional[Contact]: diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index 77fac073d..e666b6cd4 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -162,8 +162,8 @@ class Chat: :param name: as a unicode string. :returns: True on success, False otherwise """ - name = as_dc_charpointer(name) - return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name)) + name_c = as_dc_charpointer(name) + return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name_c)) def get_color(self): """return the color of the chat. From d1d43e889a6ff030dd58b4fd586ecc020823292c Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 16 Feb 2023 18:24:19 +0000 Subject: [PATCH 51/57] python: add more type annotations --- python/pyproject.toml | 2 +- python/src/deltachat/account.py | 8 ++++---- python/src/deltachat/chat.py | 12 ++++++------ python/src/deltachat/contact.py | 6 +++--- python/src/deltachat/direct_imap.py | 2 +- python/src/deltachat/events.py | 9 ++++++--- python/src/deltachat/message.py | 4 ++-- python/src/deltachat/reactions.py | 4 ++-- python/src/deltachat/testplugin.py | 18 ++++++++++-------- python/src/deltachat/tracker.py | 20 ++++++++++++++++---- python/tests/stress_test_db.py | 4 ++-- python/tests/test_1_online.py | 2 +- 12 files changed, 54 insertions(+), 37 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index ce8dfaf5b..fb846cc8f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -51,7 +51,7 @@ git_describe_command = "git describe --dirty --tags --long --match v*.*" line-length = 120 [tool.ruff] -select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM", "UP004", "UP010", "UP031", "UP032"] +select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM", "UP004", "UP010", "UP031", "UP032", "ANN204"] line-length = 120 [tool.isort] diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index b1b08b43b..c41286ce8 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -121,7 +121,7 @@ class Account: """re-enable logging.""" self._logging = True - def __repr__(self): + def __repr__(self) -> str: return f"" # def __del__(self): @@ -363,12 +363,12 @@ class Account: :returns: list of :class:`deltachat.contact.Contact` objects. """ flags = 0 - query = as_dc_charpointer(query) + query_c = as_dc_charpointer(query) if only_verified: flags |= const.DC_GCL_VERIFIED_ONLY if with_self: flags |= const.DC_GCL_ADD_SELF - dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query), lib.dc_array_unref) + dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query_c), lib.dc_array_unref) return list(iter_array(dc_array, lambda x: Contact(self, x))) def get_fresh_messages(self) -> Generator[Message, None, None]: @@ -767,7 +767,7 @@ class Account: class ScannedQRCode: - def __init__(self, dc_lot): + def __init__(self, dc_lot) -> None: self._dc_lot = dc_lot def is_ask_verifycontact(self): diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index e666b6cd4..6275a61fe 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -24,7 +24,7 @@ class Chat: You obtain instances of it through :class:`deltachat.account.Account`. """ - def __init__(self, account, id) -> None: + def __init__(self, account, id: int) -> None: from .account import Account assert isinstance(account, Account), repr(account) @@ -532,13 +532,13 @@ class Chat: # ------ location streaming API ------------------------------ - def is_sending_locations(self): + def is_sending_locations(self) -> bool: """return True if this chat has location-sending enabled currently. :returns: True if location sending is enabled. """ - return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id) + return bool(lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)) - def enable_sending_locations(self, seconds): + def enable_sending_locations(self, seconds) -> None: """enable sending locations for this chat. all subsequent messages will carry a location with them. @@ -572,7 +572,7 @@ class Chat: class Location: - def __init__(self, latitude, longitude, accuracy, timestamp, marker): + def __init__(self, latitude, longitude, accuracy, timestamp, marker) -> None: assert isinstance(timestamp, datetime) self.latitude = latitude self.longitude = longitude @@ -580,5 +580,5 @@ class Location: self.timestamp = timestamp self.marker = marker - def __eq__(self, other): + def __eq__(self, other) -> bool: return self.__dict__ == other.__dict__ diff --git a/python/src/deltachat/contact.py b/python/src/deltachat/contact.py index f317eb315..f49b91fbc 100644 --- a/python/src/deltachat/contact.py +++ b/python/src/deltachat/contact.py @@ -15,7 +15,7 @@ class Contact: You obtain instances of it through :class:`deltachat.account.Account`. """ - def __init__(self, account, id): + def __init__(self, account, id) -> None: from .account import Account assert isinstance(account, Account), repr(account) @@ -27,10 +27,10 @@ class Contact: return False return self.account._dc_context == other.account._dc_context and self.id == other.id - def __ne__(self, other): + def __ne__(self, other) -> bool: return not self == other - def __repr__(self): + def __repr__(self) -> str: return f"" @property diff --git a/python/src/deltachat/direct_imap.py b/python/src/deltachat/direct_imap.py index d95bd29ab..7584bcec5 100644 --- a/python/src/deltachat/direct_imap.py +++ b/python/src/deltachat/direct_imap.py @@ -191,7 +191,7 @@ class DirectImap: class IdleManager: - def __init__(self, direct_imap): + def __init__(self, direct_imap) -> None: self.direct_imap = direct_imap self.log = direct_imap.account.log # fetch latest messages before starting idle so that it only diff --git a/python/src/deltachat/events.py b/python/src/deltachat/events.py index 006095da0..38f6e2698 100644 --- a/python/src/deltachat/events.py +++ b/python/src/deltachat/events.py @@ -25,12 +25,12 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): class FFIEvent: - def __init__(self, name: str, data1, data2): + def __init__(self, name: str, data1, data2) -> None: self.name = name self.data1 = data1 self.data2 = data2 - def __str__(self): + def __str__(self) -> str: if self.name == "DC_EVENT_INFO": return f"INFO {self.data2}" if self.name == "DC_EVENT_WARNING": @@ -84,7 +84,10 @@ class FFIEventLogger: class FFIEventTracker: - def __init__(self, account, timeout=None): + account: Account + _event_queue: Queue + + def __init__(self, account: Account, timeout=None) -> None: self.account = account self._timeout = timeout self._event_queue = Queue() diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index f0a1cd08b..1c2986728 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -19,7 +19,7 @@ class Message: :class:`deltachat.chat.Chat`. """ - def __init__(self, account, dc_msg): + def __init__(self, account, dc_msg) -> None: self.account = account assert isinstance(self.account._dc_context, ffi.CData) assert isinstance(dc_msg, ffi.CData) @@ -33,7 +33,7 @@ class Message: return False return self.account == other.account and self.id == other.id - def __repr__(self): + def __repr__(self) -> str: c = self.get_sender_contact() typ = "outgoing" if self.is_outgoing() else "incoming" return ( diff --git a/python/src/deltachat/reactions.py b/python/src/deltachat/reactions.py index 1ab2744d0..9838174ce 100644 --- a/python/src/deltachat/reactions.py +++ b/python/src/deltachat/reactions.py @@ -10,14 +10,14 @@ class Reactions: You obtain instances of it through :class:`deltachat.message.Message`. """ - def __init__(self, account, dc_reactions): + def __init__(self, account, dc_reactions) -> None: assert isinstance(account._dc_context, ffi.CData) assert isinstance(dc_reactions, ffi.CData) assert dc_reactions != ffi.NULL self.account = account self._dc_reactions = dc_reactions - def __repr__(self): + def __repr__(self) -> str: return f"" @classmethod diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index ebbb13aaf..c48b1101f 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -9,7 +9,7 @@ import threading import time import weakref from queue import Queue -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Dict, Set import pytest import requests @@ -65,8 +65,8 @@ def pytest_configure(config): # Additionally make the acfactory use a logging/no-logging default. class LoggingAspect: - def __init__(self): - self._accounts = weakref.WeakSet() + def __init__(self) -> None: + self._accounts: weakref.WeakSet = weakref.WeakSet() @deltachat.global_hookimpl def dc_account_init(self, account): @@ -143,10 +143,12 @@ def testprocess(request): class TestProcess: """A pytest session-scoped instance to help with managing "live" account configurations.""" - def __init__(self, pytestconfig): + _addr2files: Dict[str, Dict[pathlib.Path, bytes]] + + def __init__(self, pytestconfig) -> None: self.pytestconfig = pytestconfig self._addr2files = {} - self._configlist = [] + self._configlist: List[Dict[str, str]] = [] def get_liveconfig_producer(self): """provide live account configs, cached on a per-test-process scope @@ -277,10 +279,10 @@ class ACSetup: _configured_events: Queue - def __init__(self, testprocess, init_time): + def __init__(self, testprocess, init_time) -> None: self._configured_events = Queue() - self._account2state = {} - self._imap_cleaned = set() + self._account2state: Dict[Account, str] = {} + self._imap_cleaned: Set[str] = set() self.testprocess = testprocess self.init_time = init_time diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 2dc2ad41b..f8fbc3f8f 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -1,19 +1,25 @@ from queue import Queue from threading import Event +from typing import List, TYPE_CHECKING from .hookspec import Global, account_hookimpl +if TYPE_CHECKING: + from .events import FFIEvent + class ImexFailed(RuntimeError): """Exception for signalling that import/export operations failed.""" class ImexTracker: - def __init__(self): + _imex_events: Queue + + def __init__(self) -> None: self._imex_events = Queue() @account_hookimpl - def ac_process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event: "FFIEvent") -> None: if ffi_event.name == "DC_EVENT_IMEX_PROGRESS": self._imex_events.put(ffi_event.data1) elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN": @@ -50,7 +56,13 @@ class ConfigureFailed(RuntimeError): class ConfigureTracker: ConfigureFailed = ConfigureFailed - def __init__(self, account): + _configure_events: Queue + _smtp_finished: Event + _imap_finished: Event + _ffi_events: List["FFIEvent"] + _progress: Queue + + def __init__(self, account) -> None: self.account = account self._configure_events = Queue() self._smtp_finished = Event() @@ -60,7 +72,7 @@ class ConfigureTracker: self._gm = Global._get_plugin_manager() @account_hookimpl - def ac_process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event: "FFIEvent") -> None: self._ffi_events.append(ffi_event) if ffi_event.name == "DC_EVENT_SMTP_CONNECTED": self._smtp_finished.set() diff --git a/python/tests/stress_test_db.py b/python/tests/stress_test_db.py index a8b942a09..43a04116a 100644 --- a/python/tests/stress_test_db.py +++ b/python/tests/stress_test_db.py @@ -77,7 +77,7 @@ class ReportType: class AutoReplier: - def __init__(self, account, log, num_send, num_bigfiles, report_func): + def __init__(self, account, log, num_send, num_bigfiles, report_func) -> None: self.account = account self._log = log self.report_func = report_func @@ -90,7 +90,7 @@ class AutoReplier: self._thread.setDaemon(True) self._thread.start() - def log(self, message): + def log(self, message) -> None: self._log(f"{self.addr} {message}") def thread_stats(self): diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index c25fcb85b..a807bbe3c 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1604,7 +1604,7 @@ def test_add_remove_member_remote_events(acfactory, lp): in_list = queue.Queue() class EventHolder: - def __init__(self, **kwargs): + def __init__(self, **kwargs) -> None: self.__dict__.update(kwargs) class InPlugin: From c79ded140621ce03ea1f910f42c3b1796a8e2e3f Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 27 Feb 2023 14:36:57 +0000 Subject: [PATCH 52/57] Fixes for `get_webxdc_blob` documentation --- src/webxdc.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webxdc.rs b/src/webxdc.rs index de786ea99..12d9abd6b 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -670,8 +670,10 @@ impl Message { Ok(archive) } - /// Return file form inside an archive. + /// Return file from inside an archive. /// Currently, this works only if the message is an webxdc instance. + /// + /// `name` is the filename within the archive, e.g. `index.html`. pub async fn get_webxdc_blob(&self, context: &Context, name: &str) -> Result> { ensure!(self.viewtype == Viewtype::Webxdc, "No webxdc instance."); From fc019de18ccfd52a9373ac284170238e440e7cb2 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Mon, 27 Feb 2023 18:46:13 +0100 Subject: [PATCH 53/57] jsonrpc: add get webxdc blob API `getWebxdcBlob` (#4070) * jsonrpc: add get webxdc blob API `getWebxdcBlob` * add info about path * format --- CHANGELOG.md | 2 +- Cargo.lock | 43 +++++++++++++++++++++----------- deltachat-jsonrpc/Cargo.toml | 5 ++-- deltachat-jsonrpc/src/api/mod.rs | 17 +++++++++++++ 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4ccc5956..9f6dfa314 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ ### API-Changes - jsonrpc: add more advanced API to send a message. #4097 - +- jsonrpc: add get webxdc blob API `getWebxdcBlob` #4070 ## 1.110.0 diff --git a/Cargo.lock b/Cargo.lock index 1a6765405..55d00a08a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,13 +234,13 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.17" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" +checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" dependencies = [ "async-trait", "axum-core", - "base64 0.13.1", + "base64 0.21.0", "bitflags", "bytes", "futures-util", @@ -253,10 +253,12 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", + "rustversion", "serde", "serde_json", + "serde_path_to_error", "serde_urlencoded", - "sha-1", + "sha1", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -268,9 +270,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.2.9" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" +checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" dependencies = [ "async-trait", "bytes", @@ -278,6 +280,7 @@ dependencies = [ "http", "http-body", "mime", + "rustversion", "tower-layer", "tower-service", ] @@ -910,6 +913,7 @@ dependencies = [ "anyhow", "async-channel", "axum", + "base64 0.21.0", "deltachat", "env_logger 0.10.0", "futures", @@ -2094,9 +2098,9 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "matchit" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" [[package]] name = "md-5" @@ -3154,6 +3158,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.1" @@ -3556,9 +3569,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", @@ -3756,9 +3769,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "tungstenite" -version = "0.17.3" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", @@ -3767,7 +3780,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha-1", + "sha1", "thiserror", "url", "utf-8", @@ -4225,9 +4238,9 @@ dependencies = [ [[package]] name = "yerpc" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d383dfbc1842f5b915a01deeaea3523eef8653fcd734f79f6c696d51521c70f" +checksum = "b6a0257f42e6bdc187f37074723b6094da3502cee21ae517b3c54d2c37d506e7" dependencies = [ "anyhow", "async-channel", diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 7b9a26719..d445da858 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -21,14 +21,15 @@ log = "0.4" async-channel = { version = "1.8.0" } futures = { version = "0.3.26" } serde_json = "1.0.91" -yerpc = { version = "^0.4.0", features = ["anyhow_expose"] } +yerpc = { version = "0.4.3", features = ["anyhow_expose"] } typescript-type-def = { version = "0.5.5", features = ["json_value"] } tokio = { version = "1.25.0" } sanitize-filename = "0.4" walkdir = "2.3.2" +base64 = "0.21" # optional dependencies -axum = { version = "0.5.9", optional = true, features = ["ws"] } +axum = { version = "0.6.6", optional = true, features = ["ws"] } env_logger = { version = "0.10.0", optional = true } [dev-dependencies] diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 576b15246..0f466d1d4 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1463,6 +1463,23 @@ impl CommandApi { WebxdcMessageInfo::get_for_message(&ctx, MsgId::new(instance_msg_id)).await } + /// Get blob encoded as base64 from a webxdc message + /// + /// path is the path of the file within webxdc archive + async fn get_webxdc_blob( + &self, + account_id: u32, + instance_msg_id: u32, + path: String, + ) -> Result { + let ctx = self.get_context(account_id).await?; + let message = Message::load_from_db(&ctx, MsgId::new(instance_msg_id)).await?; + let blob = message.get_webxdc_blob(&ctx, &path).await?; + + use base64::{engine::general_purpose, Engine as _}; + Ok(general_purpose::STANDARD_NO_PAD.encode(blob)) + } + /// Forward messages to another chat. /// /// All types of messages can be forwarded, From e767d84bea6265cff4dc3d27b27cf8074e26759e Mon Sep 17 00:00:00 2001 From: iequidoo Date: Sat, 25 Feb 2023 08:12:52 -0300 Subject: [PATCH 54/57] Switch to DEFERRED transactions We do not make all transactions [IMMEDIATE](https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions) for more parallelism -- at least read transactions can be made DEFERRED to run in parallel w/o any drawbacks. But if we make write transactions DEFERRED also w/o any external locking, then they are upgraded from read to write ones on the first write statement. This has some drawbacks: - If there are other write transactions, we block the thread and the db connection until upgraded. Also if some reader comes then, it has to get next, less used connection with a worse per-connection page cache. - If a transaction is blocked for more than busy_timeout, it fails with SQLITE_BUSY. - Configuring busy_timeout is not the best way to manage transaction timeouts, we would prefer it to be integrated with Rust/tokio asyncs. Moreover, SQLite implements waiting using sleeps. - If upon a successful upgrade to a write transaction the db has been modified by another one, the transaction has to be rolled back and retried. It is an extra work in terms of CPU/battery. - Maybe minor, but we lose some fairness in servicing write transactions, i.e. we service them in the order of the first write statement, not in the order they come. The only pro of making write transactions DEFERRED w/o the external locking is some parallelism between them. Also we have an option to make write transactions IMMEDIATE, also w/o the external locking. But then the most of cons above are still valid. Instead, if we perform all write transactions under an async mutex, the only cons is losing some parallelism for write transactions. --- CHANGELOG.md | 1 + src/imex.rs | 2 +- src/location.rs | 2 +- src/receive_imf.rs | 2 +- src/sql.rs | 79 +++++++++++++++++++++++++++++++++++----------- src/webxdc.rs | 1 + 6 files changed, 66 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f6dfa314..605aa175a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Set minimum TLS version to 1.2. #4096 - Run `cargo-deny` in CI. #4101 - Check provider database with CI. #4099 +- Switch to DEFERRED transactions #4100 ### Fixes - Do not block async task executor while decrypting the messages. #4079 diff --git a/src/imex.rs b/src/imex.rs index ffc2589dd..23b876ba7 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -540,7 +540,7 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res context .sql - .call(|conn| { + .call_write(|conn| { if let Err(err) = conn.execute("VACUUM", params![]) { info!(context, "Vacuum failed, exporting anyway: {:#}.", err); } diff --git a/src/location.rs b/src/location.rs index 6698dea3b..7bb381497 100644 --- a/src/location.rs +++ b/src/location.rs @@ -603,7 +603,7 @@ pub(crate) async fn save( context .sql - .call(|conn| { + .call_write(|conn| { let mut stmt_test = conn .prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?; let mut stmt_insert = conn.prepare_cached(stmt_insert)?; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index e8200ad62..7ee1b8a00 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1151,7 +1151,7 @@ async fn add_parts( let row_id = context .sql - .call(|conn| { + .call_write(|conn| { let mut stmt = conn.prepare_cached( r#" INSERT INTO msgs diff --git a/src/sql.rs b/src/sql.rs index 07884f875..afc77ef1b 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -5,8 +5,8 @@ use std::convert::TryFrom; use std::path::{Path, PathBuf}; use anyhow::{bail, Context as _, Result}; -use rusqlite::{self, config::DbConfig, Connection, OpenFlags, TransactionBehavior}; -use tokio::sync::RwLock; +use rusqlite::{self, config::DbConfig, Connection, OpenFlags}; +use tokio::sync::{Mutex, MutexGuard, RwLock}; use crate::blob::BlobObject; use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon}; @@ -56,6 +56,11 @@ pub struct Sql { /// Database file path pub(crate) dbfile: PathBuf, + /// Write transaction mutex. + /// + /// See [`Self::write_lock`]. + write_mtx: Mutex<()>, + /// SQL connection pool. pool: RwLock>, @@ -72,6 +77,7 @@ impl Sql { pub fn new(dbfile: PathBuf) -> Sql { Self { dbfile, + write_mtx: Mutex::new(()), pool: Default::default(), is_encrypted: Default::default(), config_cache: Default::default(), @@ -130,7 +136,7 @@ impl Sql { .with_context(|| format!("path {path:?} is not valid unicode"))? .to_string(); let res = self - .call(move |conn| { + .call_write(move |conn| { // Check that backup passphrase is correct before resetting our database. conn.execute( "ATTACH DATABASE ? AS backup KEY ?", @@ -299,10 +305,40 @@ impl Sql { } } - /// Allocates a connection and calls given function with the connection. + /// Locks the write transactions mutex. + /// We do not make all transactions + /// [IMMEDIATE](https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions) + /// for more parallelism -- at least read transactions can be made DEFERRED to run in parallel + /// w/o any drawbacks. But if we make write transactions DEFERRED also w/o any external locking, + /// then they are upgraded from read to write ones on the first write statement. This has some + /// drawbacks: + /// - If there are other write transactions, we block the thread and the db connection until + /// upgraded. Also if some reader comes then, it has to get next, less used connection with a + /// worse per-connection page cache. + /// - If a transaction is blocked for more than busy_timeout, it fails with SQLITE_BUSY. + /// - Configuring busy_timeout is not the best way to manage transaction timeouts, we would + /// prefer it to be integrated with Rust/tokio asyncs. Moreover, SQLite implements waiting + /// using sleeps. + /// - If upon a successful upgrade to a write transaction the db has been modified by another + /// one, the transaction has to be rolled back and retried. It is an extra work in terms of + /// CPU/battery. + /// - Maybe minor, but we lose some fairness in servicing write transactions, i.e. we service + /// them in the order of the first write statement, not in the order they come. + /// The only pro of making write transactions DEFERRED w/o the external locking is some + /// parallelism between them. Also we have an option to make write transactions IMMEDIATE, also + /// w/o the external locking. But then the most of cons above are still valid. Instead, if we + /// perform all write transactions under an async mutex, the only cons is losing some + /// parallelism for write transactions. + pub async fn write_lock(&self) -> MutexGuard<'_, ()> { + self.write_mtx.lock().await + } + + /// Allocates a connection and calls `function` with the connection. If `function` does write + /// queries, either a lock must be taken first using `write_lock()` or `call_write()` used + /// instead. /// /// Returns the result of the function. - pub async fn call<'a, F, R>(&'a self, function: F) -> Result + async fn call<'a, F, R>(&'a self, function: F) -> Result where F: 'a + FnOnce(&mut Connection) -> Result + Send, R: Send + 'static, @@ -314,13 +350,26 @@ impl Sql { Ok(res) } - /// Execute the given query, returning the number of affected rows. + /// Allocates a connection and calls given function, assuming it does write queries, with the + /// connection. + /// + /// Returns the result of the function. + pub async fn call_write<'a, F, R>(&'a self, function: F) -> Result + where + F: 'a + FnOnce(&mut Connection) -> Result + Send, + R: Send + 'static, + { + let _lock = self.write_lock().await; + self.call(function).await + } + + /// Execute `query` assuming it is a write query, returning the number of affected rows. pub async fn execute( &self, query: &str, params: impl rusqlite::Params + Send, ) -> Result { - self.call(move |conn| { + self.call_write(move |conn| { let res = conn.execute(query, params)?; Ok(res) }) @@ -329,7 +378,7 @@ impl Sql { /// Executes the given query, returning the last inserted row ID. pub async fn insert(&self, query: &str, params: impl rusqlite::Params + Send) -> Result { - self.call(move |conn| { + self.call_write(move |conn| { conn.execute(query, params)?; Ok(conn.last_insert_rowid()) }) @@ -390,23 +439,17 @@ impl Sql { .await } - /// Execute the function inside a transaction. + /// Execute the function inside a transaction assuming that it does write queries. /// /// If the function returns an error, the transaction will be rolled back. If it does not return an /// error, the transaction will be committed. - /// - /// Transactions started use IMMEDIATE behavior - /// rather than default DEFERRED behavior - /// to avoid "database is busy" errors - /// which may happen when DEFERRED transaction - /// is attempted to be promoted to a write transaction. pub async fn transaction(&self, callback: G) -> Result where H: Send + 'static, G: Send + FnOnce(&mut rusqlite::Transaction<'_>) -> Result, { - self.call(move |conn| { - let mut transaction = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + self.call_write(move |conn| { + let mut transaction = conn.transaction()?; let ret = callback(&mut transaction); match ret { @@ -617,7 +660,7 @@ fn new_connection(path: &Path, passphrase: &str) -> Result { conn.execute_batch( "PRAGMA cipher_memory_security = OFF; -- Too slow on Android PRAGMA secure_delete=on; - PRAGMA busy_timeout = 60000; -- 60 seconds + PRAGMA busy_timeout = 0; -- fail immediately PRAGMA temp_store=memory; -- Avoid SQLITE_IOERR_GETTEMPPATH errors on Android PRAGMA foreign_keys=on; ", diff --git a/src/webxdc.rs b/src/webxdc.rs index 12d9abd6b..154fb61a8 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -431,6 +431,7 @@ impl Context { async fn pop_smtp_status_update( &self, ) -> Result> { + let _lock = self.sql.write_lock().await; let res = self .sql .query_row_optional( From b149df19934aa0023b11371b7f79ea5e99f4bcfb Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 28 Feb 2023 13:35:02 -0300 Subject: [PATCH 55/57] test_{delete,trash}_multiple_messages: Continue waiting for events if not all messages disappeared --- python/tests/test_1_online.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index a807bbe3c..280210c6a 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1987,13 +1987,16 @@ def test_delete_multiple_messages(acfactory, lp): lp.sec("ac2: deleting all messages except third") assert len(to_delete) == len(texts) - 1 ac2.delete_messages(to_delete) - ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") - - ac2._evtracker.get_info_contains("close/expunge succeeded") lp.sec("ac2: test that only one message is left") - ac2.direct_imap.select_config_folder("inbox") - assert len(ac2.direct_imap.get_all_messages()) == 1 + while 1: + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") + ac2._evtracker.get_info_contains("close/expunge succeeded") + ac2.direct_imap.select_config_folder("inbox") + nr_msgs = len(ac2.direct_imap.get_all_messages()) + assert nr_msgs > 0 + if nr_msgs == 1: + break def test_trash_multiple_messages(acfactory, lp): @@ -2017,11 +2020,15 @@ def test_trash_multiple_messages(acfactory, lp): lp.sec("ac2: deleting all messages except second") assert len(to_delete) == len(texts) - 1 ac2.delete_messages(to_delete) - ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") lp.sec("ac2: test that only one message is left") - ac2.direct_imap.select_config_folder("inbox") - assert len(ac2.direct_imap.get_all_messages()) == 1 + while 1: + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + ac2.direct_imap.select_config_folder("inbox") + nr_msgs = len(ac2.direct_imap.get_all_messages()) + assert nr_msgs > 0 + if nr_msgs == 1: + break def test_configure_error_msgs_wrong_pw(acfactory): From 2dc04220b8a0cf526c9aee1247f30d93d59a40c5 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 27 Feb 2023 23:01:22 +0000 Subject: [PATCH 56/57] Simplify dc_jsonrpc_init --- deltachat-ffi/src/lib.rs | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index f16ca2916..1aa22fb92 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4607,33 +4607,22 @@ mod jsonrpc { return ptr::null_mut(); } - let cmd_api = - deltachat_jsonrpc::api::CommandApi::from_arc((*account_manager).inner.clone()); + let account_manager = &*account_manager; + let events = block_on(account_manager.read()).get_event_emitter(); + let cmd_api = deltachat_jsonrpc::api::CommandApi::from_arc(account_manager.inner.clone()); let (request_handle, receiver) = RpcClient::new(); - let request_handle2 = request_handle.clone(); - let handle = RpcSession::new(request_handle, cmd_api); + let handle = RpcSession::new(request_handle.clone(), cmd_api); - let events = block_on({ - async { - let am = (*account_manager).inner.clone(); - let ev = am.read().await.get_event_emitter(); - drop(am); - ev - } - }); - - let event_thread = spawn({ - async move { - while let Some(event) = events.recv().await { - let event = event_to_json_rpc_notification(event); - request_handle2 - .send_notification("event", Some(event)) - .await?; - } - let res: Result<(), anyhow::Error> = Ok(()); - res + let event_thread = spawn(async move { + while let Some(event) = events.recv().await { + let event = event_to_json_rpc_notification(event); + request_handle + .send_notification("event", Some(event)) + .await?; } + let res: Result<(), anyhow::Error> = Ok(()); + res }); let instance = dc_jsonrpc_instance_t { From 4e166b1b4a249fe5e6e13cae684f29fc1d23f48b Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 1 Mar 2023 22:42:03 +0000 Subject: [PATCH 57/57] Run `cargo update` --- Cargo.lock | 859 +++++++++++++++++++++++++++-------------------------- 1 file changed, 443 insertions(+), 416 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55d00a08a..79224d433 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,16 +40,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "once_cell", "version_check 0.9.4", ] [[package]] name = "aho-corasick" -version = "0.7.18" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -62,9 +62,9 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "android_system_properties" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -86,9 +86,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "ascii_utils" @@ -123,7 +123,7 @@ dependencies = [ [[package]] name = "async-imap" version = "0.6.0" -source = "git+https://github.com/async-email/async-imap?branch=master#85ff7a3d9d71a3715354fabf2fc1a8d047b5710e" +source = "git+https://github.com/async-email/async-imap?branch=master#90270474a5a494669e7c63c13471d189afdc98ae" dependencies = [ "async-channel", "async-native-tls", @@ -133,7 +133,7 @@ dependencies = [ "futures", "imap-proto", "log", - "nom 7.1.1", + "nom 7.1.3", "once_cell", "ouroboros", "pin-utils", @@ -175,7 +175,7 @@ dependencies = [ "futures", "hostname", "log", - "nom 7.1.1", + "nom 7.1.3", "pin-project", "thiserror", "tokio", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", @@ -234,9 +234,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.7" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fb79c228270dcf2426e74864cabc94babb5dbab01a4314e702d2f16540e1591" +checksum = "6137c6234afb339e75e764c866e3594900f0211e1315d33779f269bbe2ec6967" dependencies = [ "async-trait", "axum-core", @@ -326,9 +326,9 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "base64ct" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdca834647821e0b13d9539a8634eb62d3501b6b6c2cec1722786ee6671b851" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bitfield" @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ "generic-array", ] @@ -413,9 +413,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.11.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5377c8865e74a160d21f29c2d40669f53286db6eab59b88540cbb12ffc8b835" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" [[package]] name = "byteorder" @@ -425,9 +425,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "cast" @@ -446,9 +446,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.73" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfb-mode" @@ -550,15 +550,25 @@ dependencies = [ [[package]] name = "clipboard-win" -version = "4.4.2" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219" +checksum = "7191c27c2357d9b7ef96baac1773290d4ca63b24205b82a3fd8a0637afcf0362" dependencies = [ "error-code", "str-buf", "winapi", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -584,18 +594,18 @@ checksum = "82a90734b3d5dcf656e7624cca6bce9c3a90ee11f900e80141a7427ccfb3d317" [[package]] name = "concurrent-queue" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" +checksum = "c278839b831783b70278b14df4d45e1beb1aad306c07bb796637de9a0e323e8e" dependencies = [ "crossbeam-utils", ] [[package]] name = "const-oid" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722e23542a15cea1f65d4a1419c4cfd7a26706c70871a13a04238ca3f40f1661" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" [[package]] name = "convert_case" @@ -621,9 +631,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] @@ -683,9 +693,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -693,9 +703,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -704,23 +714,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.10" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", "memoffset", - "once_cell", "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ "cfg-if", "crossbeam-utils", @@ -728,12 +737,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.11" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -759,6 +767,50 @@ dependencies = [ "zeroize", ] +[[package]] +name = "cxx" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "darling" version = "0.13.4" @@ -771,12 +823,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02" +checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" dependencies = [ - "darling_core 0.14.1", - "darling_macro 0.14.1", + "darling_core 0.14.3", + "darling_macro 0.14.3", ] [[package]] @@ -795,9 +847,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f" +checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" dependencies = [ "fnv", "ident_case", @@ -820,11 +872,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5" +checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" dependencies = [ - "darling_core 0.14.1", + "darling_core 0.14.3", "quote", "syn", ] @@ -900,7 +952,7 @@ dependencies = [ "tokio-io-timeout", "tokio-stream", "tokio-tar", - "toml 0.7.1", + "toml 0.7.2", "trust-dns-resolver", "url", "uuid 1.3.0", @@ -986,9 +1038,9 @@ dependencies = [ [[package]] name = "der" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" dependencies = [ "const-oid", "pem-rfc7468", @@ -1010,7 +1062,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ - "darling 0.14.1", + "darling 0.14.3", "proc-macro2", "quote", "syn", @@ -1046,11 +1098,11 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.2", + "block-buffer 0.10.3", "const-oid", "crypto-common", ] @@ -1098,9 +1150,9 @@ dependencies = [ [[package]] name = "ed25519" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9c280362032ea4203659fc489832d0204ef09f247a0506f170dafcac08c369" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" dependencies = [ "signature", ] @@ -1121,9 +1173,9 @@ dependencies = [ [[package]] name = "either" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "email" @@ -1220,9 +1272,9 @@ checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -1359,34 +1411,34 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] [[package]] name = "fd-lock" -version = "3.0.6" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517" +checksum = "8ef1a30ae415c3a691a4f41afddc2dbcd6d70baf338368d85ebc1e8ed92cedb9" dependencies = [ "cfg-if", - "rustix 0.35.7", - "windows-sys 0.36.1", + "rustix", + "windows-sys 0.45.0", ] [[package]] name = "filetime" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys 0.36.1", + "windows-sys 0.45.0", ] [[package]] @@ -1560,9 +1612,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", @@ -1581,15 +1633,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "h2" -version = "0.3.13" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -1630,9 +1682,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -1652,6 +1704,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -1671,9 +1729,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -1699,9 +1757,9 @@ checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" [[package]] name = "httparse" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -1751,9 +1809,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.20" +version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ "bytes", "futures-channel", @@ -1788,17 +1846,28 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.46" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" dependencies = [ "android_system_properties", "core-foundation-sys", + "iana-time-zone-haiku", "js-sys", "wasm-bindgen", "winapi", ] +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1844,18 +1913,18 @@ dependencies = [ [[package]] name = "imap-proto" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f256a8086d5a408348cddb97d8a07e7d10f861067c497e850e67c9aeda014fda" +checksum = "f73b1b63179418b20aa81002d616c5f21b4ba257da9bca6989ea64dc573933e0" dependencies = [ - "nom 7.1.1", + "nom 7.1.3", ] [[package]] name = "indexmap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -1881,64 +1950,58 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "0.7.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c3f4eff5495aee4c0399d7b6a0dc2b6e81be84242ffbfcf253ebacccc1d0cb" - -[[package]] -name = "io-lifetimes" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] name = "ipconfig" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" +checksum = "bd302af1b90f2463a98fa5ad469fc212c8e3175a41c3068601bfa2727591c5be" dependencies = [ "socket2", "widestring", "winapi", - "winreg 0.7.0", + "winreg", ] [[package]] name = "ipnet" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330" +checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes 1.0.3", - "rustix 0.36.4", - "windows-sys 0.42.0", + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.45.0", ] [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "jpeg-decoder" @@ -1948,9 +2011,9 @@ checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" [[package]] name = "js-sys" -version = "0.3.59" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -1966,9 +2029,12 @@ dependencies = [ [[package]] name = "keccak" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9b7d56ba4a8344d6be9729995e6b06f928af29998cdf79fe390cbf6b1fee838" +checksum = "3afef3b6eff9ce9d8ff9b3601125eec7f0c8cbac7abd14f355d053fa56c98768" +dependencies = [ + "cpufeatures", +] [[package]] name = "lazy_static" @@ -2011,9 +2077,9 @@ checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libm" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libsqlite3-sys" @@ -2027,6 +2093,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "link-cplusplus" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -2035,21 +2110,15 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" - -[[package]] -name = "linux-raw-sys" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ "autocfg", "scopeguard", @@ -2092,9 +2161,9 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matchit" @@ -2108,7 +2177,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -2119,9 +2188,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.6.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -2149,14 +2218,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2216,28 +2285,19 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", ] -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", -] - [[package]] name = "num-bigint-dig" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566d173b2f9406afbc5510a90925d5a2cd80cae4605631f1212303df265de011" +checksum = "2399c9463abc5f909349d8aa9ba080e0b88b3ce2885389b60b993f39b1a56905" dependencies = [ "byteorder", "lazy_static", @@ -2316,18 +2376,18 @@ dependencies = [ [[package]] name = "object" -version = "0.30.0" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239da7f290cfa979f43f85a8efeee9a8a76d0827c356d37f9d3d7254d6b537fb" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "oorandom" @@ -2343,9 +2403,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.41" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if", @@ -2375,18 +2435,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.0+1.1.1t" +version = "111.25.1+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" +checksum = "1ef9a9cc6ea7d9d5e7c4a913dc4b48d0e359eddf01af1dfec96ba7064b4aba10" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.75" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -2415,9 +2475,9 @@ checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "ouroboros" -version = "0.15.2" +version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7425ea87a1e31df63a27b6d31e21a35a9003268032a876465e8d43c2364b0de2" +checksum = "e1358bd1558bd2a083fed428ffeda486fbfb323e698cdda7794259d592ca72db" dependencies = [ "aliasable", "ouroboros_macro", @@ -2425,9 +2485,9 @@ dependencies = [ [[package]] name = "ouroboros_macro" -version = "0.15.2" +version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "734aa7a4a6390b162112523cac2923a18e4f23b917880a68c826bf6e8bf48f06" +checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" dependencies = [ "Inflector", "proc-macro-error", @@ -2454,15 +2514,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys 0.36.1", + "windows-sys 0.45.0", ] [[package]] @@ -2500,7 +2560,7 @@ dependencies = [ "crc24", "derive_builder", "des", - "digest 0.10.5", + "digest 0.10.6", "ed25519-dalek", "flate2", "generic-array", @@ -2527,18 +2587,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", @@ -2581,15 +2641,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "plotters" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9428003b84df1496fb9d6eeee9c5f8145cb41ca375eb0dad204328888832811f" +checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97" dependencies = [ "num-traits", "plotters-backend", @@ -2606,9 +2666,9 @@ checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142" [[package]] name = "plotters-svg" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0918736323d1baff32ee0eade54984f6f201ad7e97d5cfb5d6ab4a358529615" +checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f" dependencies = [ "plotters-backend", ] @@ -2627,9 +2687,9 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pretty_env_logger" @@ -2667,18 +2727,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] [[package]] name = "proptest" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5" +checksum = "29f1b898011ce9595050a68e60f90bad083ff2987a695a42357134c8381fba70" dependencies = [ "bitflags", "byteorder", @@ -2689,6 +2749,7 @@ dependencies = [ "rand_chacha 0.3.1", "rand_xorshift", "regex-syntax", + "unarray", ] [[package]] @@ -2729,9 +2790,9 @@ dependencies = [ [[package]] name = "quoted_printable" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20f14e071918cbeefc5edc986a7aa92c425dae244e003a35e1cdddb5ca39b5cb" +checksum = "a24039f627d8285853cc90dcddf8c1ebfaa91f834566948872b225b9a28ed1b6" [[package]] name = "radix_trie" @@ -2764,7 +2825,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2784,7 +2845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2798,11 +2859,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", ] [[package]] @@ -2820,7 +2881,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2829,21 +2890,19 @@ version = "1.0.0" [[package]] name = "rayon" -version = "1.5.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" dependencies = [ - "autocfg", - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.9.3" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -2866,7 +2925,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "redox_syscall", "thiserror", ] @@ -2884,18 +2943,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "reqwest" @@ -2931,7 +2981,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.10.1", + "winreg", ] [[package]] @@ -2950,24 +3000,24 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", ] [[package]] name = "rsa" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0ecc3307be66bfb3574577895555bacfb9a37a8d5cd959444b72ff02495c618" +checksum = "094052d5470cbcef561cb848a7209968c9f12dfa6d668f4bca048ac5de51099c" dependencies = [ "byteorder", - "digest 0.10.5", + "digest 0.10.6", "num-bigint-dig", "num-integer", "num-iter", "num-traits", "pkcs1", "pkcs8", - "rand_core 0.6.3", + "rand_core 0.6.4", "signature", "smallvec", "subtle", @@ -3002,37 +3052,23 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustix" -version = "0.35.7" +version = "0.36.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51cc38aa10f6bbb377ed28197aa052aa4e2b762c22be9d3153d01822587e787" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" dependencies = [ "bitflags", "errno", - "io-lifetimes 0.7.2", + "io-lifetimes", "libc", - "linux-raw-sys 0.0.46", - "windows-sys 0.36.1", -] - -[[package]] -name = "rustix" -version = "0.36.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes 1.0.3", - "libc", - "linux-raw-sys 0.1.3", - "windows-sys 0.42.0", + "linux-raw-sys", + "windows-sys 0.45.0", ] [[package]] name = "rustversion" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "rustyline" @@ -3059,9 +3095,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "safemem" @@ -3090,12 +3126,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys 0.42.0", ] [[package]] @@ -3105,10 +3140,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] -name = "security-framework" -version = "2.6.1" +name = "scratch" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" + +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ "bitflags", "core-foundation", @@ -3119,9 +3160,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -3149,9 +3190,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ "itoa", "ryu", @@ -3196,7 +3237,7 @@ checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -3207,7 +3248,7 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -3231,7 +3272,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.5", + "digest 0.10.6", ] [[package]] @@ -3240,15 +3281,15 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdf0c33fae925bdc080598b84bc15c55e7b9a4a43b3c704da051f977469691c9" dependencies = [ - "digest 0.10.5", + "digest 0.10.6", "keccak", ] [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -3259,15 +3300,15 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest 0.10.5", - "rand_core 0.6.3", + "digest 0.10.6", + "rand_core 0.6.4", ] [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -3286,9 +3327,9 @@ checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" [[package]] name = "socket2" -version = "0.4.4" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" dependencies = [ "libc", "winapi", @@ -3361,15 +3402,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -3378,9 +3419,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "synstructure" @@ -3402,23 +3443,22 @@ checksum = "6aaa6f5d645d1dae4cd0286e9f8bf15b75a31656348e5e106eb1a940abd34b63" [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] @@ -3456,9 +3496,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -3486,15 +3526,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -3507,7 +3547,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -3522,9 +3562,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -3533,9 +3573,9 @@ dependencies = [ [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -3543,9 +3583,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", @@ -3581,9 +3621,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.3" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -3604,9 +3644,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772c1426ab886e7362aedf4abc9c0d1348a979517efedfc25862944d10137af0" +checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6" dependencies = [ "serde", "serde_spanned", @@ -3625,15 +3665,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.1" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90a238ee2e6ede22fb95350acc78e21dc40da00bb66c0334bde83de4ed89424e" +checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825" dependencies = [ "indexmap", - "nom8", "serde", "serde_spanned", "toml_datetime", + "winnow", ] [[package]] @@ -3654,9 +3694,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.3.4" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" +checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658" dependencies = [ "bitflags", "bytes", @@ -3685,9 +3725,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", @@ -3698,9 +3738,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", @@ -3709,9 +3749,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", ] @@ -3763,9 +3803,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "tungstenite" @@ -3797,15 +3837,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "typescript-type-def" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947b91d2fe9ec02a6749b8b645541f16f527e2ea88a60b3f774eca26fd657325" +checksum = "4e6b74ffbd5684d318252bb7182051df8c4ecc098b542f63fddf792e7f42aa02" dependencies = [ "serde_json", "typescript-type-def-derive", @@ -3813,9 +3853,9 @@ dependencies = [ [[package]] name = "typescript-type-def-derive" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c1bfe689e4067733530495b04959b00f05cd95f038bed59af4fc70b3e26240" +checksum = "b10a4f5dd87c279f90beef31edb7055bfd1ceb66e73148de107a5c9005e9f864" dependencies = [ "darling 0.13.4", "ident_case", @@ -3826,16 +3866,22 @@ dependencies = [ ] [[package]] -name = "unicode-bidi" -version = "0.3.8" +name = "unarray" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-bidi" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-linebreak" @@ -3849,18 +3895,18 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854cbdc4f7bc6ae19c820d44abdc3277ac3e1b2b93db20a636825d9322fb60e6" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -3870,9 +3916,9 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "url" @@ -3903,7 +3949,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", ] [[package]] @@ -3912,7 +3958,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" dependencies = [ - "getrandom 0.2.7", + "getrandom 0.2.8", "serde", ] @@ -3981,9 +4027,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3991,9 +4037,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -4006,9 +4052,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -4018,9 +4064,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4028,9 +4074,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -4041,15 +4087,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.59" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -4098,19 +4144,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" -dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", -] - [[package]] name = "windows-sys" version = "0.42.0" @@ -4118,93 +4151,87 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc 0.42.0", - "windows_i686_gnu 0.42.0", - "windows_i686_msvc 0.42.0", - "windows_x86_64_gnu 0.42.0", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", "windows_x86_64_gnullvm", - "windows_x86_64_msvc 0.42.0", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.0" +name = "winnow" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" - -[[package]] -name = "winreg" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" dependencies = [ - "winapi", + "memchr", ] [[package]] @@ -4260,12 +4287,12 @@ dependencies = [ [[package]] name = "yerpc_derive" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd5e0da8e7a58236986d9032bad52f30995999f94cd39142aa1144b5875e8bce" +checksum = "6bd53ff9053698697b92c2535bf7ecb983fd5d546d690b7c725e5070d6d9a620" dependencies = [ "convert_case", - "darling 0.14.1", + "darling 0.14.3", "proc-macro2", "quote", "syn", @@ -4282,9 +4309,9 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" +checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2", "quote",