From 9d77f65f0ec6e17c37c72eb74b652408add40ba9 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 5 Dec 2024 02:48:49 +0000 Subject: [PATCH 01/20] docs: update links to Node.js bindings in the README CFFI and napi.rs bindings are not maintained, JSON-RPC client should be used with deltachat-rpc-server instead. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d9ba4ce15..1e8d6f437 100644 --- a/README.md +++ b/README.md @@ -177,8 +177,8 @@ Language bindings are available for: - **C** \[[πŸ“‚ source](./deltachat-ffi) | [πŸ“š docs](https://c.delta.chat)\] - **Node.js** - - over cffi: \[[πŸ“‚ source](./node) | [πŸ“¦ npm](https://www.npmjs.com/package/deltachat-node) | [πŸ“š docs](https://js.delta.chat)\] - - over jsonrpc built with napi.rs (experimental): \[[πŸ“‚ source](https://github.com/deltachat/napi-jsonrpc) | [πŸ“¦ npm](https://www.npmjs.com/package/@deltachat/napi-jsonrpc)\] + - over JSON-RPC: \[[πŸ“‚ source](./deltachat-rpc-client) | [πŸ“¦ npm](https://www.npmjs.com/package/@deltachat/jsonrpc-client) | [πŸ“š docs](https://js.jsonrpc.delta.chat/)\] + - over CFFI[^1]: \[[πŸ“‚ source](./node) | [πŸ“¦ npm](https://www.npmjs.com/package/deltachat-node) | [πŸ“š docs](https://js.delta.chat/)\] - **Python** \[[πŸ“‚ source](./python) | [πŸ“¦ pypi](https://pypi.org/project/deltachat) | [πŸ“š docs](https://py.delta.chat)\] - **Go** - over jsonrpc: \[[πŸ“‚ source](https://github.com/deltachat/deltachat-rpc-client-go/)\] From 590f91331070088b49bbd84e1eb5bf16a2bc14bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2024 17:33:45 +0000 Subject: [PATCH 02/20] chore(deps): bump quinn-proto from 0.11.3 to 0.11.9 in /fuzz Bumps [quinn-proto](https://github.com/quinn-rs/quinn) from 0.11.3 to 0.11.9. - [Release notes](https://github.com/quinn-rs/quinn/releases) - [Commits](https://github.com/quinn-rs/quinn/compare/quinn-proto-0.11.3...quinn-proto-0.11.9) --- updated-dependencies: - dependency-name: quinn-proto dependency-type: indirect ... Signed-off-by: dependabot[bot] --- fuzz/Cargo.lock | 184 +++++++++++++++++++++++++++++------------------- 1 file changed, 110 insertions(+), 74 deletions(-) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index fec882ad1..13489ec30 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -179,7 +179,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror", + "thiserror 1.0.58", "time 0.3.36", ] @@ -191,7 +191,7 @@ checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", "synstructure 0.13.1", ] @@ -203,7 +203,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -275,7 +275,7 @@ dependencies = [ "pin-utils", "self_cell", "stop-token", - "thiserror", + "thiserror 1.0.58", "tokio", ] @@ -286,7 +286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9343dc5acf07e79ff82d0c37899f079db3534d99f189a1837c8e549c99405bec" dependencies = [ "native-tls", - "thiserror", + "thiserror 1.0.58", "tokio", "url", ] @@ -299,7 +299,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -314,7 +314,7 @@ dependencies = [ "log", "nom", "pin-project", - "thiserror", + "thiserror 1.0.58", "tokio", ] @@ -326,7 +326,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -339,7 +339,7 @@ dependencies = [ "crc32fast", "futures-lite 2.5.0", "pin-project", - "thiserror", + "thiserror 1.0.58", "tokio", "tokio-util", ] @@ -1040,7 +1040,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1108,7 +1108,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1119,7 +1119,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1220,7 +1220,7 @@ dependencies = [ "strum_macros", "tagger", "textwrap", - "thiserror", + "thiserror 1.0.58", "tokio", "tokio-io-timeout", "tokio-rustls", @@ -1263,7 +1263,7 @@ name = "deltachat_derive" version = "2.0.0" dependencies = [ "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1300,7 +1300,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1331,7 +1331,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1341,7 +1341,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1361,7 +1361,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", "unicode-xid", ] @@ -1579,7 +1579,7 @@ dependencies = [ "hex", "lazy_static", "regex", - "thiserror", + "thiserror 1.0.58", ] [[package]] @@ -1670,7 +1670,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1690,7 +1690,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -1814,7 +1814,7 @@ dependencies = [ "anyhow", "async-trait", "log", - "thiserror", + "thiserror 1.0.58", "tokio", "tokio-stream", ] @@ -2061,7 +2061,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -2331,7 +2331,7 @@ dependencies = [ "ipnet", "once_cell", "rand 0.8.5", - "thiserror", + "thiserror 1.0.58", "time 0.3.36", "tinyvec", "tokio", @@ -2355,7 +2355,7 @@ dependencies = [ "rand 0.8.5", "resolv-conf", "smallvec", - "thiserror", + "thiserror 1.0.58", "tokio", "tracing", ] @@ -2718,7 +2718,7 @@ dependencies = [ "rand_core 0.6.4", "serde", "ssh-key", - "thiserror", + "thiserror 1.0.58", "ttl_cache", "url", "zeroize", @@ -2848,7 +2848,7 @@ dependencies = [ "strum", "stun-rs", "surge-ping", - "thiserror", + "thiserror 1.0.58", "time 0.3.36", "tokio", "tokio-rustls", @@ -2880,7 +2880,7 @@ dependencies = [ "rustc-hash 2.0.0", "rustls", "socket2 0.5.6", - "thiserror", + "thiserror 1.0.58", "tokio", "tracing", ] @@ -2898,7 +2898,7 @@ dependencies = [ "rustls", "rustls-platform-verifier", "slab", - "thiserror", + "thiserror 1.0.58", "tinyvec", "tracing", ] @@ -2954,7 +2954,7 @@ dependencies = [ "combine", "jni-sys", "log", - "thiserror", + "thiserror 1.0.58", "walkdir", ] @@ -3186,7 +3186,7 @@ dependencies = [ "serde_bencode", "serde_bytes", "sha1_smol", - "thiserror", + "thiserror 1.0.58", "tracing", ] @@ -3349,7 +3349,7 @@ dependencies = [ "anyhow", "byteorder", "paste", - "thiserror", + "thiserror 1.0.58", ] [[package]] @@ -3363,7 +3363,7 @@ dependencies = [ "log", "netlink-packet-core", "netlink-sys", - "thiserror", + "thiserror 1.0.58", "tokio", ] @@ -3401,7 +3401,7 @@ dependencies = [ "rtnetlink", "serde", "socket2 0.5.6", - "thiserror", + "thiserror 1.0.58", "time 0.3.36", "tokio", "tracing", @@ -3500,7 +3500,7 @@ checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -3561,7 +3561,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -3792,7 +3792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", - "thiserror", + "thiserror 1.0.58", "ucd-trie", ] @@ -3816,7 +3816,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -3890,7 +3890,7 @@ dependencies = [ "sha3", "signature", "smallvec", - "thiserror", + "thiserror 1.0.58", "twofish", "x25519-dalek", "x448", @@ -3914,7 +3914,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -3946,7 +3946,7 @@ dependencies = [ "mainline", "self_cell", "simple-dns", - "thiserror", + "thiserror 1.0.58", "tracing", "ureq", "wasm-bindgen", @@ -4000,7 +4000,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -4086,7 +4086,7 @@ dependencies = [ "serde", "smallvec", "socket2 0.5.6", - "thiserror", + "thiserror 1.0.58", "time 0.3.36", "tokio", "tokio-util", @@ -4221,9 +4221,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -4248,7 +4248,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -4305,26 +4305,29 @@ dependencies = [ "quinn-udp", "rustc-hash 1.1.0", "rustls", - "thiserror", + "thiserror 1.0.58", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom 0.2.11", "rand 0.8.5", "ring", - "rustc-hash 1.1.0", + "rustc-hash 2.0.0", "rustls", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.6", "tinyvec", "tracing", + "web-time", ] [[package]] @@ -4670,7 +4673,7 @@ dependencies = [ "netlink-proto", "netlink-sys", "nix", - "thiserror", + "thiserror 1.0.58", "tokio", ] @@ -4800,6 +4803,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-platform-verifier" @@ -5018,7 +5024,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -5151,7 +5157,7 @@ dependencies = [ "shadowsocks-crypto", "socket2 0.5.6", "spin 0.9.8", - "thiserror", + "thiserror 1.0.58", "tokio", "tokio-tfo", "url", @@ -5384,7 +5390,7 @@ dependencies = [ "proc-macro2", "quote", "struct_iterable_internal", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -5412,7 +5418,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -5456,7 +5462,7 @@ dependencies = [ "pnet_packet", "rand 0.8.5", "socket2 0.5.6", - "thiserror", + "thiserror 1.0.58", "tokio", "tracing", ] @@ -5474,9 +5480,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.52" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -5520,7 +5526,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -5595,7 +5601,16 @@ version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.58", +] + +[[package]] +name = "thiserror" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +dependencies = [ + "thiserror-impl 2.0.6", ] [[package]] @@ -5606,7 +5621,18 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] @@ -5712,7 +5738,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -5792,7 +5818,7 @@ dependencies = [ "http 1.1.0", "httparse", "js-sys", - "thiserror", + "thiserror 1.0.58", "tokio", "tokio-tungstenite", "wasm-bindgen", @@ -5875,7 +5901,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -5946,7 +5972,7 @@ dependencies = [ "log", "rand 0.8.5", "sha1", - "thiserror", + "thiserror 1.0.58", "url", "utf-8", ] @@ -6175,7 +6201,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", "wasm-bindgen-shared", ] @@ -6209,7 +6235,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6229,7 +6255,7 @@ dependencies = [ "event-listener 4.0.3", "futures-util", "parking_lot", - "thiserror", + "thiserror 1.0.58", ] [[package]] @@ -6242,6 +6268,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.7" @@ -6351,7 +6387,7 @@ checksum = "12168c33176773b86799be25e2a2ba07c7aab9968b37541f1094dbd7a60c8946" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -6362,7 +6398,7 @@ checksum = "9d8dc32e0095a7eeccebd0e3f09e9509365ecb3fc6ac4d6f5f14a3f6392942d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] @@ -6651,7 +6687,7 @@ dependencies = [ "futures", "log", "serde", - "thiserror", + "thiserror 1.0.58", "windows 0.52.0", ] @@ -6700,7 +6736,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror", + "thiserror 1.0.58", "time 0.3.36", ] @@ -6760,7 +6796,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.90", ] [[package]] From 73687642101fe479dcdf5d676db9592e29064575 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 9 Dec 2024 19:16:18 +0000 Subject: [PATCH 03/20] docs: move rPGP to the security section of changelog --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ddeb238ac..fe7cea473 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,19 @@ - [**breaking**] Remove dc_all_work_done(). +### Security + +- cargo: Update rPGP to 0.14.2. + + This fixes [Panics on Malformed Untrusted Input](https://github.com/rpgp/rpgp/security/advisories/GHSA-9rmp-2568-59rv) + and [Potential Resource Exhaustion when handling Untrusted Messages](https://github.com/rpgp/rpgp/security/advisories/GHSA-4grw-m28r-q285). + This allows the attacker to crash the application via specially crafted messages and keys. + We recommend all users and bot operators to upgrade to the latest version. + There is no impact on the confidentiality of the messages and keys so no action other than upgrading is needed. + ### Fixes - Store plaintext in mime_headers of truncated sent messages ([#6273](https://github.com/deltachat/deltachat-core-rust/pull/6273)). -- cargo: Update rPGP to 0.14.2. ### Documentation From a373dd4e99332251f6f64ff3ba69ce6be0748a10 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 9 Dec 2024 13:04:00 +0000 Subject: [PATCH 04/20] fix: do not subscribe to heartbeat if already subscribed via metadata --- src/imap.rs | 84 +++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/src/imap.rs b/src/imap.rs index d6319e67d..50a159b38 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -1560,52 +1560,54 @@ impl Session { return Ok(()); }; - let device_token_changed = context - .get_config(Config::DeviceToken) - .await? - .map_or(true, |config_token| device_token != config_token); - - if device_token_changed && self.can_metadata() && self.can_push() { - let folder = context - .get_config(Config::ConfiguredInboxFolder) + if self.can_metadata() && self.can_push() { + let device_token_changed = context + .get_config(Config::DeviceToken) .await? - .context("INBOX is not configured")?; + .map_or(true, |config_token| device_token != config_token); - let encrypted_device_token = - encrypt_device_token(&device_token).context("Failed to encrypt device token")?; + if device_token_changed { + let folder = context + .get_config(Config::ConfiguredInboxFolder) + .await? + .context("INBOX is not configured")?; - // We expect that the server supporting `XDELTAPUSH` capability - // has non-synchronizing literals support as well: - // . - let encrypted_device_token_len = encrypted_device_token.len(); + let encrypted_device_token = encrypt_device_token(&device_token) + .context("Failed to encrypt device token")?; - if encrypted_device_token_len <= 4096 { - self.run_command_and_check_ok(&format_setmetadata( - &folder, - &encrypted_device_token, - )) - .await - .context("SETMETADATA command failed")?; + // We expect that the server supporting `XDELTAPUSH` capability + // has non-synchronizing literals support as well: + // . + let encrypted_device_token_len = encrypted_device_token.len(); - // Store device token saved on the server - // to prevent storing duplicate tokens. - // The server cannot deduplicate on its own - // because encryption gives a different - // result each time. - context - .set_config_internal(Config::DeviceToken, Some(&device_token)) - .await?; - } else { - // If Apple or Google (FCM) gives us a very large token, - // do not even try to give it to IMAP servers. - // - // Limit of 4096 is arbitrarily selected - // to be the same as required by LITERAL- IMAP extension. - // - // Dovecot supports LITERAL+ and non-synchronizing literals - // of any length, but there is no reason for tokens - // to be that large even after OpenPGP encryption. - warn!(context, "Device token is too long for LITERAL-, ignoring."); + if encrypted_device_token_len <= 4096 { + self.run_command_and_check_ok(&format_setmetadata( + &folder, + &encrypted_device_token, + )) + .await + .context("SETMETADATA command failed")?; + + // Store device token saved on the server + // to prevent storing duplicate tokens. + // The server cannot deduplicate on its own + // because encryption gives a different + // result each time. + context + .set_config_internal(Config::DeviceToken, Some(&device_token)) + .await?; + } else { + // If Apple or Google (FCM) gives us a very large token, + // do not even try to give it to IMAP servers. + // + // Limit of 4096 is arbitrarily selected + // to be the same as required by LITERAL- IMAP extension. + // + // Dovecot supports LITERAL+ and non-synchronizing literals + // of any length, but there is no reason for tokens + // to be that large even after OpenPGP encryption. + warn!(context, "Device token is too long for LITERAL-, ignoring."); + } } context.push_subscribed.store(true, Ordering::Relaxed); } else if !context.push_subscriber.heartbeat_subscribed().await { From b650b96ccd4e9222309e9b0ff61d4330745d4656 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Dec 2024 09:30:42 +0000 Subject: [PATCH 05/20] chore(release): prepare for 1.151.6 --- CHANGELOG.md | 29 +++++++++++++++++++ Cargo.lock | 10 +++---- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 2 +- deltachat-repl/Cargo.toml | 2 +- deltachat-rpc-client/pyproject.toml | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- deltachat-rpc-server/npm-package/package.json | 2 +- package.json | 2 +- python/pyproject.toml | 2 +- release-date.in | 2 +- 13 files changed, 45 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7cea473..bee527dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [1.151.6] - 2024-12-11 + +### Features / Changes + +- Don't add "Failed to send message to ..." info messages to group chats. +- Add info messages about implicit membership changes if group member list is recreated ([#6314](https://github.com/deltachat/deltachat-core-rust/pull/6314)). + +### Fixes + +- Add self-addition message to chat when recreating member list. +- Do not subscribe to heartbeat if already subscribed via metadata. + +### Build system + +- Add idna 0.5.0 exception into deny.toml. + +### Documentation + +- Update links to Node.js bindings in the README. + +### Refactor + +- Factor out `wait_for_all_work_done()`. + +### Tests + +- Notifiy more prominently & in more tests about false positives when running `cargo test` ([#6308](https://github.com/deltachat/deltachat-core-rust/pull/6308)). + ## [1.151.5] - 2024-12-05 ### API-Changes @@ -5458,3 +5486,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed [1.151.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.2..v1.151.3 [1.151.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.3..v1.151.4 [1.151.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.4..v1.151.5 +[1.151.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.5..v1.151.6 diff --git a/Cargo.lock b/Cargo.lock index 3380d99d0..56d44a529 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1306,7 +1306,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.151.5" +version = "1.151.6" dependencies = [ "anyhow", "async-broadcast", @@ -1407,7 +1407,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.151.5" +version = "1.151.6" dependencies = [ "anyhow", "async-channel 2.3.1", @@ -1432,7 +1432,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.151.5" +version = "1.151.6" dependencies = [ "anyhow", "deltachat", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.151.5" +version = "1.151.6" dependencies = [ "anyhow", "deltachat", @@ -1477,7 +1477,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.151.5" +version = "1.151.6" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index 0428022fe..b604729f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.151.5" +version = "1.151.6" edition = "2021" license = "MPL-2.0" rust-version = "1.77" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index d00b1961b..4146922fa 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.151.5" +version = "1.151.6" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 3dddc6577..4dd09d8e0 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.151.5" +version = "1.151.6" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 50d988841..74bd0678d 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -58,5 +58,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.151.5" + "version": "1.151.6" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index d3c815e6e..b60055e72 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.151.5" +version = "1.151.6" license = "MPL-2.0" edition = "2021" repository = "https://github.com/deltachat/deltachat-core-rust" diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index 50bb6ba20..e607ee6a7 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deltachat-rpc-client" -version = "1.151.5" +version = "1.151.6" description = "Python client for Delta Chat core JSON-RPC interface" classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index d8228348b..95c8b816a 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.151.5" +version = "1.151.6" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/deltachat-rpc-server/npm-package/package.json b/deltachat-rpc-server/npm-package/package.json index 37866eba6..ae21c50e1 100644 --- a/deltachat-rpc-server/npm-package/package.json +++ b/deltachat-rpc-server/npm-package/package.json @@ -15,5 +15,5 @@ }, "type": "module", "types": "index.d.ts", - "version": "1.151.5" + "version": "1.151.6" } diff --git a/package.json b/package.json index 64054d5b4..ba9823238 100644 --- a/package.json +++ b/package.json @@ -55,5 +55,5 @@ "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.151.5" + "version": "1.151.6" } diff --git a/python/pyproject.toml b/python/pyproject.toml index 3b455338b..a5cb5f23d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deltachat" -version = "1.151.5" +version = "1.151.6" description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat" readme = "README.rst" requires-python = ">=3.7" diff --git a/release-date.in b/release-date.in index c0b24a784..b57a0e640 100644 --- a/release-date.in +++ b/release-date.in @@ -1 +1 @@ -2024-12-05 \ No newline at end of file +2024-12-11 \ No newline at end of file From bc3b6ae30910d904d966c703db96091343b9b26e Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Tue, 10 Dec 2024 15:31:06 +0100 Subject: [PATCH 06/20] feat: prefix server-url in info without the prefix, it looks as if it is part of the Message-ID, esp. if Message-ID is longer, a break on different delimiters may look exactly the same. see #6329 for some screenshots that initially confused me :) --- src/message.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index 75f3813cd..a47d83e2b 100644 --- a/src/message.rs +++ b/src/message.rs @@ -348,7 +348,7 @@ impl MsgId { let server_urls = Self::get_info_server_urls(context, msg.rfc724_mid).await?; for server_url in server_urls { // Format as RFC 5092 relative IMAP URL. - ret += &format!("\n{server_url}"); + ret += &format!("\nServer-URL: {server_url}"); } } let hop_info = self.hop_info(context).await?; From 72558af98cef3a39d927097e65ff7dc6e2af0fc8 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 7 Dec 2024 18:27:26 +0000 Subject: [PATCH 07/20] api!: remove `dc_prepare_msg` and `dc_msg_is_increation` --- deltachat-ffi/deltachat.h | 71 +------- deltachat-ffi/src/lib.rs | 31 ---- node/lib/message.ts | 4 - node/src/module.c | 12 -- node/test/test.mjs | 1 - python/src/deltachat/chat.py | 34 +--- python/tests/test_1_online.py | 8 +- python/tests/test_2_increation.py | 107 ------------ python/tests/test_3_offline.py | 42 +---- src/blob.rs | 35 ---- src/chat.rs | 269 +++++++++++------------------- src/ephemeral.rs | 3 - src/message.rs | 48 +----- src/mimefactory.rs | 20 ++- src/mimeparser.rs | 6 +- src/param.rs | 22 +-- src/sql.rs | 1 - 17 files changed, 125 insertions(+), 589 deletions(-) delete mode 100644 python/tests/test_2_increation.py diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 19bbdeb89..805b88849 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -963,54 +963,6 @@ uint32_t dc_create_chat_by_contact_id (dc_context_t* context, uint32_t co uint32_t dc_get_chat_id_by_contact_id (dc_context_t* context, uint32_t contact_id); -/** - * Prepare a message for sending. - * - * Call this function if the file to be sent is still in creation. - * Once you're done with creating the file, call dc_send_msg() as usual - * and the message will really be sent. - * - * This is useful as the user can already send the next messages while - * e.g. the recoding of a video is not yet finished. Or the user can even forward - * the message with the file being still in creation to other groups. - * - * Files being sent with the increation-method must be placed in the - * blob directory, see dc_get_blobdir(). - * If the increation-method is not used - which is probably the normal case - - * dc_send_msg() copies the file to the blob directory if it is not yet there. - * To distinguish the two cases, msg->state must be set properly. The easiest - * way to ensure this is to reuse the same object for both calls. - * - * Example: - * ~~~ - * char* blobdir = dc_get_blobdir(context); - * char* file_to_send = mprintf("%s/%s", blobdir, "send.mp4") - * - * dc_msg_t* msg = dc_msg_new(context, DC_MSG_VIDEO); - * dc_msg_set_file(msg, file_to_send, NULL); - * dc_prepare_msg(context, chat_id, msg); - * - * // ... create the file ... - * - * dc_send_msg(context, chat_id, msg); - * - * dc_msg_unref(msg); - * free(file_to_send); - * dc_str_unref(file_to_send); - * ~~~ - * - * @memberof dc_context_t - * @param context The context object as returned from dc_context_new(). - * @param chat_id The chat ID to send the message to. - * @param msg The message object to send to the chat defined by the chat ID. - * On success, msg_id and state of the object are set up, - * The function does not take ownership of the object, - * so you have to free it using dc_msg_unref() as usual. - * @return The ID of the message that is being prepared. - */ -uint32_t dc_prepare_msg (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg); - - /** * Send a message defined by a dc_msg_t object to a chat. * @@ -1035,13 +987,11 @@ uint32_t dc_prepare_msg (dc_context_t* context, uint32_t ch * If that fails, is not possible, or the image is already small enough, the image is sent as original. * If you want images to be always sent as the original file, use the #DC_MSG_FILE type. * - * Videos and other file types are currently not recoded by the library, - * with dc_prepare_msg(), however, you can do that from the UI. + * Videos and other file types are currently not recoded by the library. * * @memberof dc_context_t * @param context The context object as returned from dc_context_new(). * @param chat_id The chat ID to send the message to. - * If dc_prepare_msg() was called before, this parameter can be 0. * @param msg The message object to send to the chat defined by the chat ID. * On success, msg_id of the object is set up, * The function does not take ownership of the object, @@ -1058,7 +1008,6 @@ uint32_t dc_send_msg (dc_context_t* context, uint32_t ch * @memberof dc_context_t * @param context The context object as returned from dc_context_new(). * @param chat_id The chat ID to send the message to. - * If dc_prepare_msg() was called before, this parameter can be 0. * @param msg The message object to send to the chat defined by the chat ID. * On success, msg_id of the object is set up, * The function does not take ownership of the object, @@ -3985,7 +3934,7 @@ int dc_msg_get_viewtype (const dc_msg_t* msg); * * Outgoing message states: * - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent, - * the message enters this state before @ref DC_STATE_OUT_PENDING. + * the message enters this state before @ref DC_STATE_OUT_PENDING. Deprecated. * - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft() * - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the * message is not yet sent and is pending in some way. Maybe we're offline (no checkmark). @@ -4535,20 +4484,6 @@ int dc_msg_get_info_type (const dc_msg_t* msg); */ char* dc_msg_get_webxdc_href (const dc_msg_t* msg); -/** - * Check if a message is still in creation. A message is in creation between - * the calls to dc_prepare_msg() and dc_send_msg(). - * - * Typically, this is used for videos that are recoded by the UI before - * they can be sent. - * - * @memberof dc_msg_t - * @param msg The message object. - * @return 1=message is still in creation (dc_send_msg() was not called yet), - * 0=message no longer in creation. - */ -int dc_msg_is_increation (const dc_msg_t* msg); - /** * Check if the message is an Autocrypt Setup Message. @@ -5562,6 +5497,8 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); /** * Outgoing message being prepared. See dc_msg_get_state() for details. + * + * @deprecated 2024-12-07 */ #define DC_STATE_OUT_PREPARING 18 diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 694654e36..c9c0cf843 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -976,27 +976,6 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id( }) } -#[no_mangle] -pub unsafe extern "C" fn dc_prepare_msg( - context: *mut dc_context_t, - chat_id: u32, - msg: *mut dc_msg_t, -) -> u32 { - if context.is_null() || chat_id == 0 || msg.is_null() { - eprintln!("ignoring careless call to dc_prepare_msg()"); - return 0; - } - let ctx = &mut *context; - let ffi_msg: &mut MessageWrapper = &mut *msg; - - block_on(async move { - chat::prepare_msg(ctx, ChatId::new(chat_id), &mut ffi_msg.message) - .await - .unwrap_or_log_default(ctx, "Failed to prepare message") - }) - .to_u32() -} - #[no_mangle] pub unsafe extern "C" fn dc_send_msg( context: *mut dc_context_t, @@ -3713,16 +3692,6 @@ pub unsafe extern "C" fn dc_msg_get_webxdc_href(msg: *mut dc_msg_t) -> *mut libc ffi_msg.message.get_webxdc_href().strdup() } -#[no_mangle] -pub unsafe extern "C" fn dc_msg_is_increation(msg: *mut dc_msg_t) -> libc::c_int { - if msg.is_null() { - eprintln!("ignoring careless call to dc_msg_is_increation()"); - return 0; - } - let ffi_msg = &*msg; - ffi_msg.message.is_increation().into() -} - #[no_mangle] pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_int { if msg.is_null() { diff --git a/node/lib/message.ts b/node/lib/message.ts index 5b2ad67d9..c4036ad6b 100644 --- a/node/lib/message.ts +++ b/node/lib/message.ts @@ -299,10 +299,6 @@ export class Message { return Boolean(binding.dcn_msg_is_forwarded(this.dc_msg)) } - isIncreation() { - return Boolean(binding.dcn_msg_is_increation(this.dc_msg)) - } - isInfo() { return Boolean(binding.dcn_msg_is_info(this.dc_msg)) } diff --git a/node/src/module.c b/node/src/module.c index 33492aeec..02dda50c1 100644 --- a/node/src/module.c +++ b/node/src/module.c @@ -2374,17 +2374,6 @@ NAPI_METHOD(dcn_msg_is_forwarded) { NAPI_RETURN_INT32(is_forwarded); } -NAPI_METHOD(dcn_msg_is_increation) { - NAPI_ARGV(1); - NAPI_DC_MSG(); - - //TRACE("calling.."); - int is_increation = dc_msg_is_increation(dc_msg); - //TRACE("result %d", is_increation); - - NAPI_RETURN_INT32(is_increation); -} - NAPI_METHOD(dcn_msg_is_info) { NAPI_ARGV(1); NAPI_DC_MSG(); @@ -3555,7 +3544,6 @@ NAPI_INIT() { NAPI_EXPORT_FUNCTION(dcn_msg_has_location); NAPI_EXPORT_FUNCTION(dcn_msg_has_html); NAPI_EXPORT_FUNCTION(dcn_msg_is_forwarded); - NAPI_EXPORT_FUNCTION(dcn_msg_is_increation); NAPI_EXPORT_FUNCTION(dcn_msg_is_info); NAPI_EXPORT_FUNCTION(dcn_msg_is_sent); NAPI_EXPORT_FUNCTION(dcn_msg_is_setupmessage); diff --git a/node/test/test.mjs b/node/test/test.mjs index 5c9d67ef7..a7ae73727 100644 --- a/node/test/test.mjs +++ b/node/test/test.mjs @@ -536,7 +536,6 @@ describe('Offline Tests with unconfigured account', function () { strictEqual(msg.getWidth(), 0, 'no message width') strictEqual(msg.isDeadDrop(), false, 'not deaddrop') strictEqual(msg.isForwarded(), false, 'not forwarded') - strictEqual(msg.isIncreation(), false, 'not in creation') strictEqual(msg.isInfo(), false, 'not an info message') strictEqual(msg.isSent(), false, 'messge is not sent') strictEqual(msg.isSetupmessage(), false, 'not an autocrypt setup message') diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index d275e09cc..b5cfa9235 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -271,8 +271,7 @@ class Chat: :param msg: a :class:`deltachat.message.Message` instance previously returned by - e.g. :meth:`deltachat.message.Message.new_empty` or - :meth:`prepare_file`. + e.g. :meth:`deltachat.message.Message.new_empty`. :raises ValueError: if message can not be sent. :returns: a :class:`deltachat.message.Message` instance as @@ -341,37 +340,6 @@ class Chat: raise ValueError("message could not be sent") return Message.from_db(self.account, sent_id) - def prepare_message(self, msg): - """prepare a message for sending. - - :param msg: the message to be prepared. - :returns: a :class:`deltachat.message.Message` instance. - This is the same object that was passed in, which - has been modified with the new state of the core. - """ - msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg) - if msg_id == 0: - raise ValueError("message could not be prepared") - # modify message in place to avoid bad state for the caller - msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg - return msg - - def prepare_message_file(self, path, mime_type=None, view_type="file"): - """prepare a message for sending and return the resulting Message instance. - - To actually send the message, call :meth:`send_prepared`. - The file must be inside the blob directory. - - :param path: path to the file. - :param mime_type: the mime-type of this file, defaults to auto-detection. - :param view_type: "text", "image", "gif", "audio", "video", "file" - :raises ValueError: if message can not be prepared/chat does not exist. - :returns: the resulting :class:`Message` instance - """ - msg = Message.new_empty(self.account, view_type) - msg.set_file(path, mime_type) - return self.prepare_message(msg) - def send_prepared(self, message): """send a previously prepared message. diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index d6bc3d6e9..de7b38f6d 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1366,10 +1366,9 @@ def test_quote_encrypted(acfactory, lp): msg_draft.quote = quoted_msg chat.set_draft(msg_draft) - # Get the draft, prepare and send it. + # Get the draft and send it. msg_draft = chat.get_draft() - msg_out = chat.prepare_message(msg_draft) - chat.send_prepared(msg_out) + chat.send_msg(msg_draft) chat.set_draft(None) assert chat.get_draft() is None @@ -2291,9 +2290,8 @@ def test_group_quote(acfactory, lp): reply_msg = Message.new_empty(msg.chat.account, "text") reply_msg.set_text("reply") reply_msg.quote = msg - reply_msg = msg.chat.prepare_message(reply_msg) assert reply_msg.quoted_text == "hello" - msg.chat.send_prepared(reply_msg) + msg.chat.send_msg(reply_msg) lp.sec("ac3: receiving reply") received_reply = ac3._evtracker.wait_next_incoming_message() diff --git a/python/tests/test_2_increation.py b/python/tests/test_2_increation.py deleted file mode 100644 index 1fe85e02a..000000000 --- a/python/tests/test_2_increation.py +++ /dev/null @@ -1,107 +0,0 @@ -import os.path -import shutil -from filecmp import cmp - -import pytest - - -def wait_msg_delivered(account, msg_list): - """wait for one or more MSG_DELIVERED events to match msg_list contents.""" - msg_list = list(msg_list) - while msg_list: - ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") - msg_list.remove((ev.data1, ev.data2)) - - -def wait_msgs_changed(account, msgs_list): - """wait for one or more MSGS_CHANGED events to match msgs_list contents.""" - account.log(f"waiting for msgs_list={msgs_list}") - msgs_list = list(msgs_list) - while msgs_list: - ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - for i, (data1, data2) in enumerate(msgs_list): - if ev.data1 == data1: - if data2 is None or ev.data2 == data2: - del msgs_list[i] - break - else: - account.log(f"waiting mismatch data1={data1} data2={data2}") - return ev.data2 - - -class TestOnlineInCreation: - def test_increation_not_blobdir(self, tmp_path, acfactory, lp): - ac1, ac2 = acfactory.get_online_accounts(2) - chat = ac1.create_chat(ac2) - - lp.sec("Creating in-creation file outside of blobdir") - assert str(tmp_path) != ac1.get_blobdir() - src = tmp_path / "file.txt" - src.touch() - with pytest.raises(Exception): - chat.prepare_message_file(str(src)) - - def test_no_increation_copies_to_blobdir(self, tmp_path, acfactory, lp): - ac1, ac2 = acfactory.get_online_accounts(2) - chat = ac1.create_chat(ac2) - - lp.sec("Creating file outside of blobdir") - assert str(tmp_path) != ac1.get_blobdir() - src = tmp_path / "file.txt" - src.write_text("hello there\n") - msg = chat.send_file(str(src)) - assert msg.filename.startswith(os.path.join(ac1.get_blobdir(), "file")) - assert msg.filename.endswith(".txt") - - def test_forward_increation(self, acfactory, data, lp): - ac1, ac2 = acfactory.get_online_accounts(2) - - chat = ac1.create_chat(ac2) - wait_msgs_changed(ac1, [(0, 0)]) # why no chat id? - - lp.sec("create a message with a file in creation") - orig = data.get_path("d.png") - path = os.path.join(ac1.get_blobdir(), "d.png") - with open(path, "x") as fp: - fp.write("preparing") - prepared_original = chat.prepare_message_file(path) - assert prepared_original.is_out_preparing() - wait_msgs_changed(ac1, [(chat.id, prepared_original.id)]) - - lp.sec("create a new group") - chat2 = ac1.create_group_chat("newgroup") - wait_msgs_changed(ac1, [(0, 0)]) - - lp.sec("add a contact to new group") - chat2.add_contact(ac2) - wait_msgs_changed(ac1, [(chat2.id, None)]) - - lp.sec("forward the message while still in creation") - ac1.forward_messages([prepared_original], chat2) - forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)]) - forwarded_msg = ac1.get_message_by_id(forwarded_id) - assert forwarded_msg.is_out_preparing() - - lp.sec("finish creating the file and send it") - assert prepared_original.is_out_preparing() - shutil.copyfile(orig, path) - chat.send_prepared(prepared_original) - assert prepared_original.is_out_pending() or prepared_original.is_out_delivered() - - lp.sec("check that both forwarded and original message are proper.") - wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)]) - - fwd_msg = ac1.get_message_by_id(forwarded_id) - assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered() - - lp.sec("wait for both messages to be delivered to SMTP") - wait_msg_delivered(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)]) - - lp.sec("wait1 for original or forwarded messages to arrive") - received_original = ac2._evtracker.wait_next_incoming_message() - assert cmp(received_original.filename, orig, shallow=False) - - lp.sec("wait2 for original or forwarded messages to arrive") - received_copy = ac2._evtracker.wait_next_incoming_message() - assert received_copy.id != received_original.id - assert cmp(received_copy.filename, orig, shallow=False) diff --git a/python/tests/test_3_offline.py b/python/tests/test_3_offline.py index dcd752d55..cb5d2ac77 100644 --- a/python/tests/test_3_offline.py +++ b/python/tests/test_3_offline.py @@ -378,30 +378,6 @@ class TestOfflineChat: with pytest.raises(ValueError): chat1.send_text("msg1") - def test_prepare_message_and_send(self, ac1, chat1): - msg = chat1.prepare_message(Message.new_empty(chat1.account, "text")) - msg.set_text("hello world") - assert msg.text == "hello world" - assert msg.id > 0 - chat1.send_prepared(msg) - assert "Sent" in msg.get_message_info() - str(msg) - repr(msg) - assert msg == ac1.get_message_by_id(msg.id) - - def test_prepare_file(self, ac1, chat1): - blobdir = ac1.get_blobdir() - p = os.path.join(blobdir, "somedata.txt") - with open(p, "w") as f: - f.write("some data") - message = chat1.prepare_message_file(p) - assert message.id > 0 - message.set_text("hello world") - assert message.is_out_preparing() - assert message.text == "hello world" - chat1.send_prepared(message) - assert "Sent" in message.get_message_info() - def test_message_eq_contains(self, chat1): msg = chat1.send_text("msg1") msg2 = None @@ -691,8 +667,7 @@ class TestOfflineChat: assert os.path.exists(messages[1].filename) def test_set_get_draft(self, chat1): - msg = Message.new_empty(chat1.account, "text") - msg1 = chat1.prepare_message(msg) + msg1 = Message.new_empty(chat1.account, "text") msg1.set_text("hello") chat1.set_draft(msg1) msg1.set_text("obsolete") @@ -711,21 +686,6 @@ class TestOfflineChat: assert not res.is_ask_verifygroup() assert res.contact_id == 10 - def test_quote(self, chat1): - """Offline quoting test""" - msg = Message.new_empty(chat1.account, "text") - msg.set_text("Multi\nline\nmessage") - assert msg.quoted_text is None - - # Prepare message to assign it a Message-Id. - # Messages without Message-Id cannot be quoted. - msg = chat1.prepare_message(msg) - - reply_msg = Message.new_empty(chat1.account, "text") - reply_msg.set_text("reply") - reply_msg.quote = msg - assert reply_msg.quoted_text == "Multi\nline\nmessage" - def test_group_chat_many_members_add_remove(self, ac1, lp): lp.sec("ac1: creating group chat with 10 other members") chat = ac1.create_group_chat(name="title1") diff --git a/src/blob.rs b/src/blob.rs index 43a42850f..63c235056 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -763,7 +763,6 @@ mod tests { use fs::File; use super::*; - use crate::chat::{self, create_group_chat, ProtectionStatus}; use crate::message::{Message, Viewtype}; use crate::test_utils::{self, TestContext}; @@ -1456,38 +1455,4 @@ mod tests { check_image_size(file_saved, width, height); Ok(()) } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_increation_in_blobdir() -> Result<()> { - let t = TestContext::new_alice().await; - let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?; - - let file = t.get_blobdir().join("anyfile.dat"); - fs::write(&file, b"bla").await?; - let mut msg = Message::new(Viewtype::File); - msg.set_file(file.to_str().unwrap(), None); - let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?; - assert_eq!(prepared_id, msg.id); - assert!(msg.is_increation()); - - let msg = Message::load_from_db(&t, prepared_id).await?; - assert!(msg.is_increation()); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_increation_not_blobdir() -> Result<()> { - let t = TestContext::new_alice().await; - let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?; - assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str()); - - let file = t.dir.path().join("anyfile.dat"); - fs::write(&file, b"bla").await?; - let mut msg = Message::new(Viewtype::File); - msg.set_file(file.to_str().unwrap(), None); - assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err()); - - Ok(()) - } } diff --git a/src/chat.rs b/src/chat.rs index d05ce56f7..437dbff4e 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -888,7 +888,7 @@ impl ChatId { _ => { let blob = msg .param - .get_blob(Param::File, context, !msg.is_increation()) + .get_blob(Param::File, context) .await? .context("no file stored in params")?; msg.param.set(Param::File, blob.as_name()); @@ -2677,26 +2677,13 @@ impl ChatIdBlocked { } } -/// Prepares a message for sending. -pub async fn prepare_msg(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result { - ensure!( - !chat_id.is_special(), - "Cannot prepare message for special chat" - ); - - let msg_id = prepare_msg_common(context, chat_id, msg, MessageState::OutPreparing).await?; - context.emit_msgs_changed(msg.chat_id, msg.id); - - Ok(msg_id) -} - async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> { if msg.viewtype == Viewtype::Text || msg.viewtype == Viewtype::VideochatInvitation { // the caller should check if the message text is empty } else if msg.viewtype.has_file() { let mut blob = msg .param - .get_blob(Param::File, context, !msg.is_increation()) + .get_blob(Param::File, context) .await? .with_context(|| format!("attachment missing for message of type #{}", msg.viewtype))?; let send_as_is = msg.viewtype == Viewtype::File; @@ -2771,13 +2758,92 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> { Ok(()) } +/// Returns whether a contact is in a chat or not. +pub async fn is_contact_in_chat( + context: &Context, + chat_id: ChatId, + contact_id: ContactId, +) -> Result { + // this function works for group and for normal chats, however, it is more useful + // for group chats. + // ContactId::SELF may be used to check, if the user itself is in a group + // chat (ContactId::SELF is not added to normal chats) + + let exists = context + .sql + .exists( + "SELECT COUNT(*) FROM chats_contacts WHERE chat_id=? AND contact_id=?;", + (chat_id, contact_id), + ) + .await?; + Ok(exists) +} + +/// Sends a message object to a chat. +/// +/// Sends the event #DC_EVENT_MSGS_CHANGED on success. +/// However, this does not imply, the message really reached the recipient - +/// sending may be delayed eg. due to network problems. However, from your +/// view, you're done with the message. Sooner or later it will find its way. +pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result { + ensure!( + !chat_id.is_special(), + "chat_id cannot be a special chat: {chat_id}" + ); + + if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing { + msg.param.remove(Param::GuaranteeE2ee); + msg.param.remove(Param::ForcePlaintext); + msg.update_param(context).await?; + } + + // protect all system messages against RTLO attacks + if msg.is_system_message() { + msg.text = sanitize_bidi_characters(&msg.text); + } + + if !prepare_send_msg(context, chat_id, msg).await?.is_empty() { + if !msg.hidden { + context.emit_msgs_changed(msg.chat_id, msg.id); + } + + if msg.param.exists(Param::SetLatitude) { + context.emit_location_changed(Some(ContactId::SELF)).await?; + } + + context.scheduler.interrupt_smtp().await; + } + + Ok(msg.id) +} + +/// Tries to send a message synchronously. +/// +/// Creates jobs in the `smtp` table, then drectly opens an SMTP connection and sends the +/// message. If this fails, the jobs remain in the database for later sending. +pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result { + let rowids = prepare_send_msg(context, chat_id, msg).await?; + if rowids.is_empty() { + return Ok(msg.id); + } + let mut smtp = crate::smtp::Smtp::new(); + for rowid in rowids { + send_msg_to_smtp(context, &mut smtp, rowid) + .await + .context("failed to send message, queued for later sending")?; + } + context.emit_msgs_changed(msg.chat_id, msg.id); + Ok(msg.id) +} + /// Prepares a message to be sent out. -async fn prepare_msg_common( +/// +/// Returns row ids of the `smtp` table. +async fn prepare_send_msg( context: &Context, chat_id: ChatId, msg: &mut Message, - change_state_to: MessageState, -) -> Result { +) -> Result> { let mut chat = Chat::load_from_db(context, chat_id).await?; // Check if the chat can be sent to. @@ -2821,7 +2887,7 @@ async fn prepare_msg_common( }; // ... then change the MessageState in the message object - msg.state = change_state_to; + msg.state = MessageState::OutPending; prepare_msg_blob(context, msg).await?; if !msg.hidden { @@ -2837,125 +2903,6 @@ async fn prepare_msg_common( .await?; msg.chat_id = chat_id; - Ok(msg.id) -} - -/// Returns whether a contact is in a chat or not. -pub async fn is_contact_in_chat( - context: &Context, - chat_id: ChatId, - contact_id: ContactId, -) -> Result { - // this function works for group and for normal chats, however, it is more useful - // for group chats. - // ContactId::SELF may be used to check, if the user itself is in a group - // chat (ContactId::SELF is not added to normal chats) - - let exists = context - .sql - .exists( - "SELECT COUNT(*) FROM chats_contacts WHERE chat_id=? AND contact_id=?;", - (chat_id, contact_id), - ) - .await?; - Ok(exists) -} - -/// Sends a message object to a chat. -/// -/// Sends the event #DC_EVENT_MSGS_CHANGED on success. -/// However, this does not imply, the message really reached the recipient - -/// sending may be delayed eg. due to network problems. However, from your -/// view, you're done with the message. Sooner or later it will find its way. -// TODO: Do not allow ChatId to be 0, if prepare_msg had been called -// the caller can get it from msg.chat_id. Forwards would need to -// be fixed for this somehow too. -pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result { - if chat_id.is_unset() { - let forwards = msg.param.get(Param::PrepForwards); - if let Some(forwards) = forwards { - for forward in forwards.split(' ') { - if let Ok(msg_id) = forward.parse::().map(MsgId::new) { - if let Ok(mut msg) = Message::load_from_db(context, msg_id).await { - send_msg_inner(context, chat_id, &mut msg).await?; - }; - } - } - msg.param.remove(Param::PrepForwards); - msg.update_param(context).await?; - } - return send_msg_inner(context, chat_id, msg).await; - } - - if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing { - msg.param.remove(Param::GuaranteeE2ee); - msg.param.remove(Param::ForcePlaintext); - msg.update_param(context).await?; - } - send_msg_inner(context, chat_id, msg).await -} - -/// Tries to send a message synchronously. -/// -/// Creates jobs in the `smtp` table, then drectly opens an SMTP connection and sends the -/// message. If this fails, the jobs remain in the database for later sending. -pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result { - let rowids = prepare_send_msg(context, chat_id, msg).await?; - if rowids.is_empty() { - return Ok(msg.id); - } - let mut smtp = crate::smtp::Smtp::new(); - for rowid in rowids { - send_msg_to_smtp(context, &mut smtp, rowid) - .await - .context("failed to send message, queued for later sending")?; - } - context.emit_msgs_changed(msg.chat_id, msg.id); - Ok(msg.id) -} - -async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result { - // protect all system messages against RTLO attacks - if msg.is_system_message() { - msg.text = sanitize_bidi_characters(&msg.text); - } - - if !prepare_send_msg(context, chat_id, msg).await?.is_empty() { - if !msg.hidden { - context.emit_msgs_changed(msg.chat_id, msg.id); - } - - if msg.param.exists(Param::SetLatitude) { - context.emit_location_changed(Some(ContactId::SELF)).await?; - } - - context.scheduler.interrupt_smtp().await; - } - - Ok(msg.id) -} - -/// Returns row ids of the `smtp` table. -async fn prepare_send_msg( - context: &Context, - chat_id: ChatId, - msg: &mut Message, -) -> Result> { - // prepare_msg() leaves the message state to OutPreparing, we - // only have to change the state to OutPending in this case. - // Otherwise we still have to prepare the message, which will set - // the state to OutPending. - if msg.state != MessageState::OutPreparing { - // automatically prepare normal messages - prepare_msg_common(context, chat_id, msg, MessageState::OutPending).await?; - } else { - // update message state of separately prepared messages - ensure!( - chat_id.is_unset() || chat_id == msg.chat_id, - "Inconsistent chat ID" - ); - message::update_msg_state(context, msg.id, MessageState::OutPending).await?; - } let row_ids = create_send_msg_jobs(context, msg) .await .context("Failed to create send jobs")?; @@ -4173,8 +4120,6 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) bail!("cannot forward drafts."); } - let original_param = msg.param.clone(); - // we tested a sort of broadcast // by not marking own forwarded messages as such, // however, this turned out to be to confusing and unclear. @@ -4197,33 +4142,13 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) // do not leak data as group names; a default subject is generated by mimefactory msg.subject = "".to_string(); - let new_msg_id: MsgId; - if msg.state == MessageState::OutPreparing { - new_msg_id = chat - .prepare_msg_raw(context, &mut msg, None, curr_timestamp) - .await?; - curr_timestamp += 1; - msg.param = original_param; - msg.id = src_msg_id; - - if let Some(old_fwd) = msg.param.get(Param::PrepForwards) { - let new_fwd = format!("{} {}", old_fwd, new_msg_id.to_u32()); - msg.param.set(Param::PrepForwards, new_fwd); - } else { - msg.param - .set(Param::PrepForwards, new_msg_id.to_u32().to_string()); - } - - msg.update_param(context).await?; - } else { - msg.state = MessageState::OutPending; - new_msg_id = chat - .prepare_msg_raw(context, &mut msg, None, curr_timestamp) - .await?; - curr_timestamp += 1; - if !create_send_msg_jobs(context, &mut msg).await?.is_empty() { - context.scheduler.interrupt_smtp().await; - } + msg.state = MessageState::OutPending; + let new_msg_id = chat + .prepare_msg_raw(context, &mut msg, None, curr_timestamp) + .await?; + curr_timestamp += 1; + if !create_send_msg_jobs(context, &mut msg).await?.is_empty() { + context.scheduler.interrupt_smtp().await; } created_chats.push(chat_id); created_msgs.push(new_msg_id); @@ -4866,15 +4791,12 @@ mod tests { assert_eq!(test.text, "hello2".to_string()); assert_eq!(test.state, MessageState::OutDraft); - let id_after_prepare = prepare_msg(&t, *chat_id, &mut msg).await?; - assert_eq!(id_after_prepare, id_after_1st_set); - let test = Message::load_from_db(&t, id_after_prepare).await?; - assert_eq!(test.state, MessageState::OutPreparing); - assert!(!test.hidden); // sent draft must no longer be hidden - let id_after_send = send_msg(&t, *chat_id, &mut msg).await?; assert_eq!(id_after_send, id_after_1st_set); + let test = Message::load_from_db(&t, id_after_send).await?; + assert!(!test.hidden); // sent draft must no longer be hidden + Ok(()) } @@ -5626,7 +5548,6 @@ mod tests { let mut msg = Message::new_text("message text".to_string()); assert!(send_msg(&t, device_chat_id, &mut msg).await.is_err()); - assert!(prepare_msg(&t, device_chat_id, &mut msg).await.is_err()); let msg_id = add_device_msg(&t, None, Some(&mut msg)).await.unwrap(); assert!(forward_msgs(&t, &[msg_id], device_chat_id).await.is_err()); diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 276119028..f8ff056da 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -930,7 +930,6 @@ mod tests { // Alice sends a text message. let mut msg = Message::new(Viewtype::Text); - chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?; chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?; let sent = alice.pop_sent_msg().await; @@ -957,14 +956,12 @@ mod tests { // Alice sends message to Bob let mut msg = Message::new(Viewtype::Text); - chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?; chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?; let sent = alice.pop_sent_msg().await; bob.recv_msg(&sent).await; // Alice sends second message to Bob, with no timer let mut msg = Message::new(Viewtype::Text); - chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?; chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?; let sent = alice.pop_sent_msg().await; diff --git a/src/message.rs b/src/message.rs index a47d83e2b..f7467b009 100644 --- a/src/message.rs +++ b/src/message.rs @@ -953,18 +953,6 @@ impl Message { cmd != SystemMessage::Unknown } - /// Whether the message is still being created. - /// - /// Messages with attachments might be created before the - /// attachment is ready. In this case some more restrictions on - /// the attachment apply, e.g. if the file to be attached is still - /// being written to or otherwise will still change it can not be - /// copied to the blobdir. Thus those attachments need to be - /// created immediately in the blobdir with a valid filename. - pub fn is_increation(&self) -> bool { - self.viewtype.has_file() && self.state == MessageState::OutPreparing - } - /// Returns true if the message is an Autocrypt Setup Message. pub fn is_setupmessage(&self) -> bool { if self.viewtype != Viewtype::File { @@ -2206,38 +2194,6 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_prepare_message_and_send() { - let d = test::TestContext::new().await; - let ctx = &d.ctx; - - ctx.set_config(Config::ConfiguredAddr, Some("self@example.com")) - .await - .unwrap(); - - let chat = d.create_chat_with_contact("", "dest@example.com").await; - - let mut msg = Message::new(Viewtype::Text); - - let msg_id = chat::prepare_msg(ctx, chat.id, &mut msg).await.unwrap(); - - let _msg2 = Message::load_from_db(ctx, msg_id).await.unwrap(); - assert_eq!(_msg2.get_filemime(), None); - } - - /// Tests that message can be prepared even if account has no configured address. - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_prepare_not_configured() { - let d = test::TestContext::new().await; - let ctx = &d.ctx; - - let chat = d.create_chat_with_contact("", "dest@example.com").await; - - let mut msg = Message::new(Viewtype::Text); - - assert!(chat::prepare_msg(ctx, chat.id, &mut msg).await.is_ok()); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_parse_webrtc_instance() { let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar"); @@ -2357,9 +2313,9 @@ mod tests { let mut msg = Message::new_text("Quoted message".to_string()); - // Prepare message for sending, so it gets a Message-Id. + // Send message, so it gets a Message-Id. assert!(msg.rfc724_mid.is_empty()); - let msg_id = chat::prepare_msg(ctx, chat.id, &mut msg).await.unwrap(); + let msg_id = chat::send_msg(ctx, chat.id, &mut msg).await.unwrap(); let msg = Message::load_from_db(ctx, msg_id).await.unwrap(); assert!(!msg.rfc724_mid.is_empty()); diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 52820a143..8f5f66f01 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1516,7 +1516,7 @@ async fn build_body_file( ) -> Result<(PartBuilder, String)> { let blob = msg .param - .get_blob(Param::File, context, true) + .get_blob(Param::File, context) .await? .context("msg has no file")?; let suffix = blob.suffix().unwrap_or("dat"); @@ -1905,7 +1905,7 @@ mod tests { ) .await .unwrap(); - let new_msg = incoming_msg_to_reply_msg( + let mut new_msg = incoming_msg_to_reply_msg( b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ From: bob@example.com\n\ To: alice@example.org\n\ @@ -1931,6 +1931,9 @@ mod tests { Original-Message-ID: <2893@example.com>\n\ Disposition: manual-action/MDN-sent-automatically; displayed\n\ \n", &t).await; + chat::send_msg(&t, new_msg.chat_id, &mut new_msg) + .await + .unwrap(); let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap(); // The subject string should not be "Re: message opened" assert_eq!("Re: Hello, Bob", mf.subject_str(&t).await.unwrap()); @@ -2077,7 +2080,7 @@ mod tests { let mut new_msg = Message::new_text("Hi".to_string()); new_msg.chat_id = chat_id; - chat::prepare_msg(&t, chat_id, &mut new_msg).await.unwrap(); + chat::send_msg(&t, chat_id, &mut new_msg).await.unwrap(); let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap(); @@ -2134,7 +2137,7 @@ mod tests { ) -> String { let t = TestContext::new_alice().await; let mut new_msg = incoming_msg_to_reply_msg(imf_raw, &t).await; - let incoming_msg = get_chat_msg(&t, new_msg.chat_id, 0, 2).await; + let incoming_msg = get_chat_msg(&t, new_msg.chat_id, 0, 1).await; if delete_original_msg { incoming_msg.id.trash(&t, false).await.unwrap(); @@ -2164,6 +2167,9 @@ mod tests { new_msg.set_quote(&t, Some(&incoming_msg)).await.unwrap(); } + chat::send_msg(&t, new_msg.chat_id, &mut new_msg) + .await + .unwrap(); let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap(); mf.subject_str(&t).await.unwrap() } @@ -2184,9 +2190,6 @@ mod tests { let mut new_msg = Message::new_text("Hi".to_string()); new_msg.chat_id = chat_id; - chat::prepare_msg(context, chat_id, &mut new_msg) - .await - .unwrap(); new_msg } @@ -2197,7 +2200,7 @@ mod tests { let t = TestContext::new_alice().await; let context = &t; - let msg = incoming_msg_to_reply_msg( + let mut msg = incoming_msg_to_reply_msg( b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\ From: Charlie \n\ To: alice@example.org\n\ @@ -2210,6 +2213,7 @@ mod tests { context, ) .await; + chat::send_msg(&t, msg.chat_id, &mut msg).await.unwrap(); let mimefactory = MimeFactory::from_msg(&t, msg).await.unwrap(); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 52219a8f0..38a143625 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -3091,11 +3091,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== // Make sure the file is there even though the html is wrong: let param = &message.parts[0].param; - let blob: BlobObject = param - .get_blob(Param::File, &t, false) - .await - .unwrap() - .unwrap(); + let blob: BlobObject = param.get_blob(Param::File, &t).await.unwrap().unwrap(); let f = tokio::fs::File::open(blob.to_abs_path()).await.unwrap(); let size = f.metadata().await.unwrap().len(); assert_eq!(size, 154); diff --git a/src/param.rs b/src/param.rs index b65cfd1d3..d83beef6b 100644 --- a/src/param.rs +++ b/src/param.rs @@ -366,20 +366,16 @@ impl Params { /// /// This parses the parameter value as a [ParamsFile] and than /// tries to return a [BlobObject] for that file. If the file is - /// not yet a valid blob, one will be created by copying the file - /// only if `create` is set to `true`, otherwise an error is - /// returned. + /// not yet a valid blob, one will be created by copying the file. /// /// Note that in the [ParamsFile::FsPath] case the blob can be /// created without copying if the path already refers to a valid - /// blob. If so a [BlobObject] will be returned regardless of the - /// `create` argument. + /// blob. If so a [BlobObject] will be returned. #[allow(clippy::needless_lifetimes)] pub async fn get_blob<'a>( &self, key: Param, context: &'a Context, - create: bool, ) -> Result>> { let val = match self.get(key) { Some(val) => val, @@ -387,10 +383,7 @@ impl Params { }; let file = ParamsFile::from_param(context, val)?; let blob = match file { - ParamsFile::FsPath(path) => match create { - true => BlobObject::new_from_path(context, &path).await?, - false => BlobObject::from_path(context, &path)?, - }, + ParamsFile::FsPath(path) => BlobObject::new_from_path(context, &path).await?, ParamsFile::Blob(blob) => blob, }; Ok(Some(blob)) @@ -546,23 +539,20 @@ mod tests { let path: PathBuf = p.get_path(Param::File, &t).unwrap().unwrap(); assert_eq!(path, fname); - // Blob does not exist yet, expect error. - assert!(p.get_blob(Param::File, &t, false).await.is_err()); - fs::write(fname, b"boo").await.unwrap(); - let blob = p.get_blob(Param::File, &t, true).await.unwrap().unwrap(); + let blob = p.get_blob(Param::File, &t).await.unwrap().unwrap(); assert!(blob.as_file_name().starts_with("foo")); // Blob in blobdir, expect blob. let bar_path = t.get_blobdir().join("bar"); p.set(Param::File, bar_path.to_str().unwrap()); - let blob = p.get_blob(Param::File, &t, false).await.unwrap().unwrap(); + let blob = p.get_blob(Param::File, &t).await.unwrap().unwrap(); assert_eq!(blob, BlobObject::from_name(&t, "bar".to_string()).unwrap()); p.remove(Param::File); assert!(p.get_file(Param::File, &t).unwrap().is_none()); assert!(p.get_path(Param::File, &t).unwrap().is_none()); - assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none()); + assert!(p.get_blob(Param::File, &t).await.unwrap().is_none()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/sql.rs b/src/sql.rs index 47604778d..c1c46e2fb 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -864,7 +864,6 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> { if p == blobdir && (is_file_in_use(&files_in_use, None, &name_s) - || is_file_in_use(&files_in_use, Some(".increation"), &name_s) || is_file_in_use(&files_in_use, Some(".waveform"), &name_s) || is_file_in_use(&files_in_use, Some("-preview.jpg"), &name_s)) { From ee20887782f048d784f27dc52ea1d6c702b9c081 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 10 Dec 2024 02:29:34 +0000 Subject: [PATCH 08/20] feat: cache HTTP GET requests --- src/net/http.rs | 223 +++++++++++++++++++++++++++++++++++++++++- src/sql.rs | 23 +++++ src/sql/migrations.rs | 15 +++ 3 files changed, 256 insertions(+), 5 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index 94e85f68c..8006ba267 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -6,14 +6,17 @@ use http_body_util::BodyExt; use hyper_util::rt::TokioIo; use mime::Mime; use serde::Serialize; +use tokio::fs; +use crate::blob::BlobObject; use crate::context::Context; use crate::net::proxy::ProxyConfig; use crate::net::session::SessionStream; use crate::net::tls::wrap_rustls; +use crate::tools::{create_id, time}; /// HTTP(S) GET response. -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Response { /// Response body. pub blob: Vec, @@ -90,9 +93,127 @@ where Ok(sender) } +/// Converts the URL to expiration timestamp. +fn http_url_cache_expires(url: &str, mimetype: Option<&str>) -> i64 { + let now = time(); + if url.ends_with(".xdc") { + // WebXDCs expire in 5 weeks. + now + 3600 * 24 * 35 + } else if mimetype.is_some_and(|s| s.starts_with("image/")) { + // Cache images for 1 day. + now + 3600 * 24 + } else { + // Cache everything else for 1 hour. + now + 3600 + } +} + +/// Places the binary into HTTP cache. +async fn http_cache_put(context: &Context, url: &str, response: &Response) -> Result<()> { + let blob = BlobObject::create( + context, + &format!("http_cache_{}", create_id()), + response.blob.as_slice(), + ) + .await?; + + context + .sql + .insert( + "INSERT OR IGNORE INTO http_cache (url, expires, blobname, mimetype, encoding) + VALUES (?, ?, ?, ?, ?)", + ( + url, + http_url_cache_expires(url, response.mimetype.as_deref()), + blob.as_name(), + response.mimetype.as_deref().unwrap_or_default(), + response.encoding.as_deref().unwrap_or_default(), + ), + ) + .await?; + + Ok(()) +} + +/// Retrieves the binary from HTTP cache. +async fn http_cache_get(context: &Context, url: &str) -> Result> { + let Some((blob_name, mimetype, encoding)) = context + .sql + .query_row_optional( + "SELECT blobname, mimetype, encoding + FROM http_cache WHERE url=? AND expires > ?", + (url, time()), + |row| { + let blob_name: String = row.get(0)?; + let mimetype: Option = Some(row.get(1)?).filter(|s: &String| !s.is_empty()); + let encoding: Option = Some(row.get(2)?).filter(|s: &String| !s.is_empty()); + Ok((blob_name, mimetype, encoding)) + }, + ) + .await? + else { + return Ok(None); + }; + + let blob_object = BlobObject::from_name(context, blob_name)?; + let blob_abs_path = blob_object.to_abs_path(); + let blob = match fs::read(blob_abs_path) + .await + .with_context(|| format!("Failed to read blob for {url:?} cache entry.")) + { + Ok(blob) => blob, + Err(err) => { + // This should not happen, but user may go into the blobdir and remove files, + // antivirus may delete the file or there may be a bug in housekeeping. + warn!(context, "{err:?}."); + return Ok(None); + } + }; + + let expires = http_url_cache_expires(url, mimetype.as_deref()); + let response = Response { + blob, + mimetype, + encoding, + }; + + // Update expiration timestamp. + context + .sql + .execute( + "UPDATE http_cache SET expires=? WHERE url=?", + (expires, url), + ) + .await?; + + Ok(Some(response)) +} + +/// Removes expired cache entries. +pub(crate) async fn http_cache_cleanup(context: &Context) -> Result<()> { + // Remove cache entries that are already expired + // or entries that will not expire in a year + // to make sure we don't have invalid timestamps that are way forward in the future. + context + .sql + .execute( + "DELETE FROM http_cache + WHERE ?1 > expires OR expires > ?1 + 31536000", + (time(),), + ) + .await?; + Ok(()) +} + /// Retrieves the binary contents of URL using HTTP GET request. -pub async fn read_url_blob(context: &Context, url: &str) -> Result { - let mut url = url.to_string(); +pub async fn read_url_blob(context: &Context, original_url: &str) -> Result { + if let Some(response) = http_cache_get(context, original_url).await? { + info!(context, "Returning {original_url:?} from cache."); + return Ok(response); + } + + info!(context, "Not found {original_url:?} in cache."); + let mut url = original_url.to_string(); // Follow up to 10 http-redirects for _i in 0..10 { @@ -139,11 +260,14 @@ pub async fn read_url_blob(context: &Context, url: &str) -> Result { }); let body = response.collect().await?.to_bytes(); let blob: Vec = body.to_vec(); - return Ok(Response { + let response = Response { blob, mimetype, encoding, - }); + }; + info!(context, "Inserting {original_url:?} into cache."); + http_cache_put(context, &url, &response).await?; + return Ok(response); } Err(anyhow!("Followed 10 redirections")) @@ -241,3 +365,92 @@ pub(crate) async fn post_form( let bytes = response.collect().await?.to_bytes(); Ok(bytes) } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + use crate::sql::housekeeping; + use crate::test_utils::TestContext; + use crate::tools::SystemTime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_http_cache() -> Result<()> { + let t = &TestContext::new().await; + + assert_eq!(http_cache_get(t, "https://webxdc.org/").await?, None); + + let html_response = Response { + blob: b" ...".to_vec(), + mimetype: Some("text/html".to_string()), + encoding: None, + }; + + let xdc_response = Response { + blob: b"PK...".to_vec(), + mimetype: Some("application/octet-stream".to_string()), + encoding: None, + }; + let xdc_editor_url = "https://apps.testrun.org/webxdc-editor-v3.2.0.xdc"; + let xdc_pixel_url = "https://apps.testrun.org/webxdc-pixel-v2.xdc"; + + http_cache_put(t, "https://webxdc.org/", &html_response).await?; + + assert_eq!(http_cache_get(t, xdc_editor_url).await?, None); + assert_eq!(http_cache_get(t, xdc_pixel_url).await?, None); + assert_eq!( + http_cache_get(t, "https://webxdc.org/").await?, + Some(html_response.clone()) + ); + + http_cache_put(t, xdc_editor_url, &xdc_response).await?; + assert_eq!( + http_cache_get(t, xdc_editor_url).await?, + Some(xdc_response.clone()) + ); + + assert_eq!( + http_cache_get(t, "https://webxdc.org/").await?, + Some(html_response.clone()) + ); + + // HTML expires after 1 hour, but .xdc does not. + SystemTime::shift(Duration::from_secs(3600 + 100)); + assert_eq!(http_cache_get(t, "https://webxdc.org/").await?, None); + assert_eq!( + http_cache_get(t, xdc_editor_url).await?, + Some(xdc_response.clone()) + ); + + // 35 days later pixel .xdc expires because we did not request it for 35 days and 1 hour. + // But editor is still there because we did not request it for just 35 days. + SystemTime::shift(Duration::from_secs(3600 * 24 * 35 - 100)); + + // Run housekeeping to test that it does not delete the blob too early. + housekeeping(t).await?; + + assert_eq!( + http_cache_get(t, xdc_editor_url).await?, + Some(xdc_response.clone()) + ); + assert_eq!(http_cache_get(t, xdc_pixel_url).await?, None); + + // Test that if the file is accidentally removed from the blobdir, + // there is no error when trying to load the cache entry. + for entry in std::fs::read_dir(t.get_blobdir())? { + let entry = entry.unwrap(); + let path = entry.path(); + std::fs::remove_file(path).expect("Failed to remove blob"); + } + + assert_eq!( + http_cache_get(t, xdc_editor_url) + .await + .context("Failed to get no cache response")?, + None + ); + + Ok(()) + } +} diff --git a/src/sql.rs b/src/sql.rs index c1c46e2fb..62a0b44b9 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -19,6 +19,7 @@ use crate::location::delete_orphaned_poi_locations; use crate::log::LogExt; use crate::message::{Message, MsgId}; use crate::net::dns::prune_dns_cache; +use crate::net::http::http_cache_cleanup; use crate::net::prune_connection_history; use crate::param::{Param, Params}; use crate::peerstate::Peerstate; @@ -720,6 +721,12 @@ pub async fn housekeeping(context: &Context) -> Result<()> { warn!(context, "Can't set config: {e:#}."); } + http_cache_cleanup(context) + .await + .context("Failed to cleanup HTTP cache") + .log_err(context) + .ok(); + if let Err(err) = remove_unused_files(context).await { warn!( context, @@ -846,6 +853,22 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> { .await .context("housekeeping: failed to SELECT value FROM config")?; + context + .sql + .query_map( + "SELECT blobname FROM http_cache", + (), + |row| row.get::<_, String>(0), + |rows| { + for row in rows { + maybe_add_file(&mut files_in_use, &row?); + } + Ok(()) + }, + ) + .await + .context("Failed to SELECT blobname FROM http_cache")?; + info!(context, "{} files in use.", files_in_use.len()); /* go through directories and delete unused files */ let blobdir = context.get_blobdir(); diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 15d89c6f1..0827f924e 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1088,6 +1088,21 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid); .await?; } + inc_and_check(&mut migration_version, 125)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE http_cache ( + url TEXT PRIMARY KEY, + expires INTEGER NOT NULL, -- When the cache entry is considered expired, timestamp in seconds. + blobname TEXT NOT NULL, + mimetype TEXT NOT NULL DEFAULT '', -- MIME type extracted from Content-Type header. + encoding TEXT NOT NULL DEFAULT '' -- Encoding from Content-Type header. + ) STRICT", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? From 33169dd49a3e91967f998b6af328cbfebeb20e03 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Dec 2024 23:09:10 +0000 Subject: [PATCH 09/20] test: actually insert pixel app into HTTP cache --- src/net/http.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/net/http.rs b/src/net/http.rs index 8006ba267..814fe57ac 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -405,10 +405,15 @@ mod tests { ); http_cache_put(t, xdc_editor_url, &xdc_response).await?; + http_cache_put(t, xdc_pixel_url, &xdc_response).await?; assert_eq!( http_cache_get(t, xdc_editor_url).await?, Some(xdc_response.clone()) ); + assert_eq!( + http_cache_get(t, xdc_pixel_url).await?, + Some(xdc_response.clone()) + ); assert_eq!( http_cache_get(t, "https://webxdc.org/").await?, From 698136b30c3a7ab1f493536634e479dffe5c8b4c Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Dec 2024 23:09:39 +0000 Subject: [PATCH 10/20] test: test that HTTP cache can be renewed without housekeeping --- src/net/http.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/net/http.rs b/src/net/http.rs index 814fe57ac..fe3d90898 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -428,6 +428,14 @@ mod tests { Some(xdc_response.clone()) ); + // Expired cache entry can be renewed + // even before housekeeping removes old one. + http_cache_put(t, "https://webxdc.org/", &html_response).await?; + assert_eq!( + http_cache_get(t, "https://webxdc.org/").await?, + Some(html_response.clone()) + ); + // 35 days later pixel .xdc expires because we did not request it for 35 days and 1 hour. // But editor is still there because we did not request it for just 35 days. SystemTime::shift(Duration::from_secs(3600 * 24 * 35 - 100)); From cc672b81faaf1cde62f665b1ef5d2af525f02878 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 11 Dec 2024 23:16:52 +0000 Subject: [PATCH 11/20] fix: renew HTTP cache entry if it already exists --- src/net/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/http.rs b/src/net/http.rs index fe3d90898..210114cab 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -120,7 +120,7 @@ async fn http_cache_put(context: &Context, url: &str, response: &Response) -> Re context .sql .insert( - "INSERT OR IGNORE INTO http_cache (url, expires, blobname, mimetype, encoding) + "INSERT OR REPLACE INTO http_cache (url, expires, blobname, mimetype, encoding) VALUES (?, ?, ?, ?, ?)", ( url, From ffe6efe819abdd894dd317d46b62cc46a0814301 Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 9 Dec 2024 21:09:14 +0000 Subject: [PATCH 12/20] build: increase MSRV to 1.81.0 --- .github/workflows/ci.yml | 4 ++-- Cargo.toml | 2 +- src/chat.rs | 4 ++-- src/contact.rs | 1 - src/imap.rs | 3 +-- src/mimefactory.rs | 1 - src/mimeparser.rs | 2 +- src/param.rs | 1 - src/pgp.rs | 2 -- src/quota.rs | 2 +- src/receive_imf.rs | 9 ++++----- src/smtp.rs | 2 +- src/smtp/connect.rs | 2 +- 13 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab63e5b16..dbd77e872 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,9 +103,9 @@ jobs: - os: macos-latest rust: 1.83.0 - # Minimum Supported Rust Version = 1.77.0 + # Minimum Supported Rust Version = 1.81.0 - os: ubuntu-latest - rust: 1.77.0 + rust: 1.81.0 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/Cargo.toml b/Cargo.toml index b604729f9..8a42dba29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "deltachat" version = "1.151.6" edition = "2021" license = "MPL-2.0" -rust-version = "1.77" +rust-version = "1.81" repository = "https://github.com/deltachat/deltachat-core-rust" [profile.dev] diff --git a/src/chat.rs b/src/chat.rs index 437dbff4e..320fffb85 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -312,7 +312,7 @@ impl ChatId { /// Create a group or mailinglist raw database record with the given parameters. /// The function does not add SELF nor checks if the record already exists. - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] pub(crate) async fn create_multiuser_record( context: &Context, chattype: Chattype, @@ -4444,7 +4444,7 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul /// /// For example, it can be a message showing that a member was added to a group. /// Doesn't fail if the chat doesn't exist. -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] pub(crate) async fn add_info_msg_with_cmd( context: &Context, chat_id: ChatId, diff --git a/src/contact.rs b/src/contact.rs index 6ce4ecfa9..61ad7db1f 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -805,7 +805,6 @@ impl Contact { } let mut name = sanitize_name(name); - #[allow(clippy::collapsible_if)] if origin <= Origin::OutgoingTo { // The user may accidentally have written to a "noreply" address with another MUA: if addr.contains("noreply") diff --git a/src/imap.rs b/src/imap.rs index 50a159b38..cbf241071 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -1301,7 +1301,7 @@ impl Session { /// Returns the last UID fetched successfully and the info about each downloaded message. /// If the message is incorrect or there is a failure to write a message to the database, /// it is skipped and the error is logged. - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] pub(crate) async fn fetch_many_msgs( &mut self, context: &Context, @@ -2689,7 +2689,6 @@ mod tests { } } - #[allow(clippy::too_many_arguments)] async fn check_target_folder_combination( folder: &str, mvbox_move: bool, diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 8f5f66f01..878007304 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1047,7 +1047,6 @@ impl MimeFactory { part.body(text) } - #[allow(clippy::cognitive_complexity)] async fn render_message( &mut self, context: &Context, diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 38a143625..42f4fda54 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -1281,7 +1281,7 @@ impl MimeMessage { Ok(self.parts.len() > old_part_count) } - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] async fn do_add_single_file_part( &mut self, context: &Context, diff --git a/src/param.rs b/src/param.rs index d83beef6b..f1d3cb761 100644 --- a/src/param.rs +++ b/src/param.rs @@ -371,7 +371,6 @@ impl Params { /// Note that in the [ParamsFile::FsPath] case the blob can be /// created without copying if the path already refers to a valid /// blob. If so a [BlobObject] will be returned. - #[allow(clippy::needless_lifetimes)] pub async fn get_blob<'a>( &self, key: Param, diff --git a/src/pgp.rs b/src/pgp.rs index f729237d9..804d520c0 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -21,11 +21,9 @@ use tokio::runtime::Handle; use crate::constants::KeyGenType; use crate::key::{DcKey, Fingerprint}; -#[allow(missing_docs)] #[cfg(test)] pub(crate) const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt"; -#[allow(missing_docs)] pub const HEADER_SETUPCODE: &str = "passphrase-begin"; /// Preferred symmetric encryption algorithm. diff --git a/src/quota.rs b/src/quota.rs index 0070a76ca..a0e8df346 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -187,7 +187,7 @@ mod tests { Ok(()) } - #[allow(clippy::assertions_on_constants)] + #[expect(clippy::assertions_on_constants)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_quota_thresholds() -> anyhow::Result<()> { assert!(QUOTA_ALLCLEAR_PERCENTAGE > 50); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 0d2d6ee97..12a776750 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -158,7 +158,7 @@ async fn insert_tombstone(context: &Context, rfc724_mid: &str) -> Result /// If `is_partial_download` is set, it contains the full message size in bytes. /// Do not confuse that with `replace_msg_id` that will be set when the full message is loaded /// later. -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] pub(crate) async fn receive_imf_inner( context: &Context, folder: &str, @@ -679,7 +679,7 @@ pub async fn from_field_to_contact_id( /// Creates a `ReceivedMsg` from given parts which might consist of /// multiple messages (if there are multiple attachments). /// Every entry in `mime_parser.parts` produces a new row in the `msgs` table. -#[allow(clippy::too_many_arguments, clippy::cognitive_complexity)] +#[expect(clippy::too_many_arguments)] async fn add_parts( context: &Context, mime_parser: &mut MimeMessage, @@ -1836,7 +1836,7 @@ async fn lookup_chat_by_reply( Ok(Some((parent_chat.id, parent_chat.blocked))) } -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] async fn lookup_chat_or_create_adhoc_group( context: &Context, mime_parser: &MimeMessage, @@ -1971,7 +1971,7 @@ async fn is_probably_private_reply( /// than two members, a new ad hoc group is created. /// /// On success the function returns the created (chat_id, chat_blocked) tuple. -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] async fn create_group( context: &Context, mime_parser: &mut MimeMessage, @@ -2091,7 +2091,6 @@ async fn create_group( /// just omitted. /// /// * `is_partial_download` - whether the message is not fully downloaded. -#[allow(clippy::too_many_arguments)] async fn apply_group_changes( context: &Context, mime_parser: &mut MimeMessage, diff --git a/src/smtp.rs b/src/smtp.rs index 7bc51a278..84b093642 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -104,7 +104,7 @@ impl Smtp { } /// Connect using the provided login params. - #[allow(clippy::too_many_arguments)] + #[expect(clippy::too_many_arguments)] pub async fn connect( &mut self, context: &Context, diff --git a/src/smtp/connect.rs b/src/smtp/connect.rs index 5652d1427..869405cd3 100644 --- a/src/smtp/connect.rs +++ b/src/smtp/connect.rs @@ -45,7 +45,7 @@ async fn new_smtp_transport( Ok(transport) } -#[allow(clippy::too_many_arguments)] +#[expect(clippy::too_many_arguments)] pub(crate) async fn connect_and_auth( context: &Context, proxy_config: &Option, From bb3075c6fd63449b4817ee3012a7d82dbe76afb5 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 11 Nov 2024 12:16:04 -0300 Subject: [PATCH 13/20] test: Record the current wrong behaviour of HTML display of multipart messages (#4462) --- src/receive_imf/tests.rs | 34 ++ .../big_forwarded_with_big_attachment.eml | 305 ++++++++++++++++++ 2 files changed, 339 insertions(+) create mode 100644 test-data/message/big_forwarded_with_big_attachment.eml diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index f798407f8..a9e9dfabf 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -3827,6 +3827,40 @@ async fn test_messed_up_message_id() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_big_forwarded_with_big_attachment() -> Result<()> { + let t = &TestContext::new_bob().await; + + let raw = include_bytes!("../../test-data/message/big_forwarded_with_big_attachment.eml"); + let rcvd = receive_imf(t, raw, false).await?.unwrap(); + assert_eq!(rcvd.msg_ids.len(), 3); + let msg = Message::load_from_db(t, rcvd.msg_ids[0]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::Text); + assert_eq!(msg.get_text(), "Hello!"); + // Wrong: the second bubble's text is truncated, but "Show Full Message..." is going to be shown + // in the first message bubble in the UIs. + assert_eq!( + msg.id + .get_html(t) + .await? + .unwrap() + .matches("Hello!") + .count(), + 1 + ); + let msg = Message::load_from_db(t, rcvd.msg_ids[1]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::Text); + assert!(msg.get_text().starts_with("this text with 42 chars is just repeated.")); + assert!(msg.get_text().ends_with("[...]")); + // Wrong: the text is truncated, but it's not possible to see the full text in HTML. + assert!(!msg.has_html()); + let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::File); + assert!(!msg.has_html()); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_mua_user_adds_member() -> Result<()> { let t = TestContext::new_alice().await; diff --git a/test-data/message/big_forwarded_with_big_attachment.eml b/test-data/message/big_forwarded_with_big_attachment.eml new file mode 100644 index 000000000..f6ae39132 --- /dev/null +++ b/test-data/message/big_forwarded_with_big_attachment.eml @@ -0,0 +1,305 @@ +From: Alice +To: Bob +Date: Fri, 2 Jun 2023 13:29:17 +0000 +Message-ID: +Content-Type: multipart/mixed; boundary="zRs3OquGy6eU58KF" + + +--zRs3OquGy6eU58KF +Content-Type: text/plain; charset=us-ascii +Content-Disposition: inline + +Hello! + +--zRs3OquGy6eU58KF +Content-Type: message/rfc822 +Content-Disposition: inline + +From: AAA +To: Alice +Subject: Some subject +Date: Fri, 2 Jun 2023 12:29:17 +0000 +Message-ID: +In-Reply-To: +Content-Type: multipart/mixed; + boundary="_innerboundary_" +MIME-Version: 1.0 + +--_innerboundary_ +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: quoted-printable + +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. + +--=20 + +Kind regards, + +Bob + +--_innerboundary_ +Content-Type: text/plain; name="deltachat-log.txt" +Content-Description: deltachat-log.txt +Content-Disposition: attachment; filename="deltachat-log.txt"; + size=55254; creation-date="Fri, 02 Jun 2023 11:33:49 GMT"; + modification-date="Fri, 02 Jun 2023 12:29:17 GMT" + +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. +this text with 42 chars is just repeated. + +--_innerboundary_-- + +--zRs3OquGy6eU58KF-- From 253362899bd7c7e8b8333689a34deba92ea3a420 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 10 Dec 2024 15:33:30 -0300 Subject: [PATCH 14/20] feat: Set `mime_modified` for the last message part, not the first (#4462) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise the "Show Full Message..." button appears somewhere in the middle of the multipart message, e.g. after a text in the first message bubble, but before a text in the second bubble. Moreover, if the second/n-th bubble's text is shortened (ends with "[...]"), the user should scroll up to click on "Show Full Message..." which doesn't look reasonable. Scrolling down looks more acceptable (e.g. if the first bubble's text is shortened in a multipart message). I'd even suggest to show somehow that message bubbles belong to the same multipart message, e.g. add "[↡]" to the text of all bubbles except the last one, but let's discuss this first. --- src/mimeparser.rs | 10 ++++------ src/receive_imf.rs | 26 ++++++++++++-------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 42f4fda54..4eb80b4d9 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -105,14 +105,12 @@ pub(crate) struct MimeMessage { /// received. pub(crate) footer: Option, - // if this flag is set, the parts/text/etc. are just close to the original mime-message; - // clients should offer a way to view the original message in this case + /// If set, this is a modified MIME message; clients should offer a way to view the original + /// MIME message in this case. pub is_mime_modified: bool, - /// The decrypted, raw mime structure. - /// - /// This is non-empty iff `is_mime_modified` and the message was actually encrypted. It is used - /// for e.g. late-parsing HTML. + /// Decrypted, raw MIME structure. Nonempty iff `is_mime_modified` and the message was actually + /// encrypted. pub decoded_data: Vec, /// Hop info for debugging. diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 12a776750..439731e9b 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1406,10 +1406,11 @@ async fn add_parts( // we save the full mime-message and add a flag // that the ui should show button to display the full message. - // a flag used to avoid adding "show full message" button to multiple parts of the message. - let mut save_mime_modified = mime_parser.is_mime_modified; + // We add "Show Full Message" button to the last message bubble (part) if this flag evaluates to + // `true` finally. + let mut save_mime_modified = false; - let mime_headers = if save_mime_headers || save_mime_modified { + let mime_headers = if save_mime_headers || mime_parser.is_mime_modified { let headers = if !mime_parser.decoded_data.is_empty() { mime_parser.decoded_data.clone() } else { @@ -1475,7 +1476,8 @@ async fn add_parts( } } - for part in &mime_parser.parts { + let mut parts = mime_parser.parts.iter().peekable(); + while let Some(part) = parts.next() { if part.is_reaction { let reaction_str = simplify::remove_footers(part.msg.as_str()); let is_incoming_fresh = mime_parser.incoming && !seen && !fetching_existing_messages; @@ -1519,14 +1521,11 @@ async fn add_parts( } else { (&part.msg, part.typ) }; - let part_is_empty = typ == Viewtype::Text && msg.is_empty() && part.param.get(Param::Quote).is_none(); - let mime_modified = save_mime_modified && !part_is_empty; - if mime_modified { - // Avoid setting mime_modified for more than one part. - save_mime_modified = false; - } + + save_mime_modified |= mime_parser.is_mime_modified && !part_is_empty && !hidden; + let save_mime_modified = save_mime_modified && parts.peek().is_none(); if part.typ == Viewtype::Text { let msg_raw = part.msg_raw.as_ref().cloned().unwrap_or_default(); @@ -1546,8 +1545,7 @@ async fn add_parts( // If you change which information is skipped if the message is trashed, // also change `MsgId::trash()` and `delete_expired_messages()` - let trash = - chat_id.is_trash() || (is_location_kml && msg.is_empty() && typ == Viewtype::Text); + let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified); let row_id = context .sql @@ -1610,14 +1608,14 @@ RETURNING id }, hidden, part.bytes as isize, - if (save_mime_headers || mime_modified) && !trash { + if (save_mime_headers || save_mime_modified) && !trash { mime_headers.clone() } else { Vec::new() }, mime_in_reply_to, mime_references, - mime_modified, + save_mime_modified, part.error.as_deref().unwrap_or_default(), ephemeral_timer, ephemeral_timestamp, From cb2157822a597d8f0ae268be96abe28fb436ba99 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 10 Dec 2024 16:04:07 -0300 Subject: [PATCH 15/20] fix: Render "message" parts in multipart messages' HTML (#4462) This fixes the HTML display of messages containing forwarded messages. Before, forwarded messages weren't rendered in HTML and if a forwarded message is long and therefore truncated in the chat, it could only be seen in the "Message Info". In #4462 it was suggested to display "Show Full Message..." for each truncated message part and save to `msgs.mime_headers` only the corresponding part, but this is a quite huge change and refactoring and also it may be good that currently we save the full message structure to `msgs.mime_headers`, so i'd suggest not to change this for now. --- src/html.rs | 78 +++++++++++++++++++++++++++------------- src/receive_imf/tests.rs | 59 ++++++++++++++++++++---------- 2 files changed, 93 insertions(+), 44 deletions(-) diff --git a/src/html.rs b/src/html.rs index e32ee4357..c58f099c8 100644 --- a/src/html.rs +++ b/src/html.rs @@ -7,6 +7,8 @@ //! `MsgId.get_html()` will return HTML - //! this allows nice quoting, handling linebreaks properly etc. +use std::mem; + use anyhow::{Context as _, Result}; use base64::Engine as _; use lettre_email::mime::Mime; @@ -77,21 +79,26 @@ fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType { struct HtmlMsgParser { pub html: String, pub plain: Option, + pub(crate) msg_html: String, } impl HtmlMsgParser { /// Function takes a raw mime-message string, /// searches for the main-text part /// and returns that as parser.html - pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> { + pub async fn from_bytes<'a>( + context: &Context, + rawmime: &'a [u8], + ) -> Result<(Self, mailparse::ParsedMail<'a>)> { let mut parser = HtmlMsgParser { html: "".to_string(), plain: None, + msg_html: "".to_string(), }; - let parsedmail = mailparse::parse_mail(rawmime)?; + let parsedmail = mailparse::parse_mail(rawmime).context("Failed to parse mail")?; - parser.collect_texts_recursive(&parsedmail).await?; + parser.collect_texts_recursive(context, &parsedmail).await?; if parser.html.is_empty() { if let Some(plain) = &parser.plain { @@ -100,8 +107,8 @@ impl HtmlMsgParser { } else { parser.cid_to_data_recursive(context, &parsedmail).await?; } - - Ok(parser) + parser.html += &mem::take(&mut parser.msg_html); + Ok((parser, parsedmail)) } /// Function iterates over all mime-parts @@ -114,12 +121,13 @@ impl HtmlMsgParser { /// therefore we use the first one. async fn collect_texts_recursive<'a>( &'a mut self, + context: &'a Context, mail: &'a mailparse::ParsedMail<'a>, ) -> Result<()> { match get_mime_multipart_type(&mail.ctype) { MimeMultipartType::Multiple => { for cur_data in &mail.subparts { - Box::pin(self.collect_texts_recursive(cur_data)).await? + Box::pin(self.collect_texts_recursive(context, cur_data)).await? } Ok(()) } @@ -128,8 +136,35 @@ impl HtmlMsgParser { if raw.is_empty() { return Ok(()); } - let mail = mailparse::parse_mail(&raw).context("failed to parse mail")?; - Box::pin(self.collect_texts_recursive(&mail)).await + let (parser, mail) = Box::pin(HtmlMsgParser::from_bytes(context, &raw)).await?; + if !parser.html.is_empty() { + let mut text = "\r\n\r\n".to_string(); + for h in mail.headers { + let key = h.get_key(); + if matches!( + key.to_lowercase().as_str(), + "date" + | "from" + | "sender" + | "reply-to" + | "to" + | "cc" + | "bcc" + | "subject" + ) { + text += &format!("{key}: {}\r\n", h.get_value()); + } + } + text += "\r\n"; + self.msg_html += &PlainText { + text, + flowed: false, + delsp: false, + } + .to_html(); + self.msg_html += &parser.html; + } + Ok(()) } MimeMultipartType::Single => { let mimetype = mail.ctype.mimetype.parse::<Mime>()?; @@ -175,14 +210,7 @@ impl HtmlMsgParser { } Ok(()) } - MimeMultipartType::Message => { - let raw = mail.get_body_raw()?; - if raw.is_empty() { - return Ok(()); - } - let mail = mailparse::parse_mail(&raw).context("failed to parse mail")?; - Box::pin(self.cid_to_data_recursive(context, &mail)).await - } + MimeMultipartType::Message => Ok(()), MimeMultipartType::Single => { let mimetype = mail.ctype.mimetype.parse::<Mime>()?; if mimetype.type_() == mime::IMAGE { @@ -240,7 +268,7 @@ impl MsgId { warn!(context, "get_html: parser error: {:#}", err); Ok(None) } - Ok(parser) => Ok(Some(parser.html)), + Ok((parser, _)) => Ok(Some(parser.html)), } } else { warn!(context, "get_html: no mime for {}", self); @@ -274,7 +302,7 @@ mod tests { async fn test_htmlparse_plain_unspecified() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert_eq!( parser.html, r#"<!DOCTYPE html> @@ -292,7 +320,7 @@ This message does not have Content-Type nor Subject.<br/> async fn test_htmlparse_plain_iso88591() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert_eq!( parser.html, r#"<!DOCTYPE html> @@ -310,7 +338,7 @@ message with a non-UTF-8 encoding: Γ€ΓΆΓΌΓŸΓ„Γ–Γœ<br/> async fn test_htmlparse_plain_flowed() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_plain_flowed.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert!(parser.plain.unwrap().flowed); assert_eq!( parser.html, @@ -332,7 +360,7 @@ and will be wrapped as usual.<br/> async fn test_htmlparse_alt_plain() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_alt_plain.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert_eq!( parser.html, r#"<!DOCTYPE html> @@ -353,7 +381,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x async fn test_htmlparse_html() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_html.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); // on windows, `\r\n` linends are returned from mimeparser, // however, rust multiline-strings use just `\n`; @@ -371,7 +399,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x async fn test_htmlparse_alt_html() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_alt_html.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert_eq!( parser.html.replace('\r', ""), // see comment in test_htmlparse_html() r##"<html> @@ -386,7 +414,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x async fn test_htmlparse_alt_plain_html() { let t = TestContext::new().await; let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml"); - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert_eq!( parser.html.replace('\r', ""), // see comment in test_htmlparse_html() r##"<html> @@ -411,7 +439,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x assert!(test.find("data:").is_none()); // parsing converts cid: to data: - let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); + let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap(); assert!(parser.html.contains("<html>")); assert!(!parser.html.contains("Content-Id:")); assert!(parser.html.contains("data:image/jpeg;base64,/9j/4AAQ")); diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index a9e9dfabf..0bb83ceb9 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -3834,30 +3834,51 @@ async fn test_big_forwarded_with_big_attachment() -> Result<()> { let raw = include_bytes!("../../test-data/message/big_forwarded_with_big_attachment.eml"); let rcvd = receive_imf(t, raw, false).await?.unwrap(); assert_eq!(rcvd.msg_ids.len(), 3); + let msg = Message::load_from_db(t, rcvd.msg_ids[0]).await?; assert_eq!(msg.get_viewtype(), Viewtype::Text); assert_eq!(msg.get_text(), "Hello!"); - // Wrong: the second bubble's text is truncated, but "Show Full Message..." is going to be shown - // in the first message bubble in the UIs. - assert_eq!( - msg.id - .get_html(t) - .await? - .unwrap() - .matches("Hello!") - .count(), - 1 - ); - let msg = Message::load_from_db(t, rcvd.msg_ids[1]).await?; - assert_eq!(msg.get_viewtype(), Viewtype::Text); - assert!(msg.get_text().starts_with("this text with 42 chars is just repeated.")); - assert!(msg.get_text().ends_with("[...]")); - // Wrong: the text is truncated, but it's not possible to see the full text in HTML. - assert!(!msg.has_html()); - let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?; - assert_eq!(msg.get_viewtype(), Viewtype::File); assert!(!msg.has_html()); + let msg = Message::load_from_db(t, rcvd.msg_ids[1]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::Text); + assert!(msg + .get_text() + .starts_with("this text with 42 chars is just repeated.")); + assert!(msg.get_text().ends_with("[...]")); + assert!(!msg.has_html()); + + let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::File); + assert!(msg.has_html()); + let html = msg.id.get_html(t).await?.unwrap(); + let tail = html + .split_once("Hello!") + .unwrap() + .1 + .split_once("From: AAA") + .unwrap() + .1 + .split_once("aaa@example.org") + .unwrap() + .1 + .split_once("To: Alice") + .unwrap() + .1 + .split_once("alice@example.org") + .unwrap() + .1 + .split_once("Subject: Some subject") + .unwrap() + .1 + .split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000") + .unwrap() + .1; + assert_eq!( + tail.matches("this text with 42 chars is just repeated.") + .count(), + 128 + ); Ok(()) } From 2f976d8050bb55993b80a3695a1bebb5d1d29eed Mon Sep 17 00:00:00 2001 From: link2xt <link2xt@testrun.org> Date: Thu, 12 Dec 2024 11:17:13 +0000 Subject: [PATCH 16/20] feat: implement stale-while-revalidate for HTTP cache --- src/net/http.rs | 118 +++++++++++++++++++++++++++++------------- src/sql/migrations.rs | 18 +++++++ 2 files changed, 100 insertions(+), 36 deletions(-) diff --git a/src/net/http.rs b/src/net/http.rs index 210114cab..7360ed362 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -93,19 +93,28 @@ where Ok(sender) } -/// Converts the URL to expiration timestamp. -fn http_url_cache_expires(url: &str, mimetype: Option<&str>) -> i64 { +/// Converts the URL to expiration and stale timestamps. +fn http_url_cache_timestamps(url: &str, mimetype: Option<&str>) -> (i64, i64) { let now = time(); - if url.ends_with(".xdc") { - // WebXDCs expire in 5 weeks. - now + 3600 * 24 * 35 + + let expires = now + 3600 * 24 * 35; + let stale = if url.ends_with(".xdc") { + // WebXDCs are never stale, they just expire. + expires } else if mimetype.is_some_and(|s| s.starts_with("image/")) { // Cache images for 1 day. + // + // As of 2024-12-12 WebXDC icons at <https://webxdc.org/apps/> + // use the same path for all app versions, + // so may change, but it is not critical if outdated icon is displayed. now + 3600 * 24 } else { - // Cache everything else for 1 hour. + // Revalidate everything else after 1 hour. + // + // This includes HTML, CSS and JS. now + 3600 - } + }; + (expires, stale) } /// Places the binary into HTTP cache. @@ -117,14 +126,16 @@ async fn http_cache_put(context: &Context, url: &str, response: &Response) -> Re ) .await?; + let (expires, stale) = http_url_cache_timestamps(url, response.mimetype.as_deref()); context .sql .insert( - "INSERT OR REPLACE INTO http_cache (url, expires, blobname, mimetype, encoding) - VALUES (?, ?, ?, ?, ?)", + "INSERT OR REPLACE INTO http_cache (url, expires, stale, blobname, mimetype, encoding) + VALUES (?, ?, ?, ?, ?, ?)", ( url, - http_url_cache_expires(url, response.mimetype.as_deref()), + expires, + stale, blob.as_name(), response.mimetype.as_deref().unwrap_or_default(), response.encoding.as_deref().unwrap_or_default(), @@ -136,18 +147,22 @@ async fn http_cache_put(context: &Context, url: &str, response: &Response) -> Re } /// Retrieves the binary from HTTP cache. -async fn http_cache_get(context: &Context, url: &str) -> Result<Option<Response>> { - let Some((blob_name, mimetype, encoding)) = context +/// +/// Also returns if the response is stale and should be revalidated in the background. +async fn http_cache_get(context: &Context, url: &str) -> Result<Option<(Response, bool)>> { + let now = time(); + let Some((blob_name, mimetype, encoding, is_stale)) = context .sql .query_row_optional( - "SELECT blobname, mimetype, encoding + "SELECT blobname, mimetype, encoding, stale FROM http_cache WHERE url=? AND expires > ?", - (url, time()), + (url, now), |row| { let blob_name: String = row.get(0)?; let mimetype: Option<String> = Some(row.get(1)?).filter(|s: &String| !s.is_empty()); let encoding: Option<String> = Some(row.get(2)?).filter(|s: &String| !s.is_empty()); - Ok((blob_name, mimetype, encoding)) + let stale_timestamp: i64 = row.get(3)?; + Ok((blob_name, mimetype, encoding, now > stale_timestamp)) }, ) .await? @@ -170,14 +185,20 @@ async fn http_cache_get(context: &Context, url: &str) -> Result<Option<Response> } }; - let expires = http_url_cache_expires(url, mimetype.as_deref()); + let (expires, _stale) = http_url_cache_timestamps(url, mimetype.as_deref()); let response = Response { blob, mimetype, encoding, }; - // Update expiration timestamp. + // Update expiration timestamp + // to prevent deletion of the file still in use. + // + // We do not update stale timestamp here + // as we have not revalidated the response. + // Stale timestamp is updated only + // when the URL is sucessfully fetched. context .sql .execute( @@ -186,7 +207,7 @@ async fn http_cache_get(context: &Context, url: &str) -> Result<Option<Response> ) .await?; - Ok(Some(response)) + Ok(Some((response, is_stale))) } /// Removes expired cache entries. @@ -205,14 +226,10 @@ pub(crate) async fn http_cache_cleanup(context: &Context) -> Result<()> { Ok(()) } -/// Retrieves the binary contents of URL using HTTP GET request. -pub async fn read_url_blob(context: &Context, original_url: &str) -> Result<Response> { - if let Some(response) = http_cache_get(context, original_url).await? { - info!(context, "Returning {original_url:?} from cache."); - return Ok(response); - } - - info!(context, "Not found {original_url:?} in cache."); +/// Fetches URL and updates the cache. +/// +/// URL is fetched regardless of whether there is an existing result in the cache. +async fn fetch_url(context: &Context, original_url: &str) -> Result<Response> { let mut url = original_url.to_string(); // Follow up to 10 http-redirects @@ -273,6 +290,31 @@ pub async fn read_url_blob(context: &Context, original_url: &str) -> Result<Resp Err(anyhow!("Followed 10 redirections")) } +/// Retrieves the binary contents of URL using HTTP GET request. +pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> { + if let Some((response, is_stale)) = http_cache_get(context, url).await? { + info!(context, "Returning {url:?} from cache."); + if is_stale { + let context = context.clone(); + let url = url.to_string(); + tokio::spawn(async move { + // Fetch URL in background to update the cache. + info!(context, "Fetching stale {url:?} in background."); + if let Err(err) = fetch_url(&context, &url).await { + warn!(context, "Failed to revalidate {url:?}: {err:#}."); + } + }); + } + + // Return stale result. + return Ok(response); + } + + info!(context, "Not found {url:?} in cache, fetching."); + let response = fetch_url(context, url).await?; + Ok(response) +} + /// Sends an empty POST request to the URL. /// /// Returns response text and whether request was successful or not. @@ -401,43 +443,47 @@ mod tests { assert_eq!(http_cache_get(t, xdc_pixel_url).await?, None); assert_eq!( http_cache_get(t, "https://webxdc.org/").await?, - Some(html_response.clone()) + Some((html_response.clone(), false)) ); http_cache_put(t, xdc_editor_url, &xdc_response).await?; http_cache_put(t, xdc_pixel_url, &xdc_response).await?; assert_eq!( http_cache_get(t, xdc_editor_url).await?, - Some(xdc_response.clone()) + Some((xdc_response.clone(), false)) ); assert_eq!( http_cache_get(t, xdc_pixel_url).await?, - Some(xdc_response.clone()) + Some((xdc_response.clone(), false)) ); assert_eq!( http_cache_get(t, "https://webxdc.org/").await?, - Some(html_response.clone()) + Some((html_response.clone(), false)) ); - // HTML expires after 1 hour, but .xdc does not. + // HTML is stale after 1 hour, but .xdc is not. SystemTime::shift(Duration::from_secs(3600 + 100)); - assert_eq!(http_cache_get(t, "https://webxdc.org/").await?, None); + assert_eq!( + http_cache_get(t, "https://webxdc.org/").await?, + Some((html_response.clone(), true)) + ); assert_eq!( http_cache_get(t, xdc_editor_url).await?, - Some(xdc_response.clone()) + Some((xdc_response.clone(), false)) ); - // Expired cache entry can be renewed + // Stale cache entry can be renewed // even before housekeeping removes old one. http_cache_put(t, "https://webxdc.org/", &html_response).await?; assert_eq!( http_cache_get(t, "https://webxdc.org/").await?, - Some(html_response.clone()) + Some((html_response.clone(), false)) ); // 35 days later pixel .xdc expires because we did not request it for 35 days and 1 hour. // But editor is still there because we did not request it for just 35 days. + // We have not renewed the editor however, so it becomes stale. SystemTime::shift(Duration::from_secs(3600 * 24 * 35 - 100)); // Run housekeeping to test that it does not delete the blob too early. @@ -445,7 +491,7 @@ mod tests { assert_eq!( http_cache_get(t, xdc_editor_url).await?, - Some(xdc_response.clone()) + Some((xdc_response.clone(), true)) ); assert_eq!(http_cache_get(t, xdc_pixel_url).await?, None); diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 0827f924e..dcaafbbff 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1103,6 +1103,24 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid); .await?; } + inc_and_check(&mut migration_version, 126)?; + if dbversion < migration_version { + // Recreate http_cache table with new `stale` column. + sql.execute_migration( + "DROP TABLE http_cache; + CREATE TABLE http_cache ( + url TEXT PRIMARY KEY, + expires INTEGER NOT NULL, -- When the cache entry is considered expired, timestamp in seconds. + stale INTEGER NOT NULL, -- When the cache entry is considered stale, timestamp in seconds. + blobname TEXT NOT NULL, + mimetype TEXT NOT NULL DEFAULT '', -- MIME type extracted from Content-Type header. + encoding TEXT NOT NULL DEFAULT '' -- Encoding from Content-Type header. + ) STRICT", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? From 7d1e3c4812dff2968a297debc3bc1ec13b89dd9c Mon Sep 17 00:00:00 2001 From: link2xt <link2xt@testrun.org> Date: Thu, 12 Dec 2024 09:25:07 +0000 Subject: [PATCH 17/20] fix: ignore garbage at the end of the keys --- src/key.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/key.rs b/src/key.rs index b03d06610..dc3aca799 100644 --- a/src/key.rs +++ b/src/key.rs @@ -30,7 +30,39 @@ use crate::tools::{self, time_elapsed}; pub(crate) trait DcKey: Serialize + Deserializable + PublicKeyTrait + Clone { /// Create a key from some bytes. fn from_slice(bytes: &[u8]) -> Result<Self> { - Ok(<Self as Deserializable>::from_bytes(Cursor::new(bytes))?) + let res = <Self as Deserializable>::from_bytes(Cursor::new(bytes)); + if let Ok(res) = res { + return Ok(res); + } + + // Workaround for keys imported using + // Delta Chat core < 1.0.0. + // Old Delta Chat core had a bug + // that resulted in treating CRC24 checksum + // as part of the key when reading ASCII Armor. + // Some users that started using Delta Chat in 2019 + // have such corrupted keys with garbage bytes at the end. + // + // Garbage is at least 3 bytes long + // and may be longer due to padding + // at the end of the real key data + // and importing the key multiple times. + // + // If removing 10 bytes is not enough, + // the key is likely actually corrupted. + for garbage_bytes in 3..std::cmp::min(bytes.len(), 10) { + let res = <Self as Deserializable>::from_bytes(Cursor::new( + bytes + .get(..bytes.len().saturating_sub(garbage_bytes)) + .unwrap_or_default(), + )); + if let Ok(res) = res { + return Ok(res); + } + } + + // Removing garbage bytes did not help, return the error. + Ok(res?) } /// Create a key from a base64 string. @@ -565,6 +597,36 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD } } + /// Tests workaround for Delta Chat core < 1.0.0 + /// which parsed CRC24 at the end of ASCII Armor + /// as the part of the key. + /// Depending on the alignment and the number of + /// `=` characters at the end of the key, + /// this resulted in various number of garbage + /// octets at the end of the key, starting from 3 octets, + /// but possibly 4 or 5 and maybe more octets + /// if the key is imported or transferred + /// using Autocrypt Setup Message multiple times. + #[test] + fn test_ignore_trailing_garbage() { + // Test several variants of garbage. + for garbage in [ + b"\x02\xfc\xaa\x38\x4b\x5c".as_slice(), + b"\x02\xfc\xaa".as_slice(), + b"\x01\x02\x03\x04\x05".as_slice(), + ] { + let private_key = KEYPAIR.secret.clone(); + + let mut binary = DcKey::to_bytes(&private_key); + binary.extend(garbage); + + let private_key2 = + SignedSecretKey::from_slice(&binary).expect("Failed to ignore garbage"); + + assert_eq!(private_key.dc_fingerprint(), private_key2.dc_fingerprint()); + } + } + #[test] fn test_base64_roundtrip() { let key = KEYPAIR.public.clone(); From a305409627041c2a0e0a9b1908e1e3d491a18c06 Mon Sep 17 00:00:00 2001 From: link2xt <link2xt@testrun.org> Date: Thu, 12 Dec 2024 15:39:31 +0000 Subject: [PATCH 18/20] chore(release): prepare for 1.152.0 --- CHANGELOG.md | 22 +++++++++++++++++++ Cargo.lock | 10 ++++----- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 2 +- deltachat-repl/Cargo.toml | 2 +- deltachat-rpc-client/pyproject.toml | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- deltachat-rpc-server/npm-package/package.json | 2 +- package.json | 2 +- python/pyproject.toml | 2 +- release-date.in | 2 +- 13 files changed, 38 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bee527dc6..afef8795a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [1.152.0] - 2024-12-12 + +### API-Changes + +- [**breaking**] Remove `dc_prepare_msg` and `dc_msg_is_increation`. + +### Build system + +- Increase MSRV to 1.81.0. + +### Features / Changes + +- Cache HTTP GET requests. +- Prefix server-url in info. +- Set `mime_modified` for the last message part, not the first ([#4462](https://github.com/deltachat/deltachat-core-rust/pull/4462)). + +### Fixes + +- Render "message" parts in multipart messages' HTML ([#4462](https://github.com/deltachat/deltachat-core-rust/pull/4462)). +- Ignore garbage at the end of the keys. + ## [1.151.6] - 2024-12-11 ### Features / Changes @@ -5487,3 +5508,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed [1.151.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.3..v1.151.4 [1.151.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.4..v1.151.5 [1.151.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.5..v1.151.6 +[1.152.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.6..v1.152.0 diff --git a/Cargo.lock b/Cargo.lock index 56d44a529..649c4afad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1306,7 +1306,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.151.6" +version = "1.152.0" dependencies = [ "anyhow", "async-broadcast", @@ -1407,7 +1407,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.151.6" +version = "1.152.0" dependencies = [ "anyhow", "async-channel 2.3.1", @@ -1432,7 +1432,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.151.6" +version = "1.152.0" dependencies = [ "anyhow", "deltachat", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.151.6" +version = "1.152.0" dependencies = [ "anyhow", "deltachat", @@ -1477,7 +1477,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.151.6" +version = "1.152.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index 8a42dba29..1a5f56196 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.151.6" +version = "1.152.0" edition = "2021" license = "MPL-2.0" rust-version = "1.81" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 4146922fa..b290a26b1 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.151.6" +version = "1.152.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 4dd09d8e0..83e540f73 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.151.6" +version = "1.152.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 74bd0678d..f06120e9d 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -58,5 +58,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.151.6" + "version": "1.152.0" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index b60055e72..2f5827bbf 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.151.6" +version = "1.152.0" license = "MPL-2.0" edition = "2021" repository = "https://github.com/deltachat/deltachat-core-rust" diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index e607ee6a7..ccad19a52 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deltachat-rpc-client" -version = "1.151.6" +version = "1.152.0" description = "Python client for Delta Chat core JSON-RPC interface" classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 95c8b816a..73fdf2a4f 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.151.6" +version = "1.152.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/deltachat-rpc-server/npm-package/package.json b/deltachat-rpc-server/npm-package/package.json index ae21c50e1..0db6e7287 100644 --- a/deltachat-rpc-server/npm-package/package.json +++ b/deltachat-rpc-server/npm-package/package.json @@ -15,5 +15,5 @@ }, "type": "module", "types": "index.d.ts", - "version": "1.151.6" + "version": "1.152.0" } diff --git a/package.json b/package.json index ba9823238..8b33dc48a 100644 --- a/package.json +++ b/package.json @@ -55,5 +55,5 @@ "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.151.6" + "version": "1.152.0" } diff --git a/python/pyproject.toml b/python/pyproject.toml index a5cb5f23d..220cfcbb7 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "deltachat" -version = "1.151.6" +version = "1.152.0" description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat" readme = "README.rst" requires-python = ">=3.7" diff --git a/release-date.in b/release-date.in index b57a0e640..a2b343073 100644 --- a/release-date.in +++ b/release-date.in @@ -1 +1 @@ -2024-12-11 \ No newline at end of file +2024-12-12 \ No newline at end of file From b74ff278ce691160ac158768327b733f6675162d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2024 21:10:54 +0000 Subject: [PATCH 19/20] chore(cargo): bump rustyline from 14.0.0 to 15.0.0 Bumps [rustyline](https://github.com/kkawakam/rustyline) from 14.0.0 to 15.0.0. - [Release notes](https://github.com/kkawakam/rustyline/releases) - [Changelog](https://github.com/kkawakam/rustyline/blob/master/History.md) - [Commits](https://github.com/kkawakam/rustyline/compare/v14.0.0...v15.0.0) --- updated-dependencies: - dependency-name: rustyline dependency-type: direct:production update-type: version-update:semver-major ... --- Cargo.lock | 26 ++++++++++++++++---------- deltachat-repl/Cargo.toml | 2 +- deltachat-repl/src/main.rs | 6 +++--- deny.toml | 1 + 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 649c4afad..4995655e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -783,9 +783,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" @@ -3880,9 +3880,9 @@ dependencies = [ [[package]] name = "nix" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -5472,9 +5472,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "rustyline" -version = "14.0.0" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" +checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ "bitflags 2.6.0", "cfg-if", @@ -5484,12 +5484,12 @@ dependencies = [ "libc", "log", "memchr", - "nix 0.28.0", + "nix 0.29.0", "radix_trie", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.0", "utf8parse", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6273,7 +6273,7 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "smawk", "unicode-linebreak", - "unicode-width", + "unicode-width 0.1.11", ] [[package]] @@ -6779,6 +6779,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode-xid" version = "0.2.4" diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index 2f5827bbf..fd31cd9c9 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -13,7 +13,7 @@ log = { workspace = true } nu-ansi-term = { workspace = true } qr2term = "0.3.3" rusqlite = { workspace = true } -rustyline = "14" +rustyline = "15" tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] } tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/deltachat-repl/src/main.rs b/deltachat-repl/src/main.rs index 2b16cc68f..52a7a8754 100644 --- a/deltachat-repl/src/main.rs +++ b/deltachat-repl/src/main.rs @@ -22,7 +22,7 @@ use log::{error, info, warn}; use nu_ansi_term::Color; use rustyline::completion::{Completer, FilenameCompleter, Pair}; use rustyline::error::ReadlineError; -use rustyline::highlight::{Highlighter, MatchingBracketHighlighter}; +use rustyline::highlight::{CmdKind as HighlightCmdKind, Highlighter, MatchingBracketHighlighter}; use rustyline::hint::{Hinter, HistoryHinter}; use rustyline::validate::Validator; use rustyline::{ @@ -298,8 +298,8 @@ impl Highlighter for DcHelper { self.highlighter.highlight(line, pos) } - fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool { - self.highlighter.highlight_char(line, pos, forced) + fn highlight_char(&self, line: &str, pos: usize, kind: HighlightCmdKind) -> bool { + self.highlighter.highlight_char(line, pos, kind) } } diff --git a/deny.toml b/deny.toml index 533ded2bd..4fa2343aa 100644 --- a/deny.toml +++ b/deny.toml @@ -52,6 +52,7 @@ skip = [ { name = "sync_wrapper", version = "0.1.2" }, { name = "syn", version = "1.0.109" }, { name = "time", version = "<0.3" }, + { name = "unicode-width", version = "0.1.11" }, { name = "wasi", version = "<0.11" }, { name = "windows_aarch64_gnullvm", version = "<0.52" }, { name = "windows_aarch64_msvc", version = "<0.52" }, From 1e1e5793dd4034c03db259abe45954eadb4c6c09 Mon Sep 17 00:00:00 2001 From: link2xt <link2xt@testrun.org> Date: Sun, 15 Dec 2024 17:00:25 +0000 Subject: [PATCH 20/20] chore: remove contrib/ directory It only contained proxy that some users ran in Termux to look at IMAP traffic. The same can be achieved with `socat`, e.g.: socat -v TCP-LISTEN:9999,bind=127.0.0.1 OPENSSL:nine.testrun.org:993 --- contrib/proxy.py | 80 ------------------------------------------------ 1 file changed, 80 deletions(-) delete mode 100644 contrib/proxy.py diff --git a/contrib/proxy.py b/contrib/proxy.py deleted file mode 100644 index 13e1b6978..000000000 --- a/contrib/proxy.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# Examples: -# -# Original server that doesn't use SSL: -# ./proxy.py 8080 imap.nauta.cu 143 -# ./proxy.py 8081 smtp.nauta.cu 25 -# -# Original server that uses SSL: -# ./proxy.py 8080 testrun.org 993 --ssl -# ./proxy.py 8081 testrun.org 465 --ssl - -from datetime import datetime -import argparse -import selectors -import ssl -import socket -import socketserver - - -class Proxy(socketserver.ThreadingTCPServer): - allow_reuse_address = True - - def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl): - self.real_host = real_host - self.real_port = real_port - self.use_ssl = use_ssl - super().__init__((proxy_host, proxy_port), RequestHandler) - - -class RequestHandler(socketserver.BaseRequestHandler): - - def handle(self): - print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address)) - - total = 0 - real_server = (self.server.real_host, self.server.real_port) - with socket.create_connection(real_server) as sock: - if self.server.use_ssl: - context = ssl.create_default_context() - sock = context.wrap_socket( - sock, server_hostname=real_server[0]) - - forward = {self.request: sock, sock: self.request} - - sel = selectors.DefaultSelector() - sel.register(self.request, selectors.EVENT_READ, - self.client_address) - sel.register(sock, selectors.EVENT_READ, real_server) - - active = True - while active: - events = sel.select() - for key, mask in events: - print('\n{} - {} wrote:'.format(datetime.now(), key.data)) - data = key.fileobj.recv(1024) - received = len(data) - total += received - print(data) - print('{} Bytes\nTotal: {} Bytes'.format(received, total)) - if data: - forward[key.fileobj].sendall(data) - else: - print('\nCLOSING CONNECTION.\n\n') - forward[key.fileobj].close() - key.fileobj.close() - active = False - - -if __name__ == '__main__': - p = argparse.ArgumentParser(description='Simple Python Proxy') - p.add_argument( - "proxy_port", help="the port where the proxy will listen", type=int) - p.add_argument('host', help="the real host") - p.add_argument('port', help="the port of the real host", type=int) - p.add_argument("--ssl", help="use ssl to connect to the real host", - action="store_true") - args = p.parse_args() - - with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy: - proxy.serve_forever()