From 6044e5961bf98b4d6a1357359552096c34fa600f Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 1 Feb 2023 16:45:09 +0100 Subject: [PATCH 001/103] Send and receive backup over network using QR code This adds functionality to send and receive a backup over the network using a QR code. The sender or provider prepares the backup, sets up a server that waits for clients. It provides a ticket in the form of a QR code which contains connection and authentication information. The receiver uses the QR code to connect to the provider and fetches backup, restoring it locally. --- Cargo.lock | 974 +++++++++++++++++++++++++- Cargo.toml | 38 +- deltachat-ffi/deltachat.h | 9 +- deltachat-ffi/src/lot.rs | 5 + deltachat-jsonrpc/src/api/types/qr.rs | 6 + src/blob.rs | 4 +- src/context.rs | 7 + src/imex.rs | 296 ++++++-- src/qr.rs | 3 + 9 files changed, 1265 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 14cf33f92..2f3fec720 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,6 +89,21 @@ name = "anyhow" version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +dependencies = [ + "backtrace", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "ascii_utils" @@ -96,6 +111,45 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" +[[package]] +name = "asn1-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.1", + "num-traits", + "rusticata-macros", + "thiserror", + "time 0.3.17", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-channel" version = "1.8.0" @@ -353,6 +407,23 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bao" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3663af8a469bf3a2d818b8c1a7b62e4dc55c4547b4c14f5dab35fc9e707bef83" +dependencies = [ + "arrayref", + "arrayvec", + "blake3", +] + +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.11.0" @@ -401,6 +472,20 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "blake3" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "digest 0.10.5", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -544,7 +629,7 @@ dependencies = [ "js-sys", "num-integer", "num-traits", - "time", + "time 0.1.44", "wasm-bindgen", "winapi", ] @@ -593,11 +678,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "bitflags", - "clap_lex", + "clap_lex 0.2.4", "indexmap", "textwrap", ] +[[package]] +name = "clap" +version = "4.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" +dependencies = [ + "bitflags", + "clap_derive", + "clap_lex 0.3.1", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +dependencies = [ + "heck", + "proc-macro-error 1.0.4", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "0.2.4" @@ -607,6 +720,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clipboard-win" version = "4.4.2" @@ -618,6 +740,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "cobs" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" + [[package]] name = "color_quant" version = "1.1.0" @@ -633,12 +761,51 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.42.0", +] + [[package]] name = "const-oid" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "722e23542a15cea1f65d4a1419c4cfd7a26706c70871a13a04238ca3f40f1661" +[[package]] +name = "const_format" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7309d9b4d3d2c0641e018d449232f2e28f1b22933c137f157d3dbc14228b8c0e" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "constant_time_eq" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" + [[package]] name = "convert_case" version = "0.5.0" @@ -695,7 +862,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap", + "clap 3.2.23", "criterion-plot", "futures", "itertools", @@ -753,7 +920,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.6.5", "once_cell", "scopeguard", ] @@ -778,6 +945,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.3", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -788,6 +967,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "cuckoofilter" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b810a8449931679f64cd7eef1bbd0fa315801b6d5d9cdc1ace2804d6529eee18" +dependencies = [ + "byteorder", + "fnv", + "rand 0.7.3", +] + [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -931,6 +1121,7 @@ dependencies = [ "rust-hsluv", "rustyline", "sanitize-filename", + "sendme", "serde", "serde_json", "sha-1", @@ -1021,10 +1212,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f" dependencies = [ "const-oid", + "der_derive", "pem-rfc7468", "zeroize", ] +[[package]] +name = "der-parser" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.1", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ef71ddb5b3a1f53dee24817c8f70dfa1cb29e804c18d88c228d4bc9c86ee3b9" +dependencies = [ + "proc-macro-error 1.0.4", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_builder" version = "0.11.2" @@ -1083,6 +1301,7 @@ dependencies = [ "block-buffer 0.10.2", "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -1126,12 +1345,36 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "ed25519" version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9c280362032ea4203659fc489832d0204ef09f247a0506f170dafcac08c369" dependencies = [ + "serde", "signature", ] @@ -1145,6 +1388,7 @@ dependencies = [ "ed25519", "rand 0.7.3", "serde", + "serde_bytes", "sha2 0.9.9", "zeroize", ] @@ -1155,6 +1399,25 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint", + "der", + "digest 0.10.5", + "ff", + "generic-array", + "group", + "rand_core 0.6.3", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "email" version = "0.0.21" @@ -1166,10 +1429,16 @@ dependencies = [ "encoding", "lazy_static", "rand 0.7.3", - "time", + "time 0.1.44", "version_check 0.9.4", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoded-words" version = "0.2.0" @@ -1407,6 +1676,16 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.3", + "subtle", +] + [[package]] name = "filetime" version = "0.2.17" @@ -1567,6 +1846,37 @@ dependencies = [ "slab", ] +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "futures-core", + "genawaiter-macro", + "genawaiter-proc-macro", + "proc-macro-hack", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + +[[package]] +name = "genawaiter-proc-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" +dependencies = [ + "proc-macro-error 0.4.12", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -1615,6 +1925,17 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.3", + "subtle", +] + [[package]] name = "h2" version = "0.3.13" @@ -1640,6 +1961,12 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" +[[package]] +name = "hash_hasher" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" + [[package]] name = "hashbrown" version = "0.11.2" @@ -1697,6 +2024,21 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.5", +] + [[package]] name = "hostname" version = "0.3.1" @@ -1899,6 +2241,19 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indicatif" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "tokio", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -1917,6 +2272,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "intrusive-collections" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f4f90afb01281fdeffb0f8e082d230cbe4f888f837cc90759696b858db1a700" +dependencies = [ + "memoffset 0.8.0", +] + [[package]] name = "io-lifetimes" version = "0.7.2" @@ -2037,7 +2401,7 @@ dependencies = [ "lettre", "mime", "regex", - "time", + "time 0.1.44", "uuid 0.8.2", ] @@ -2128,6 +2492,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.9" @@ -2170,6 +2543,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -2285,6 +2667,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.1" @@ -2366,6 +2759,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.30.0" @@ -2375,6 +2774,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.17.0" @@ -2480,12 +2888,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "734aa7a4a6390b162112523cac2923a18e4f23b917880a68c826bf6e8bf48f06" dependencies = [ "Inflector", - "proc-macro-error", + "proc-macro-error 1.0.4", "proc-macro2", "quote", "syn", ] +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.6", +] + +[[package]] +name = "p384" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc8c5bf642dde52bb9e87c0ecd8ca5a76faac2eeed98dedb7c717997e1080aa" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2 0.10.6", +] + [[package]] name = "parking" version = "2.0.0" @@ -2515,6 +2945,15 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + [[package]] name = "pem-rfc7468" version = "0.6.0" @@ -2675,6 +3114,35 @@ dependencies = [ "miniz_oxide 0.6.2", ] +[[package]] +name = "portable-atomic" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" + +[[package]] +name = "postcard" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c2b180dc0bade59f03fd005cb967d3f1e5f69b13922dad0cd6e047cb8af2363" +dependencies = [ + "cobs", + "const_format", + "postcard-derive", + "serde", +] + +[[package]] +name = "postcard-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a87b20c0f64e4a835a7f006bff0bbcefed3bf67828fd4608579e6b51e1ebd0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -2691,19 +3159,45 @@ dependencies = [ "log", ] +[[package]] +name = "proc-macro-error" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" +dependencies = [ + "proc-macro-error-attr 0.4.12", + "proc-macro2", + "quote", + "syn", + "version_check 0.9.4", +] + [[package]] name = "proc-macro-error" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ - "proc-macro-error-attr", + "proc-macro-error-attr 1.0.4", "proc-macro2", "quote", "syn", "version_check 0.9.4", ] +[[package]] +name = "proc-macro-error-attr" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "syn-mid", + "version_check 0.9.4", +] + [[package]] name = "proc-macro-error-attr" version = "1.0.4" @@ -2715,6 +3209,12 @@ dependencies = [ "version_check 0.9.4", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + [[package]] name = "proc-macro2" version = "1.0.46" @@ -2922,6 +3422,18 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring", + "time 0.3.17", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2953,6 +3465,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.27" @@ -3015,6 +3536,32 @@ dependencies = [ "quick-error 1.2.3", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint", + "hmac", + "zeroize", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "ripemd" version = "0.1.3" @@ -3072,6 +3619,15 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.1", +] + [[package]] name = "rustix" version = "0.35.7" @@ -3100,6 +3656,27 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "rustls" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64 0.21.0", +] + [[package]] name = "rustversion" version = "1.0.9" @@ -3135,6 +3712,125 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "s2n-codec" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c76a52949171a784415293428d2c9315c249ba2091d49b27279601a55375bbb2" +dependencies = [ + "byteorder", + "bytes", + "zerocopy", + "zerocopy-derive", +] + +[[package]] +name = "s2n-quic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501b93c392899600caafd6f7b19b43a181b6afcb05c2ba0a61e65589469fd863" +dependencies = [ + "bytes", + "cfg-if", + "cuckoofilter", + "futures", + "hash_hasher", + "rand 0.8.5", + "rand_chacha 0.3.1", + "ring", + "s2n-codec", + "s2n-quic-core", + "s2n-quic-platform", + "s2n-quic-rustls", + "s2n-quic-transport", + "tokio", + "zerocopy", + "zerocopy-derive", + "zeroize", +] + +[[package]] +name = "s2n-quic-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7ea949a1e82375bc7d05ac76372fd95992e8db42485c85228b3ce421d1a9af" +dependencies = [ + "byteorder", + "bytes", + "hex-literal", + "num-rational", + "num-traits", + "once_cell", + "s2n-codec", + "subtle", + "zerocopy", + "zerocopy-derive", +] + +[[package]] +name = "s2n-quic-crypto" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2a457eebfdd37f792fdfc9283893d90767cbb6fff623cbc7ec12531d29e1a7" +dependencies = [ + "cfg-if", + "lazy_static", + "ring", + "s2n-codec", + "s2n-quic-core", + "zeroize", +] + +[[package]] +name = "s2n-quic-platform" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8926e4b76c64a06ddb7e413d18d045c619f0daef8b93bcb04dd8f322ed293755" +dependencies = [ + "cfg-if", + "errno", + "futures", + "lazy_static", + "libc", + "pin-project", + "s2n-quic-core", + "socket2", + "tokio", + "zeroize", +] + +[[package]] +name = "s2n-quic-rustls" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec8decad668229de67f0b90e98ffbba8f844dc294084af3d7e86bfee4f6f040" +dependencies = [ + "bytes", + "rustls", + "rustls-pemfile", + "s2n-codec", + "s2n-quic-core", + "s2n-quic-crypto", +] + +[[package]] +name = "s2n-quic-transport" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0506c95a703cf12dc21352275f699f9508485c89b257cf99c2b3c2f1cc25c9" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "hashbrown 0.12.3", + "intrusive-collections", + "once_cell", + "s2n-codec", + "s2n-quic-core", + "siphasher", + "smallvec", +] + [[package]] name = "safemem" version = "0.3.3" @@ -3185,6 +3881,30 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -3208,6 +3928,42 @@ dependencies = [ "libc", ] +[[package]] +name = "sendme" +version = "0.1.0" +dependencies = [ + "anyhow", + "bao", + "blake3", + "bytes", + "clap 4.1.4", + "console", + "der", + "ed25519-dalek", + "futures", + "genawaiter", + "hex", + "indicatif", + "is-terminal", + "postcard", + "rand 0.7.3", + "rcgen", + "ring", + "rustls", + "s2n-quic", + "serde", + "ssh-key", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "webpki", + "x509-parser", + "zeroize", +] + [[package]] name = "serde" version = "1.0.152" @@ -3217,6 +3973,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "718dc5fff5b36f99093fc49b280cfc96ce6fc824317783bff5a1fed0c7a64819" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.152" @@ -3325,6 +4090,15 @@ dependencies = [ "keccak", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -3344,6 +4118,12 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.7" @@ -3391,6 +4171,35 @@ dependencies = [ "der", ] +[[package]] +name = "ssh-encoding" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19cfdc32e0199062113edf41f344fbf784b8205a94600233c84eb838f45191e1" +dependencies = [ + "base64ct", + "pem-rfc7468", + "sha2 0.10.6", +] + +[[package]] +name = "ssh-key" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "288d8f5562af5a3be4bda308dd374b2c807b940ac370b5efa1c99311da91d9a1" +dependencies = [ + "ed25519-dalek", + "p256", + "p384", + "rand_core 0.6.3", + "rsa", + "sec1", + "sha2 0.10.6", + "signature", + "ssh-encoding", + "zeroize", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3457,6 +4266,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn-mid" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sync_wrapper" version = "0.1.1" @@ -3535,6 +4355,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "time" version = "0.1.44" @@ -3546,6 +4375,33 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -3807,6 +4663,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +dependencies = [ + "ansi_term", + "matchers", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -3931,7 +4817,7 @@ checksum = "e5c1bfe689e4067733530495b04959b00f05cd95f038bed59af4fc70b3e26240" dependencies = [ "darling 0.13.4", "ident_case", - "proc-macro-error", + "proc-macro-error 1.0.4", "proc-macro2", "quote", "syn", @@ -3986,6 +4872,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.3.1" @@ -4028,6 +4920,12 @@ dependencies = [ "serde", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -4167,6 +5065,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "weezl" version = "0.1.7" @@ -4339,6 +5247,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "x509-parser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" +dependencies = [ + "asn1-rs", + "base64 0.13.1", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.1", + "oid-registry", + "rusticata-macros", + "thiserror", + "time 0.3.17", +] + [[package]] name = "xattr" version = "0.2.3" @@ -4348,6 +5274,15 @@ dependencies = [ "libc", ] +[[package]] +name = "yasna" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aed2e7a52e3744ab4d0c05c20aa065258e84c49fd4226f5191b2ed29712710b4" +dependencies = [ + "time 0.3.17", +] + [[package]] name = "yerpc" version = "0.4.0" @@ -4383,6 +5318,27 @@ dependencies = [ "syn", ] +[[package]] +name = "zerocopy" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae0f717764196a220d8c58500e3a3595e2c9054f95d66267f9fd5f6e74ad0fec" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0af017aca1fa6181f5dd7a802456fe6f7666ecdcc18d0910431f0fc89d474e51" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.5.7" diff --git a/Cargo.toml b/Cargo.toml index 6a8fd9872..06909ee96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,12 +30,11 @@ ratelimit = { path = "./deltachat-ratelimit" } ansi_term = { version = "0.12.1", optional = true } anyhow = "1" +async-channel = "1.8.0" async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] } async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] } async-smtp = { version = "0.6", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] } -trust-dns-resolver = "0.22" -tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } -tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar +async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] } backtrace = "0.3" base64 = "0.21" bitflags = "1.3" @@ -44,8 +43,11 @@ dirs = { version = "4", optional=true } email = { git = "https://github.com/deltachat/rust-email", branch = "master" } encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" } escaper = "0.1" +fast-socks5 = "0.8" futures = "0.3" +futures-lite = "1.12.0" hex = "0.4.0" +humansize = "2" image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] } kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } @@ -53,44 +55,43 @@ libc = "0.2" log = {version = "0.4.16", optional = true } mailparse = "0.14" native-tls = "0.2" -num_cpus = "1.15" num-derive = "0.3" num-traits = "0.2" +num_cpus = "1.15" once_cell = "1.17.0" percent-encoding = "2.2" pgp = { version = "0.9", default-features = false } pretty_env_logger = { version = "0.4", optional = true } +qrcodegen = "1.7.0" quick-xml = "0.27" r2d2 = "0.8" r2d2_sqlite = "0.20" rand = "0.8" regex = "1.7" +reqwest = { version = "0.11.14", features = ["json"] } rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" rustyline = { version = "10", optional = true } sanitize-filename = "0.4" -serde_json = "1.0" +sendme = { git = "https://github.com/n0-computer/sendme", branch = "main" } serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" sha-1 = "0.10" sha2 = "0.10" smallvec = "1" strum = "0.24" strum_macros = "0.24" -thiserror = "1" -toml = "0.7" -url = "2" -uuid = { version = "1", features = ["serde", "v4"] } -fast-socks5 = "0.8" -humansize = "2" -qrcodegen = "1.7.0" tagger = "4.3.4" textwrap = "0.16.0" -async-channel = "1.8.0" -futures-lite = "1.12.0" -tokio-stream = { version = "0.1.11", features = ["fs"] } +thiserror = "1" +tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } tokio-io-timeout = "1.2.0" -reqwest = { version = "0.11.14", features = ["json"] } -async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] } +tokio-stream = { version = "0.1.11", features = ["fs"] } +tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar +toml = "0.7" +trust-dns-resolver = "0.22" +url = "2" +uuid = { version = "1", features = ["serde", "v4"] } [dev-dependencies] ansi_term = "0.12.0" @@ -161,3 +162,6 @@ vendored = [ "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored" ] + +[patch.'https://github.com/n0-computer/sendme'] +sendme = { path = "/home/flub/n0/sendme" } \ No newline at end of file diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 507fbe76c..d3b1219f1 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2320,7 +2320,7 @@ void dc_stop_ongoing_process (dc_context_t* context); * ask whether to verify the contact; * if so, start the protocol with dc_join_securejoin(). * - * - DC_QR_ASK_VERIFYGROUP withdc_lot_t::text1=Group name: + * - DC_QR_ASK_VERIFYGROUP with dc_lot_t::text1=Group name: * ask whether to join the group; * if so, start the protocol with dc_join_securejoin(). * @@ -2340,6 +2340,10 @@ void dc_stop_ongoing_process (dc_context_t* context); * ask the user if they want to create an account on the given domain, * if so, call dc_set_config_from_qr() and then dc_configure(). * + * - DC_QR_BACKUP: + * ask the user if they want to set up a new device. + * If so, pass the qr-code to dc_receive_backup(). + * * - DC_QR_WEBRTC_INSTANCE with dc_lot_t::text1=domain: * ask the user if they want to use the given service for video chats; * if so, call dc_set_config_from_qr(). @@ -2630,6 +2634,9 @@ char* dc_get_last_error (dc_context_t* context); void dc_str_unref (char* str); +// TODO: add New stuff + + /** * @class dc_accounts_t * diff --git a/deltachat-ffi/src/lot.rs b/deltachat-ffi/src/lot.rs index 1e6a7831a..270cd34ea 100644 --- a/deltachat-ffi/src/lot.rs +++ b/deltachat-ffi/src/lot.rs @@ -52,6 +52,7 @@ impl Lot { Qr::FprMismatch { .. } => None, Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint), Qr::Account { domain } => Some(domain), + Qr::Backup { .. } => None, Qr::WebrtcInstance { domain, .. } => Some(domain), Qr::Addr { draft, .. } => draft.as_deref(), Qr::Url { url } => Some(url), @@ -103,6 +104,7 @@ impl Lot { Qr::FprMismatch { .. } => LotState::QrFprMismatch, Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr, Qr::Account { .. } => LotState::QrAccount, + Qr::Backup { .. } => LotState::QrBackup, Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance, Qr::Addr { .. } => LotState::QrAddr, Qr::Url { .. } => LotState::QrUrl, @@ -127,6 +129,7 @@ impl Lot { Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(), Qr::FprWithoutAddr { .. } => Default::default(), Qr::Account { .. } => Default::default(), + Qr::Backup { .. } => Default::default(), Qr::WebrtcInstance { .. } => Default::default(), Qr::Addr { contact_id, .. } => contact_id.to_u32(), Qr::Url { .. } => Default::default(), @@ -175,6 +178,8 @@ pub enum LotState { /// text1=domain QrAccount = 250, + QrBackup = 251, + /// text1=domain, text2=instance pattern QrWebrtcInstance = 260, diff --git a/deltachat-jsonrpc/src/api/types/qr.rs b/deltachat-jsonrpc/src/api/types/qr.rs index 0a38d43ce..143792477 100644 --- a/deltachat-jsonrpc/src/api/types/qr.rs +++ b/deltachat-jsonrpc/src/api/types/qr.rs @@ -32,6 +32,9 @@ pub enum QrObject { Account { domain: String, }, + Backup { + ticket: Vec, + }, WebrtcInstance { domain: String, instance_pattern: String, @@ -126,6 +129,9 @@ impl From for QrObject { } Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint }, Qr::Account { domain } => QrObject::Account { domain }, + Qr::Backup { ticket } => QrObject::Backup { + ticket: ticket.to_bytes(), + }, Qr::WebrtcInstance { domain, instance_pattern, diff --git a/src/blob.rs b/src/blob.rs index ecca8129c..ef2869bed 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -160,9 +160,9 @@ impl<'a> BlobObject<'a> { pub fn from_path(context: &'a Context, path: &Path) -> Result> { let rel_path = path .strip_prefix(context.get_blobdir()) - .context("wrong blobdir")?; + .with_context(|| format!("wrong blobdir: {}", path.display()))?; if !BlobObject::is_acceptible_blob_name(rel_path) { - return Err(format_err!("wrong name")); + return Err(format_err!("bad blob name: {}", rel_path.display())); } let name = rel_path.to_str().context("wrong name")?; BlobObject::from_name(context, name.to_string()) diff --git a/src/context.rs b/src/context.rs index 54aedceb2..2b3bb17dd 100644 --- a/src/context.rs +++ b/src/context.rs @@ -513,6 +513,13 @@ impl Context { // Ongoing process allocation/free/check + /// Tries to acquire the global UI "ongoing" mutex. + /// + /// This is for modal operations during which no other user actions are allowed. Only + /// one such operation is allowed at any given time. + /// + /// The return value is a cancel token, which will release the ongoing mutex when + /// dropped. pub(crate) async fn alloc_ongoing(&self) -> Result> { let mut s = self.running_state.write().await; ensure!( diff --git a/src/imex.rs b/src/imex.rs index bcedd8b99..f19b02c4c 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -4,14 +4,18 @@ use std::any::Any; use std::ffi::OsStr; +use std::iter::FusedIterator; use std::path::{Path, PathBuf}; use ::pgp::types::KeyTrait; use anyhow::{bail, ensure, format_err, Context as _, Result}; +use async_channel::Receiver; use futures::StreamExt; use futures_lite::FutureExt; use rand::{thread_rng, Rng}; use tokio::fs::{self, File}; +use tokio::task::JoinError; +use tokio_stream::wrappers::ReadDirStream; use tokio_tar::Archive; use crate::blob::BlobObject; @@ -19,6 +23,7 @@ use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId}; use crate::config::Config; use crate::contact::ContactId; use crate::context::Context; +use crate::e2ee; use crate::events::EventType; use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey}; use crate::log::LogExt; @@ -26,13 +31,13 @@ use crate::message::{Message, MsgId, Viewtype}; use crate::mimeparser::SystemMessage; use crate::param::Param; use crate::pgp; +use crate::qr::Qr; use crate::sql; use crate::stock_str; use crate::tools::{ create_folder, delete_file, get_filesuffix_lc, open_file_std, read_file, time, write_file, EmailAddress, }; -use crate::{e2ee, tools}; // Name of the database file in the backup. const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite"; @@ -511,23 +516,9 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res let _d1 = DeleteOnDrop(temp_db_path.clone()); let _d2 = DeleteOnDrop(temp_path.clone()); - context - .sql - .set_raw_config_int("backup_time", now as i32) - .await?; - sql::housekeeping(context).await.ok_or_log(context); - - context - .sql - .execute("VACUUM;", paramsv![]) + export_database(context, &temp_db_path, passphrase) .await - .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e)) - .ok(); - - ensure!( - context.scheduler.read().await.is_none(), - "cannot export backup, IO is running" - ); + .context("could not export database")?; info!( context, @@ -536,12 +527,6 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res dest_path.display(), ); - context - .sql - .export(&temp_db_path, passphrase) - .await - .with_context(|| format!("failed to backup plaintext database to {temp_db_path:?}"))?; - let res = export_backup_inner(context, &temp_db_path, &temp_path).await; match &res { @@ -579,32 +564,11 @@ async fn export_backup_inner( .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME) .await?; - let read_dir = tools::read_dir(context.get_blobdir()).await?; - let count = read_dir.len(); - let mut written_files = 0; - - let mut last_progress = 0; - for entry in read_dir.into_iter() { - let name = entry.file_name(); - if !entry.file_type().await?.is_file() { - warn!( - context, - "Export: Found dir entry {} that is not a file, ignoring", - name.to_string_lossy() - ); - continue; - } - let mut file = File::open(entry.path()).await?; - let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(name); + let blobdir = BlobDirContents::new(context).await?; + for blob in blobdir.iter() { + let mut file = File::open(blob.to_abs_path()).await?; + let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(blob.as_name()); builder.append_file(path_in_archive, &mut file).await?; - - written_files += 1; - let progress = 1000 * written_files / count; - if progress != last_progress && progress > 10 && progress < 1000 { - // We already emitted ImexProgress(10) above - context.emit_event(EventType::ImexProgress(progress)); - last_progress = progress; - } } builder.finish().await?; @@ -763,6 +727,242 @@ where Ok(()) } +/// Exports the database to *file*, encrypted using *passphrase*. +/// +/// The directory of *file* must already exist, if *file* itself exists it will be +/// overwritten. +/// +/// This also verifies that IO is not running during the export. +async fn export_database(context: &Context, dest: &Path, passphrase: String) -> Result<()> { + ensure!( + context.scheduler.read().await.is_none(), + "cannot export backup, IO is running" + ); + let now = time().try_into().context("32-bit UNIX time overflow")?; + + context.sql.set_raw_config_int("backup_time", now).await?; + sql::housekeeping(context).await.ok_or_log(context); + context + .sql + .execute("VACUUM;", paramsv![]) + .await + .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e)) + .ok(); + context + .sql + .export(dest, passphrase) + .await + .with_context(|| format!("failed to backup database to {}", dest.display()))?; + Ok(()) +} + +#[derive(Debug)] +pub struct BackupSender { + /// A handle to the running provider. + provider: sendme::provider::Provider, + /// The ticket to retrieve the backup collection. + ticket: sendme::provider::Ticket, + /// Token holding the "ongoing" mutex. When this completes the provider should shut + /// down. + cancel_token: Receiver<()>, +} + +impl BackupSender { + // - [x] check i/o is not running + // - [x] check we have secret key + // - [x] alloc ongoing + // - [ ] correctly cancel Provider when cancelled + // - [x] create auth token + // - [x] export backup with generated token as password + // - needs a path to store the database + // - [x] create the sendme database + // - [x] start provider + // - [ ] wait for one successful connection + // - [ ] shutdown when a connection is closed, successful or not. + // + // - [ ] provide progress report + /// Prepares for sending a backup to a second device. + /// + /// Before calling this function all I/O must be stopped so that no changes to the blobs + /// or database are happening, this is done by calling the `dc_accounts_stop_io` or + /// `dc_stop_io` APIs first. TODO: Add the rust equivalents. + /// + /// This will acquire the global "ongoing" mutex. + pub async fn perpare(context: &Context, dir: &Path) -> Result { + ensure!( + // TODO: Should we worry about path normalisation? + dir != context.get_blobdir(), + "Temporary database export directory should not be in blobdir" + ); + e2ee::ensure_secret_key_exists(context) + .await + .context("Private key not available, aborting backup export")?; + + // Acquire global "ongoing" mutex. + let cancel_token = context.alloc_ongoing().await?; + let res = tokio::select! { + biased; + res = Self::prepare_inner(context, dir) => { + match res { + Ok(slf) => { + // TODO: maybe this is the wrong place to log this + // TODO: Also needs to log progress somehow. + info!(context, "Waiting for remote to connect"); + Ok(slf) + }, + Err(err) => { + error!(context, "Failed to set up second device setup: {:#}", err); + Err(err) + }, + } + }, + _ = cancel_token.recv() => Err(format_err!("cancelled")), + }; + + // TODO: This is all wrong, too early to release + context.free_ongoing().await; + res + } + + async fn prepare_inner(context: &Context, dir: &Path) -> Result { + // Generate the token up front: we also use it to encrypt the database. + let token = sendme::protocol::AuthToken::generate(); + let dbfile = dir.join(DBFILE_BACKUP_NAME); + export_database(context, &dbfile, token.to_string()) + .await + .context("Database export failed")?; + + // Now we can be sure IO is not running. + let mut files = vec![sendme::provider::DataSource::from(dbfile)]; + let blobdir = BlobDirContents::new(context).await?; + for blob in blobdir.iter() { + files.push(blob.to_abs_path().into()); + } + + // Start listening. + let (db, hash) = sendme::provider::create_collection(files).await?; + let provider = sendme::provider::Provider::builder(db) + .auth_token(token) + .spawn()?; + let ticket = provider.ticket(hash); + Ok(Self { provider, ticket }) + } + + pub fn qr(&self) -> Qr { + Qr::Backup { + ticket: self.ticket.clone(), + } + } + + pub async fn join(self) -> Result<(), JoinError> { + // TODO: should wait for 1 transfer to complete or abort + self.provider.join().await + } + + pub fn abort(&self) { + self.provider.abort() + } +} + +/// All files in the blobdir. +/// +/// This exists so we can have a [`BlobDirIter`] which needs something to own the data of +/// it's `&Path`. Use [`BlobDirContents::iter`] to create the iterator. +/// +/// Additionally pre-allocating this means we get a length for progress report. +struct BlobDirContents<'a> { + inner: Vec, + context: &'a Context, +} + +impl<'a> BlobDirContents<'a> { + async fn new(context: &'a Context) -> Result> { + let readdir = fs::read_dir(context.get_blobdir()).await?; + let inner = ReadDirStream::new(readdir) + .filter_map(|entry| async move { + match entry { + Ok(entry) => Some(entry), + Err(err) => { + error!(context, "Failed to read blob file: {err}"); + None + } + } + }) + .filter_map(|entry| async move { + match entry.file_type().await.ok()?.is_file() { + true => Some(entry.path()), + false => { + warn!( + context, + "Export: Found blob dir entry {} that is not a file, ignoring", + entry.path().display() + ); + None + } + } + }) + .collect() + .await; + Ok(Self { inner, context }) + } + + fn iter(&self) -> BlobDirIter<'_> { + BlobDirIter::new(self.context, &self.inner) + } +} + +/// A stream for [`Blob`]s. +/// +/// The stream emits [`EventType::ImexProgress`] events as it being consumed. +/// +/// Because we like to know the total number of blobs to emit progress all the blobs are +/// read up front and stored. Luckily this also makes our life easier, since we now only +/// need to implement `Iterator` and not `Stream`. +struct BlobDirIter<'a> { + paths: &'a [PathBuf], + offset: usize, + last_progress: usize, + context: &'a Context, +} + +impl<'a> BlobDirIter<'a> { + fn new(context: &'a Context, paths: &'a [PathBuf]) -> BlobDirIter<'a> { + Self { + paths, + offset: 0, + last_progress: 0, + context, + } + } +} + +impl<'a> Iterator for BlobDirIter<'a> { + type Item = BlobObject<'a>; + + fn next(&mut self) -> Option { + while let Some(path) = self.paths.get(self.offset) { + self.offset += 1; + + // In theory this can error but we'd have corrupted filenames in the blobdir, so + // silently skipping them is fine. + match BlobObject::from_path(self.context, path) { + Ok(blob) => { + let progress = 1000 * self.offset / self.paths.len(); + if progress != self.last_progress && progress > 10 && progress < 1000 { + self.context.emit_event(EventType::ImexProgress(progress)); + self.last_progress = progress; + } + return Some(blob); + } + Err(err) => warn!(self.context, "{err}"), + } + } + None + } +} + +impl FusedIterator for BlobDirIter<'_> {} + #[cfg(test)] mod tests { use ::pgp::armor::BlockType; diff --git a/src/qr.rs b/src/qr.rs index 4fa8138cf..a29062f59 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -64,6 +64,9 @@ pub enum Qr { Account { domain: String, }, + Backup { + ticket: sendme::provider::Ticket, + }, WebrtcInstance { domain: String, instance_pattern: String, From 059af398ebd6f17aff3f4a6f982d493a7286020e Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 1 Feb 2023 17:06:07 +0100 Subject: [PATCH 002/103] Allow decoding the QR code --- src/qr.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/qr.rs b/src/qr.rs index a29062f59..349a18244 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -35,6 +35,7 @@ const VCARD_SCHEME: &str = "BEGIN:VCARD"; const SMTP_SCHEME: &str = "SMTP:"; const HTTP_SCHEME: &str = "http://"; const HTTPS_SCHEME: &str = "https://"; +const DCBACKUP_SCHEME: &str = "DCBACKUP:"; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Qr { @@ -137,6 +138,8 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result { dclogin_scheme::decode_login(qr)? } else if starts_with_ignore_case(qr, DCWEBRTC_SCHEME) { decode_webrtc_instance(context, qr)? + } else if starts_with_ignore_case(qr, DCBACKUP_SCHEME) { + decode_backup(qr)? } else if qr.starts_with(MAILTO_SCHEME) { decode_mailto(context, qr).await? } else if qr.starts_with(SMTP_SCHEME) { @@ -364,6 +367,18 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result { } } +/// Decodes a [`DCBACKUP_SCHEME`] QR code. +/// +/// The format of this scheme is `DCBACKUP:`. The encoding is the +/// [`sendme::provider::Ticket`]'s `Display` impl. +fn decode_backup(qr: &str) -> Result { + let payload = qr + .strip_prefix(DCBACKUP_SCHEME) + .ok_or(anyhow!("invalid DCBACKUP scheme"))?; + let ticket: sendme::provider::Ticket = payload.parse().context("invalid DCBACKUP payload")?; + Ok(Qr::Backup { ticket }) +} + #[derive(Debug, Deserialize)] struct CreateAccountSuccessResponse { email: String, From 2a6a21c33aa92fd5d672b272d6e60cff150feba0 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 1 Feb 2023 17:53:23 +0100 Subject: [PATCH 003/103] handle the ongoing process correctly --- src/imex.rs | 47 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/imex.rs b/src/imex.rs index f19b02c4c..8310e9a13 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -8,13 +8,12 @@ use std::iter::FusedIterator; use std::path::{Path, PathBuf}; use ::pgp::types::KeyTrait; -use anyhow::{bail, ensure, format_err, Context as _, Result}; +use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures::StreamExt; use futures_lite::FutureExt; use rand::{thread_rng, Rng}; use tokio::fs::{self, File}; -use tokio::task::JoinError; use tokio_stream::wrappers::ReadDirStream; use tokio_tar::Archive; @@ -771,7 +770,7 @@ impl BackupSender { // - [x] check i/o is not running // - [x] check we have secret key // - [x] alloc ongoing - // - [ ] correctly cancel Provider when cancelled + // - [x] correctly cancel Provider when cancelled // - [x] create auth token // - [x] export backup with generated token as password // - needs a path to store the database @@ -787,7 +786,9 @@ impl BackupSender { /// or database are happening, this is done by calling the `dc_accounts_stop_io` or /// `dc_stop_io` APIs first. TODO: Add the rust equivalents. /// - /// This will acquire the global "ongoing" mutex. + /// This will acquire the global "ongoing process" mutex. You must call + /// [`BackupSender::join`] after creating this struct, otherwise this will not respect + /// the possible cancellation of the "ongoing process". pub async fn perpare(context: &Context, dir: &Path) -> Result { ensure!( // TODO: Should we worry about path normalisation? @@ -818,13 +819,24 @@ impl BackupSender { }, _ = cancel_token.recv() => Err(format_err!("cancelled")), }; - - // TODO: This is all wrong, too early to release - context.free_ongoing().await; - res + let (provider, ticket) = match res { + Ok((provider, ticket)) => (provider, ticket), + Err(err) => { + context.free_ongoing().await; + return Err(err); + } + }; + Ok(Self { + provider, + ticket, + cancel_token, + }) } - async fn prepare_inner(context: &Context, dir: &Path) -> Result { + async fn prepare_inner( + context: &Context, + dir: &Path, + ) -> Result<(sendme::provider::Provider, sendme::provider::Ticket)> { // Generate the token up front: we also use it to encrypt the database. let token = sendme::protocol::AuthToken::generate(); let dbfile = dir.join(DBFILE_BACKUP_NAME); @@ -845,7 +857,7 @@ impl BackupSender { .auth_token(token) .spawn()?; let ticket = provider.ticket(hash); - Ok(Self { provider, ticket }) + Ok((provider, ticket)) } pub fn qr(&self) -> Qr { @@ -854,9 +866,20 @@ impl BackupSender { } } - pub async fn join(self) -> Result<(), JoinError> { + /// Wait for the backup sender to complete. + /// + /// The sender completes when an authenticated client disconnects, whether the transfer + /// was successful or not. When the ongoing task is cancelled the sender also completes + /// with an error. + /// + /// Note that this must be called and awaited for the ongoing cancellation to work. + pub async fn join(self) -> Result<()> { // TODO: should wait for 1 transfer to complete or abort - self.provider.join().await + tokio::select! { + biased; + res = self.provider.join() => res.context("BackupSender failed"), + _ = self.cancel_token.recv() => Err(anyhow!("BackupSender cancelled")), + } } pub fn abort(&self) { From 5f29b9397005f58d86fd83225ecc3a799d309fd1 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Feb 2023 17:15:23 +0100 Subject: [PATCH 004/103] Start of get support and create new module. --- src/imex.rs | 140 +-------------------------- src/imex/transfer.rs | 224 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 229 insertions(+), 135 deletions(-) create mode 100644 src/imex/transfer.rs diff --git a/src/imex.rs b/src/imex.rs index 8310e9a13..34b4caa62 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -8,8 +8,7 @@ use std::iter::FusedIterator; use std::path::{Path, PathBuf}; use ::pgp::types::KeyTrait; -use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; -use async_channel::Receiver; +use anyhow::{bail, ensure, format_err, Context as _, Result}; use futures::StreamExt; use futures_lite::FutureExt; use rand::{thread_rng, Rng}; @@ -30,7 +29,6 @@ use crate::message::{Message, MsgId, Viewtype}; use crate::mimeparser::SystemMessage; use crate::param::Param; use crate::pgp; -use crate::qr::Qr; use crate::sql; use crate::stock_str; use crate::tools::{ @@ -38,6 +36,10 @@ use crate::tools::{ EmailAddress, }; +mod transfer; + +pub use transfer::{get_backup, BackupProvider}; + // Name of the database file in the backup. const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite"; const BLOBS_BACKUP_NAME: &str = "blobs_backup"; @@ -755,138 +757,6 @@ async fn export_database(context: &Context, dest: &Path, passphrase: String) -> Ok(()) } -#[derive(Debug)] -pub struct BackupSender { - /// A handle to the running provider. - provider: sendme::provider::Provider, - /// The ticket to retrieve the backup collection. - ticket: sendme::provider::Ticket, - /// Token holding the "ongoing" mutex. When this completes the provider should shut - /// down. - cancel_token: Receiver<()>, -} - -impl BackupSender { - // - [x] check i/o is not running - // - [x] check we have secret key - // - [x] alloc ongoing - // - [x] correctly cancel Provider when cancelled - // - [x] create auth token - // - [x] export backup with generated token as password - // - needs a path to store the database - // - [x] create the sendme database - // - [x] start provider - // - [ ] wait for one successful connection - // - [ ] shutdown when a connection is closed, successful or not. - // - // - [ ] provide progress report - /// Prepares for sending a backup to a second device. - /// - /// Before calling this function all I/O must be stopped so that no changes to the blobs - /// or database are happening, this is done by calling the `dc_accounts_stop_io` or - /// `dc_stop_io` APIs first. TODO: Add the rust equivalents. - /// - /// This will acquire the global "ongoing process" mutex. You must call - /// [`BackupSender::join`] after creating this struct, otherwise this will not respect - /// the possible cancellation of the "ongoing process". - pub async fn perpare(context: &Context, dir: &Path) -> Result { - ensure!( - // TODO: Should we worry about path normalisation? - dir != context.get_blobdir(), - "Temporary database export directory should not be in blobdir" - ); - e2ee::ensure_secret_key_exists(context) - .await - .context("Private key not available, aborting backup export")?; - - // Acquire global "ongoing" mutex. - let cancel_token = context.alloc_ongoing().await?; - let res = tokio::select! { - biased; - res = Self::prepare_inner(context, dir) => { - match res { - Ok(slf) => { - // TODO: maybe this is the wrong place to log this - // TODO: Also needs to log progress somehow. - info!(context, "Waiting for remote to connect"); - Ok(slf) - }, - Err(err) => { - error!(context, "Failed to set up second device setup: {:#}", err); - Err(err) - }, - } - }, - _ = cancel_token.recv() => Err(format_err!("cancelled")), - }; - let (provider, ticket) = match res { - Ok((provider, ticket)) => (provider, ticket), - Err(err) => { - context.free_ongoing().await; - return Err(err); - } - }; - Ok(Self { - provider, - ticket, - cancel_token, - }) - } - - async fn prepare_inner( - context: &Context, - dir: &Path, - ) -> Result<(sendme::provider::Provider, sendme::provider::Ticket)> { - // Generate the token up front: we also use it to encrypt the database. - let token = sendme::protocol::AuthToken::generate(); - let dbfile = dir.join(DBFILE_BACKUP_NAME); - export_database(context, &dbfile, token.to_string()) - .await - .context("Database export failed")?; - - // Now we can be sure IO is not running. - let mut files = vec![sendme::provider::DataSource::from(dbfile)]; - let blobdir = BlobDirContents::new(context).await?; - for blob in blobdir.iter() { - files.push(blob.to_abs_path().into()); - } - - // Start listening. - let (db, hash) = sendme::provider::create_collection(files).await?; - let provider = sendme::provider::Provider::builder(db) - .auth_token(token) - .spawn()?; - let ticket = provider.ticket(hash); - Ok((provider, ticket)) - } - - pub fn qr(&self) -> Qr { - Qr::Backup { - ticket: self.ticket.clone(), - } - } - - /// Wait for the backup sender to complete. - /// - /// The sender completes when an authenticated client disconnects, whether the transfer - /// was successful or not. When the ongoing task is cancelled the sender also completes - /// with an error. - /// - /// Note that this must be called and awaited for the ongoing cancellation to work. - pub async fn join(self) -> Result<()> { - // TODO: should wait for 1 transfer to complete or abort - tokio::select! { - biased; - res = self.provider.join() => res.context("BackupSender failed"), - _ = self.cancel_token.recv() => Err(anyhow!("BackupSender cancelled")), - } - } - - pub fn abort(&self) { - self.provider.abort() - } -} - /// All files in the blobdir. /// /// This exists so we can have a [`BlobDirIter`] which needs something to own the data of diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs new file mode 100644 index 000000000..131692e6c --- /dev/null +++ b/src/imex/transfer.rs @@ -0,0 +1,224 @@ +//! Transfer a backup to an other device. +//! +//! This module provides support for using n0's sendme tool to initiate transfer of a backup +//! to another device using a QR code. +//! +//! Using the sendme terminology there are two parties to this: +//! +//! - The *Provider*, which starts a server and listens for connections. +//! - The *Getter*, which connects to the server and retrieves the data. +//! +//! Sendme is designed around the idea of verifying hashes, the downloads are verified as +//! they are retrieved. The entire transfer is initiated by requesting the data of a single +//! root hash. +//! +//! Both the provider and the getter are authenticated: +//! +//! - The provider is known by its *peer ID*. +//! - The provider needs an *authentication token* from the getter before it accepts a +//! connection. +//! +//! Both these are transferred in the QR code offered to the getter. This ensures that the +//! getter can not connect to an impersonated provider and the provider does not offer the +//! download to an impersonated getter. + +use std::path::Path; + +use crate::context::Context; +use crate::e2ee; +use crate::qr::Qr; +use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; +use async_channel::Receiver; +use sendme::blobs::Collection; +use sendme::get::{AsyncSliceDecoder, Hash, Options, ReceiveStream}; +use sendme::protocol::AuthToken; +use sendme::provider::{DataSource, Provider, Ticket}; +use tokio::fs::File; +use tokio::io::{self, BufWriter}; + +use super::{export_database, BlobDirContents, DBFILE_BACKUP_NAME}; + +/// Provide or send a backup of this device. +/// +/// This creates a backup of the current device and starts a service which offers another +/// device to download this backup. +/// +/// This does not make a full backup on disk, only the SQLite database is created on disk, +/// the blobs in the blob directory are not copied. +#[derive(Debug)] +pub struct BackupProvider { + /// A handle to the running provider. + provider: Provider, + /// The ticket to retrieve the backup collection. + ticket: Ticket, + /// Token holding the "ongoing" mutex. When this completes the provider should shut + /// down. + cancel_token: Receiver<()>, +} + +impl BackupProvider { + /// Prepares for sending a backup to a second device. + /// + /// Before calling this function all I/O must be stopped so that no changes to the blobs + /// or database are happening, this is done by calling the `dc_accounts_stop_io` or + /// `dc_stop_io` APIs first. TODO: Add the rust equivalents. + /// + /// This will acquire the global "ongoing process" mutex. You must call + /// [`BackupSender::join`] after creating this struct, otherwise this will not respect + /// the possible cancellation of the "ongoing process". + pub async fn perpare(context: &Context, dir: &Path) -> Result { + ensure!( + // TODO: Should we worry about path normalisation? + dir != context.get_blobdir(), + "Temporary database export directory should not be in blobdir" + ); + e2ee::ensure_secret_key_exists(context) + .await + .context("Private key not available, aborting backup export")?; + + // Acquire global "ongoing" mutex. + let cancel_token = context.alloc_ongoing().await?; + let res = tokio::select! { + biased; + res = Self::prepare_inner(context, dir) => { + match res { + Ok(slf) => { + // TODO: maybe this is the wrong place to log this + // TODO: Also needs to log progress somehow. + info!(context, "Waiting for remote to connect"); + Ok(slf) + }, + Err(err) => { + error!(context, "Failed to set up second device setup: {:#}", err); + Err(err) + }, + } + }, + _ = cancel_token.recv() => Err(format_err!("cancelled")), + }; + let (provider, ticket) = match res { + Ok((provider, ticket)) => (provider, ticket), + Err(err) => { + context.free_ongoing().await; + return Err(err); + } + }; + Ok(Self { + provider, + ticket, + cancel_token, + }) + } + + async fn prepare_inner(context: &Context, dir: &Path) -> Result<(Provider, Ticket)> { + // Generate the token up front: we also use it to encrypt the database. + let token = AuthToken::generate(); + let dbfile = dir.join(DBFILE_BACKUP_NAME); + export_database(context, &dbfile, token.to_string()) + .await + .context("Database export failed")?; + + // Now we can be sure IO is not running. + let mut files = vec![DataSource::from(dbfile)]; + let blobdir = BlobDirContents::new(context).await?; + for blob in blobdir.iter() { + files.push(blob.to_abs_path().into()); + } + + // Start listening. + let (db, hash) = sendme::provider::create_collection(files).await?; + let provider = Provider::builder(db).auth_token(token).spawn()?; + let ticket = provider.ticket(hash); + Ok((provider, ticket)) + } + + pub fn qr(&self) -> Qr { + Qr::Backup { + ticket: self.ticket.clone(), + } + } + + /// Wait for the backup sender to complete. + /// + /// The sender completes when an authenticated client disconnects, whether the transfer + /// was successful or not. When the ongoing task is cancelled the sender also completes + /// with an error. + /// + /// Note that this must be called and awaited for the ongoing cancellation to work. + pub async fn join(self) -> Result<()> { + // TODO: should wait for 1 transfer to complete or abort + tokio::select! { + biased; + res = self.provider.join() => res.context("BackupSender failed"), + _ = self.cancel_token.recv() => Err(anyhow!("BackupSender cancelled")), + } + } + + pub fn abort(&self) { + self.provider.abort() + } +} + +/// Contacts a backup provider and receives the backup from it. +/// +/// This uses a QR code to contact another instance of deltachat which is providing a backup +/// using the [`BackupProvider`]. Once connected it will authenticate using the secrets in +/// the QR code and retrieve the backup. +/// +/// Using [`Qr`] as argument is a bit odd as it only accepts one specific variant of it. It +/// does avoid having [`sendme::provider::Ticket`] in the primary API however, without +/// having to revert to untyped bytes. +pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { + let Qr::Backup { ticket } = qr else { + bail!("QR code for backup must be of type DCBACKUP"); + }; + ensure!( + !context.is_configured().await?, + "Cannot import backups to accounts in use." + ); + ensure!( + context.scheduler.read().await.is_none(), + "cannot import backup, IO is running" + ); + let opts = Options { + addr: ticket.addr, + peer_id: Some(ticket.peer), + }; + let on_blob = |hash, reader, name| on_blob(context, hash, reader, name); + sendme::get::run( + ticket.hash, + ticket.token, + opts, + on_connected, + on_collection, + on_blob, + ) + .await?; + + todo!(); +} + +/// Get callback when the connection is established with the provider. +async fn on_connected() -> Result<()> { + Ok(()) +} + +/// Get callback when a collection is received from the provider. +async fn on_collection(_collection: Collection) -> Result<()> { + Ok(()) +} + +/// Get callback when a blob is received from the provider. +async fn on_blob( + context: &Context, + _hash: Hash, + mut reader: AsyncSliceDecoder, + name: String, +) -> Result> { + ensure!(!name.is_empty(), "Received a nameless blob"); + let path = context.get_blobdir().join(name); + let file = File::create(&path).await?; + let mut file = BufWriter::with_capacity(128 * 1024, file); + io::copy(&mut reader, &mut file).await?; + Ok(reader) +} From 3267596a3085ef364b798e5a981ee1ffecb84634 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Feb 2023 17:43:12 +0100 Subject: [PATCH 005/103] handle the database --- deltachat-ffi/src/lot.rs | 2 ++ src/blob.rs | 3 ++- src/imex/transfer.rs | 29 ++++++++++++++++++++++++----- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/deltachat-ffi/src/lot.rs b/deltachat-ffi/src/lot.rs index 270cd34ea..612f7d1cd 100644 --- a/deltachat-ffi/src/lot.rs +++ b/deltachat-ffi/src/lot.rs @@ -14,6 +14,8 @@ use crate::summary::{Summary, SummaryPrefix}; /// eg. by chatlist.get_summary() or dc_msg_get_summary(). /// /// *Lot* is used in the meaning *heap* here. +// The QR code grew too large. So be it. +#[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum Lot { Summary(Summary), diff --git a/src/blob.rs b/src/blob.rs index ef2869bed..f291fdde5 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -242,7 +242,8 @@ impl<'a> BlobObject<'a> { /// including the dot. E.g. "foo.txt" is returned as `("foo", /// ".txt")` while "bar" is returned as `("bar", "")`. /// - /// The extension part will always be lowercased. + /// The extension part will always be lowercased. Note that [`imex::transfer`] relies + /// on this for safety, if uppercase extensions are ever allowed it needs to be adapted. fn sanitise_name(name: &str) -> (String, String) { let mut name = name.to_string(); for part in name.rsplit('/') { diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 131692e6c..74dce2963 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -33,7 +33,7 @@ use sendme::blobs::Collection; use sendme::get::{AsyncSliceDecoder, Hash, Options, ReceiveStream}; use sendme::protocol::AuthToken; use sendme::provider::{DataSource, Provider, Ticket}; -use tokio::fs::File; +use tokio::fs::{self, File}; use tokio::io::{self, BufWriter}; use super::{export_database, BlobDirContents, DBFILE_BACKUP_NAME}; @@ -184,7 +184,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { addr: ticket.addr, peer_id: Some(ticket.peer), }; - let on_blob = |hash, reader, name| on_blob(context, hash, reader, name); + let on_blob = |hash, reader, name| on_blob(context, &ticket, hash, reader, name); sendme::get::run( ticket.hash, ticket.token, @@ -194,16 +194,17 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { on_blob, ) .await?; - - todo!(); + Ok(()) } /// Get callback when the connection is established with the provider. +#[allow(clippy::unused_async)] async fn on_connected() -> Result<()> { Ok(()) } /// Get callback when a collection is received from the provider. +#[allow(clippy::unused_async)] async fn on_collection(_collection: Collection) -> Result<()> { Ok(()) } @@ -211,14 +212,32 @@ async fn on_collection(_collection: Collection) -> Result<()> { /// Get callback when a blob is received from the provider. async fn on_blob( context: &Context, + ticket: &Ticket, _hash: Hash, mut reader: AsyncSliceDecoder, name: String, ) -> Result> { ensure!(!name.is_empty(), "Received a nameless blob"); - let path = context.get_blobdir().join(name); + let path = if name == DBFILE_BACKUP_NAME { + // We can only safely write to the blobdir. But the blobdir could have a file named + // exactly like our special name. We solve this by using an uppercase extension + // which is forbidden for normal blobs. + context.get_blobdir().join(format!("{name}.SPECIAL")) + } else { + context.get_blobdir().join(&name) + }; let file = File::create(&path).await?; let mut file = BufWriter::with_capacity(128 * 1024, file); io::copy(&mut reader, &mut file).await?; + if name == DBFILE_BACKUP_NAME { + context + .sql + .import(&path, ticket.token.to_string()) + .await + .context("cannot import database")?; + fs::remove_file(&path) + .await + .with_context(|| format!("Database file: {}", path.display()))?; + } Ok(reader) } From ca1533b0e4c9a85f53db36a2ac5f20ff580f17ad Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Feb 2023 17:47:41 +0100 Subject: [PATCH 006/103] delete device messages --- src/imex/transfer.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 74dce2963..30031f7da 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -24,6 +24,7 @@ use std::path::Path; +use crate::chat::delete_and_reset_all_device_msgs; use crate::context::Context; use crate::e2ee; use crate::qr::Qr; @@ -194,6 +195,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { on_blob, ) .await?; + delete_and_reset_all_device_msgs(context).await?; Ok(()) } @@ -237,7 +239,7 @@ async fn on_blob( .context("cannot import database")?; fs::remove_file(&path) .await - .with_context(|| format!("Database file: {}", path.display()))?; + .with_context(|| format!("database import file: {}", path.display()))?; } Ok(reader) } From a6c889ed5e75da7d37a421bf4001c5cdea0f3ca8 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Feb 2023 18:11:12 +0100 Subject: [PATCH 007/103] Clean up files on errors --- src/imex/transfer.rs | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 30031f7da..f97122d84 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -30,14 +30,16 @@ use crate::e2ee; use crate::qr::Qr; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; +use futures_lite::StreamExt; use sendme::blobs::Collection; use sendme::get::{AsyncSliceDecoder, Hash, Options, ReceiveStream}; use sendme::protocol::AuthToken; use sendme::provider::{DataSource, Provider, Ticket}; use tokio::fs::{self, File}; use tokio::io::{self, BufWriter}; +use tokio_stream::wrappers::ReadDirStream; -use super::{export_database, BlobDirContents, DBFILE_BACKUP_NAME}; +use super::{export_database, BlobDirContents, DeleteOnDrop, DBFILE_BACKUP_NAME}; /// Provide or send a backup of this device. /// @@ -186,7 +188,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { peer_id: Some(ticket.peer), }; let on_blob = |hash, reader, name| on_blob(context, &ticket, hash, reader, name); - sendme::get::run( + match sendme::get::run( ticket.hash, ticket.token, opts, @@ -194,9 +196,24 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { on_collection, on_blob, ) - .await?; - delete_and_reset_all_device_msgs(context).await?; - Ok(()) + .await + { + Ok(_) => { + delete_and_reset_all_device_msgs(context).await?; + Ok(()) + } + Err(err) => { + // Clean up any blobs we already wrote. + let readdir = fs::read_dir(context.get_blobdir()).await?; + let mut readdir = ReadDirStream::new(readdir); + while let Some(dirent) = readdir.next().await { + if let Ok(dirent) = dirent { + fs::remove_file(dirent.path()).await.ok(); + } + } + Err(err) + } + } } /// Get callback when the connection is established with the provider. @@ -228,6 +245,11 @@ async fn on_blob( } else { context.get_blobdir().join(&name) }; + let _guard = if name == DBFILE_BACKUP_NAME { + Some(DeleteOnDrop(path.clone())) + } else { + None + }; let file = File::create(&path).await?; let mut file = BufWriter::with_capacity(128 * 1024, file); io::copy(&mut reader, &mut file).await?; From 0b075ac762c527d210fb7fff65ef51913f2bbc59 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 6 Feb 2023 14:58:08 +0100 Subject: [PATCH 008/103] Stop after a transfer happened. --- Cargo.lock | 88 +++---------------------------------- src/imex/transfer.rs | 102 ++++++++++++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f3fec720..bcf3bdbd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -705,7 +705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" dependencies = [ "heck", - "proc-macro-error 1.0.4", + "proc-macro-error", "proc-macro2", "quote", "syn", @@ -1237,7 +1237,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ef71ddb5b3a1f53dee24817c8f70dfa1cb29e804c18d88c228d4bc9c86ee3b9" dependencies = [ - "proc-macro-error 1.0.4", + "proc-macro-error", "proc-macro2", "quote", "syn", @@ -1846,37 +1846,6 @@ dependencies = [ "slab", ] -[[package]] -name = "genawaiter" -version = "0.99.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" -dependencies = [ - "futures-core", - "genawaiter-macro", - "genawaiter-proc-macro", - "proc-macro-hack", -] - -[[package]] -name = "genawaiter-macro" -version = "0.99.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" - -[[package]] -name = "genawaiter-proc-macro" -version = "0.99.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" -dependencies = [ - "proc-macro-error 0.4.12", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "generic-array" version = "0.14.6" @@ -2888,7 +2857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "734aa7a4a6390b162112523cac2923a18e4f23b917880a68c826bf6e8bf48f06" dependencies = [ "Inflector", - "proc-macro-error 1.0.4", + "proc-macro-error", "proc-macro2", "quote", "syn", @@ -3159,45 +3128,19 @@ dependencies = [ "log", ] -[[package]] -name = "proc-macro-error" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" -dependencies = [ - "proc-macro-error-attr 0.4.12", - "proc-macro2", - "quote", - "syn", - "version_check 0.9.4", -] - [[package]] name = "proc-macro-error" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ - "proc-macro-error-attr 1.0.4", + "proc-macro-error-attr", "proc-macro2", "quote", "syn", "version_check 0.9.4", ] -[[package]] -name = "proc-macro-error-attr" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "syn-mid", - "version_check 0.9.4", -] - [[package]] name = "proc-macro-error-attr" version = "1.0.4" @@ -3209,12 +3152,6 @@ dependencies = [ "version_check 0.9.4", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.46" @@ -3934,6 +3871,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bao", + "base64 0.21.0", "blake3", "bytes", "clap 4.1.4", @@ -3941,10 +3879,7 @@ dependencies = [ "der", "ed25519-dalek", "futures", - "genawaiter", - "hex", "indicatif", - "is-terminal", "postcard", "rand 0.7.3", "rcgen", @@ -4266,17 +4201,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn-mid" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa8e7560a164edb1621a55d18a0c59abf49d360f47aa7b821061dd7eea7fac9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "sync_wrapper" version = "0.1.1" @@ -4817,7 +4741,7 @@ checksum = "e5c1bfe689e4067733530495b04959b00f05cd95f038bed59af4fc70b3e26240" dependencies = [ "darling 0.13.4", "ident_case", - "proc-macro-error 1.0.4", + "proc-macro-error", "proc-macro2", "quote", "syn", diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index f97122d84..aaeb952d2 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -32,11 +32,14 @@ use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures_lite::StreamExt; use sendme::blobs::Collection; -use sendme::get::{AsyncSliceDecoder, Hash, Options, ReceiveStream}; +use sendme::get::{DataStream, Options}; use sendme::protocol::AuthToken; -use sendme::provider::{DataSource, Provider, Ticket}; +use sendme::provider::{DataSource, Event, Provider, Ticket}; +use sendme::Hash; use tokio::fs::{self, File}; use tokio::io::{self, BufWriter}; +use tokio::sync::broadcast; +use tokio::task::JoinHandle; use tokio_stream::wrappers::ReadDirStream; use super::{export_database, BlobDirContents, DeleteOnDrop, DBFILE_BACKUP_NAME}; @@ -48,15 +51,15 @@ use super::{export_database, BlobDirContents, DeleteOnDrop, DBFILE_BACKUP_NAME}; /// /// This does not make a full backup on disk, only the SQLite database is created on disk, /// the blobs in the blob directory are not copied. +/// +/// This starts a task which acquires the global "ongoing" mutex. If you need to stop the +/// task use the [`Context::stop_ongoing`] mechanism. #[derive(Debug)] pub struct BackupProvider { - /// A handle to the running provider. - provider: Provider, + /// The supervisor task, run by [`BackupProvider::watch_provider`]. + handle: JoinHandle>, /// The ticket to retrieve the backup collection. ticket: Ticket, - /// Token holding the "ongoing" mutex. When this completes the provider should shut - /// down. - cancel_token: Receiver<()>, } impl BackupProvider { @@ -106,13 +109,13 @@ impl BackupProvider { return Err(err); } }; - Ok(Self { - provider, - ticket, - cancel_token, - }) + let handle = tokio::spawn(Self::watch_provider(provider, cancel_token)); + Ok(Self { handle, ticket }) } + /// Creates the provider and supervisor tasks. + /// + /// Having this as a function makes it easier to cancel it when needed. async fn prepare_inner(context: &Context, dir: &Path) -> Result<(Provider, Ticket)> { // Generate the token up front: we also use it to encrypt the database. let token = AuthToken::generate(); @@ -135,31 +138,62 @@ impl BackupProvider { Ok((provider, ticket)) } + /// Supervises the sendme [`Provider`] terminating it when needed. + /// + /// This will watch the provider and terminate it when: + /// - A transfer is completed, successful or unsuccessful. + /// - An event could not be observed to protect against not knowing of a completed event. + /// - The ongoing process is cancelled. + /// + /// The *cancel_token* is the handle for the ongoing process mutex, when this completes + /// we must cancel this operation. + async fn watch_provider(mut provider: Provider, cancel_token: Receiver<()>) -> Result<()> { + let mut events = provider.subscribe(); + loop { + tokio::select! { + biased; + res = &mut provider => { + return res.context("BackupSender failed"); + }, + maybe_event = events.recv() => { + match maybe_event { + Ok(event) => { + match event { + Event::TransferCompleted { .. } => { + provider.abort(); + } + Event::TransferAborted { .. } => { + provider.abort(); + return Err(anyhow!("BackupSender transfer aborted")); + } + _ => (), + } + } + Err(broadcast::error::RecvError::Closed) => { + // We should never see this, provider.join() should complete + // first. + } + Err(broadcast::error::RecvError::Lagged(_)) => { + // We really shouldn't be lagging, if we did we may have missed + // a completion event. + provider.abort(); + return Err(anyhow!("Missed events from BackupSender")); + } + } + }, + _ = cancel_token.recv() => { + provider.abort(); + return Err(anyhow!("BackupSender cancelled")); + }, + } + } + } + pub fn qr(&self) -> Qr { Qr::Backup { ticket: self.ticket.clone(), } } - - /// Wait for the backup sender to complete. - /// - /// The sender completes when an authenticated client disconnects, whether the transfer - /// was successful or not. When the ongoing task is cancelled the sender also completes - /// with an error. - /// - /// Note that this must be called and awaited for the ongoing cancellation to work. - pub async fn join(self) -> Result<()> { - // TODO: should wait for 1 transfer to complete or abort - tokio::select! { - biased; - res = self.provider.join() => res.context("BackupSender failed"), - _ = self.cancel_token.recv() => Err(anyhow!("BackupSender cancelled")), - } - } - - pub fn abort(&self) { - self.provider.abort() - } } /// Contacts a backup provider and receives the backup from it. @@ -233,9 +267,9 @@ async fn on_blob( context: &Context, ticket: &Ticket, _hash: Hash, - mut reader: AsyncSliceDecoder, + mut reader: DataStream, name: String, -) -> Result> { +) -> Result { ensure!(!name.is_empty(), "Received a nameless blob"); let path = if name == DBFILE_BACKUP_NAME { // We can only safely write to the blobdir. But the blobdir could have a file named From 187861c3b2cc2293e58cf9308675e740062ffe3d Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Feb 2023 17:18:34 +0100 Subject: [PATCH 009/103] Make stuff work. With test! --- Cargo.lock | 1123 ++++++++++++++++++++++++++++++------------ Cargo.toml | 1 + src/imex/transfer.rs | 104 +++- 3 files changed, 893 insertions(+), 335 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bcf3bdbd8..7d504893e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,7 +29,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher", "cpufeatures", ] @@ -81,7 +81,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -171,7 +171,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio", + "tokio 1.25.0", ] [[package]] @@ -184,7 +184,7 @@ dependencies = [ "base64 0.13.1", "byte-pool", "chrono", - "futures", + "futures 0.3.26", "imap-proto", "log", "nom 7.1.1", @@ -193,7 +193,7 @@ dependencies = [ "pin-utils", "stop-token", "thiserror", - "tokio", + "tokio 1.25.0", ] [[package]] @@ -213,7 +213,7 @@ checksum = "d57d4cec3c647232e1094dc013546c0b33ce785d8aeb251e1f20dfaf8a9a13fe" dependencies = [ "native-tls", "thiserror", - "tokio", + "tokio 1.25.0", "url", ] @@ -228,14 +228,14 @@ dependencies = [ "base64 0.13.1", "bufstream", "fast-socks5", - "futures", + "futures 0.3.26", "hostname", "log", "nom 7.1.1", "pin-project", "pin-utils", "thiserror", - "tokio", + "tokio 1.25.0", ] [[package]] @@ -255,7 +255,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b20cffc5590f4bf33f05f97a3ea587feba9c50d20325b401daa096b92ff7da0" dependencies = [ - "tokio", + "tokio 1.25.0", ] [[package]] @@ -269,7 +269,7 @@ dependencies = [ "chrono", "crc32fast", "thiserror", - "tokio", + "tokio 1.25.0", ] [[package]] @@ -280,7 +280,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -299,7 +299,7 @@ dependencies = [ "axum-core 0.2.9", "base64 0.13.1", "bitflags", - "bytes", + "bytes 1.2.1", "futures-util", "http", "http-body", @@ -315,7 +315,7 @@ dependencies = [ "serde_urlencoded", "sha-1", "sync_wrapper", - "tokio", + "tokio 1.25.0", "tokio-tungstenite 0.17.2", "tower", "tower-http", @@ -333,7 +333,7 @@ dependencies = [ "axum-core 0.3.2", "base64 0.20.0", "bitflags", - "bytes", + "bytes 1.2.1", "futures-util", "http", "http-body", @@ -351,7 +351,7 @@ dependencies = [ "serde_urlencoded", "sha1", "sync_wrapper", - "tokio", + "tokio 1.25.0", "tokio-tungstenite 0.18.0", "tower", "tower-http", @@ -366,7 +366,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" dependencies = [ "async-trait", - "bytes", + "bytes 1.2.1", "futures-util", "http", "http-body", @@ -382,7 +382,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" dependencies = [ "async-trait", - "bytes", + "bytes 1.2.1", "futures-util", "http", "http-body", @@ -400,7 +400,7 @@ checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide 0.6.2", "object", @@ -418,6 +418,12 @@ dependencies = [ "blake3", ] +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "base16ct" version = "0.1.1" @@ -481,7 +487,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.0", "constant_time_eq", "digest 0.10.5", ] @@ -551,7 +557,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c7230ddbb427b1094d477d821a99f3f54d36333178eeb806e279bcdcecf0ca" dependencies = [ - "crossbeam-queue", + "crossbeam-queue 0.3.6", "stable_deref_trait", ] @@ -567,12 +573,53 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +[[package]] +name = "camino" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdb825da8a5df079a43676dbe042702f1707b1109f713a01420fbb4cc71fa27" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.16", + "serde", + "serde_json", +] + [[package]] name = "cast" version = "0.3.0" @@ -603,6 +650,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -631,7 +684,7 @@ dependencies = [ "num-traits", "time 0.1.44", "wasm-bindgen", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -737,7 +790,16 @@ checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219" dependencies = [ "error-code", "str-buf", - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", ] [[package]] @@ -758,7 +820,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.8.11", ] [[package]] @@ -849,7 +911,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -864,7 +926,7 @@ dependencies = [ "ciborium", "clap 3.2.23", "criterion-plot", - "futures", + "futures 0.3.26", "itertools", "lazy_static", "num-traits", @@ -876,7 +938,7 @@ dependencies = [ "serde_derive", "serde_json", "tinytemplate", - "tokio", + "tokio 1.25.0", "walkdir", ] @@ -896,8 +958,19 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -906,9 +979,24 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.10", + "crossbeam-utils 0.8.11", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", ] [[package]] @@ -918,21 +1006,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" dependencies = [ "autocfg", - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", "memoffset 0.6.5", "once_cell", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + [[package]] name = "crossbeam-queue" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", ] [[package]] @@ -941,7 +1051,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -967,17 +1077,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "cuckoofilter" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b810a8449931679f64cd7eef1bbd0fa315801b6d5d9cdc1ace2804d6529eee18" -dependencies = [ - "byteorder", - "fnv", - "rand 0.7.3", -] - [[package]] name = "curve25519-dalek" version = "3.2.0" @@ -1067,6 +1166,26 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" +[[package]] +name = "data-encoding-macro" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" +dependencies = [ + "data-encoding", + "syn", +] + [[package]] name = "deltachat" version = "1.107.0" @@ -1090,7 +1209,7 @@ dependencies = [ "escaper", "fast-socks5", "format-flowed", - "futures", + "futures 0.3.26", "futures-lite", "hex", "humansize", @@ -1126,14 +1245,15 @@ dependencies = [ "serde_json", "sha-1", "sha2 0.10.6", - "smallvec", + "smallvec 1.10.0", "strum", "strum_macros", "tagger", "tempfile", + "testdir", "textwrap", "thiserror", - "tokio", + "tokio 1.25.0", "tokio-io-timeout", "tokio-stream", "tokio-tar", @@ -1152,14 +1272,14 @@ dependencies = [ "axum 0.6.4", "deltachat", "env_logger 0.10.0", - "futures", + "futures 0.3.26", "log", "num-traits", "sanitize-filename", "serde", "serde_json", "tempfile", - "tokio", + "tokio 1.25.0", "typescript-type-def", "walkdir", "yerpc", @@ -1176,7 +1296,7 @@ dependencies = [ "log", "serde", "serde_json", - "tokio", + "tokio 1.25.0", "yerpc", ] @@ -1202,7 +1322,7 @@ dependencies = [ "rand 0.7.3", "serde_json", "thiserror", - "tokio", + "tokio 1.25.0", ] [[package]] @@ -1319,7 +1439,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -1331,7 +1451,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1342,7 +1462,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1523,7 +1643,7 @@ version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1584,7 +1704,7 @@ checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1643,7 +1763,7 @@ dependencies = [ "anyhow", "log", "thiserror", - "tokio", + "tokio 1.25.0", "tokio-stream", ] @@ -1671,7 +1791,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "rustix 0.35.7", "windows-sys 0.36.1", ] @@ -1692,9 +1812,9 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "windows-sys 0.36.1", ] @@ -1742,6 +1862,28 @@ dependencies = [ name = "format-flowed" version = "1.0.0" +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.26" @@ -1862,7 +2004,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1873,7 +2015,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] @@ -1911,7 +2053,7 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ - "bytes", + "bytes 1.2.1", "fnv", "futures-core", "futures-sink", @@ -1919,7 +2061,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio", + "tokio 1.25.0", "tokio-util", "tracing", ] @@ -1930,12 +2072,6 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" -[[package]] -name = "hash_hasher" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c" - [[package]] name = "hashbrown" version = "0.11.2" @@ -1993,12 +2129,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hex-literal" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" - [[package]] name = "hmac" version = "0.12.1" @@ -2016,7 +2146,7 @@ checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ "libc", "match_cfg", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2025,7 +2155,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ - "bytes", + "bytes 1.2.1", "fnv", "itoa", ] @@ -2036,7 +2166,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.2.1", "http", "pin-project-lite", ] @@ -2104,7 +2234,7 @@ version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ - "bytes", + "bytes 1.2.1", "futures-channel", "futures-core", "futures-util", @@ -2116,7 +2246,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio", + "tokio 1.25.0", "tower-service", "tracing", "want", @@ -2128,10 +2258,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes", + "bytes 1.2.1", "hyper", "native-tls", - "tokio", + "tokio 1.25.0", "tokio-native-tls", ] @@ -2145,7 +2275,7 @@ dependencies = [ "core-foundation-sys", "js-sys", "wasm-bindgen", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2219,7 +2349,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "tokio", + "tokio 1.25.0", "unicode-width", ] @@ -2238,16 +2368,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", -] - -[[package]] -name = "intrusive-collections" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4f90afb01281fdeffb0f8e082d230cbe4f888f837cc90759696b858db1a700" -dependencies = [ - "memoffset 0.8.0", + "cfg-if 1.0.0", ] [[package]] @@ -2266,6 +2387,15 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "ipconfig" version = "0.3.0" @@ -2274,7 +2404,7 @@ checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" dependencies = [ "socket2", "widestring", - "winapi", + "winapi 0.3.9", "winreg 0.7.0", ] @@ -2341,6 +2471,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9b7d56ba4a8344d6be9729995e6b06f928af29998cdf79fe390cbf6b1fee838" +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2416,6 +2556,15 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.7" @@ -2432,7 +2581,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -2488,6 +2637,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md-5" version = "0.10.5" @@ -2505,18 +2660,18 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.6.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ "autocfg", ] [[package]] name = "memoffset" -version = "0.8.0" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] @@ -2551,6 +2706,25 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + [[package]] name = "mio" version = "0.8.5" @@ -2563,6 +2737,40 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + [[package]] name = "mutate_once" version = "0.1.1" @@ -2587,13 +2795,24 @@ dependencies = [ "tempfile", ] +[[package]] +name = "net2" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d0df99cfcd2530b2e694f6e17e7f37b8e26bb23983ac530c0c97408837c631" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + [[package]] name = "nibble_vec" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" dependencies = [ - "smallvec", + "smallvec 1.10.0", ] [[package]] @@ -2603,7 +2822,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "libc", ] @@ -2636,6 +2855,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ntapi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -2661,7 +2889,7 @@ dependencies = [ "num-traits", "rand 0.8.5", "serde", - "smallvec", + "smallvec 1.10.0", "zeroize", ] @@ -2777,7 +3005,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -2891,14 +3119,40 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.3", + "rustc_version", +] + [[package]] name = "parking_lot" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.7", + "parking_lot_core 0.9.3", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version", + "smallvec 0.6.14", + "winapi 0.3.9", ] [[package]] @@ -2907,10 +3161,10 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", - "redox_syscall", - "smallvec", + "redox_syscall 0.2.16", + "smallvec 1.10.0", "windows-sys 0.36.1", ] @@ -2976,7 +3230,7 @@ dependencies = [ "sha2 0.10.6", "sha3", "signature", - "smallvec", + "smallvec 1.10.0", "thiserror", "twofish", "x25519-dalek", @@ -3205,6 +3459,56 @@ dependencies = [ "memchr", ] +[[package]] +name = "quinn" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445cbfe2382fa023c4f2f3c7e1c95c03dcc1df2bf23cebcb2b13e1402c4394d1" +dependencies = [ + "bytes 1.2.1", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "thiserror", + "tokio 1.25.0", + "tracing", + "webpki", +] + +[[package]] +name = "quinn-proto" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4ced82a24bb281af338b9e8f94429b6eca01b4e66d899f40031f074e74c9" +dependencies = [ + "bytes 1.2.1", + "rand 0.8.5", + "ring", + "rustc-hash", + "rustls", + "rustls-native-certs", + "slab", + "thiserror", + "tinyvec", + "tracing", + "webpki", +] + +[[package]] +name = "quinn-udp" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "641538578b21f5e5c8ea733b736895576d0fe329bb883b937db6f4d163dbaaf4" +dependencies = [ + "libc", + "quinn-proto", + "socket2", + "tracing", + "windows-sys 0.42.0", +] + [[package]] name = "quote" version = "1.0.23" @@ -3227,7 +3531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot", + "parking_lot 0.12.1", "scheduled-thread-pool", ] @@ -3342,7 +3646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg", - "crossbeam-deque", + "crossbeam-deque 0.8.2", "either", "rayon-core", ] @@ -3354,8 +3658,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", + "crossbeam-deque 0.8.2", + "crossbeam-utils 0.8.11", "num_cpus", ] @@ -3371,6 +3675,12 @@ dependencies = [ "yasna", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -3387,7 +3697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom 0.2.7", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] @@ -3423,7 +3733,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3433,7 +3743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ "base64 0.21.0", - "bytes", + "bytes 1.2.1", "encoding_rs", "futures-core", "futures-util", @@ -3453,7 +3763,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio", + "tokio 1.25.0", "tokio-native-tls", "tower-service", "url", @@ -3496,7 +3806,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3524,7 +3834,7 @@ dependencies = [ "pkcs8", "rand_core 0.6.3", "signature", - "smallvec", + "smallvec 1.10.0", "subtle", "zeroize", ] @@ -3541,7 +3851,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "memchr", - "smallvec", + "smallvec 1.10.0", ] [[package]] @@ -3556,6 +3866,21 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -3599,12 +3924,23 @@ version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ - "log", "ring", "sct", "webpki", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -3627,7 +3963,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1cd5ae51d3f7bf65d7969d579d502168ef578f289452bd8ccc91de28fda20e" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "clipboard-win", "dirs-next", "fd-lock", @@ -3640,7 +3976,7 @@ dependencies = [ "unicode-segmentation", "unicode-width", "utf8parse", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3649,125 +3985,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" -[[package]] -name = "s2n-codec" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c76a52949171a784415293428d2c9315c249ba2091d49b27279601a55375bbb2" -dependencies = [ - "byteorder", - "bytes", - "zerocopy", - "zerocopy-derive", -] - -[[package]] -name = "s2n-quic" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501b93c392899600caafd6f7b19b43a181b6afcb05c2ba0a61e65589469fd863" -dependencies = [ - "bytes", - "cfg-if", - "cuckoofilter", - "futures", - "hash_hasher", - "rand 0.8.5", - "rand_chacha 0.3.1", - "ring", - "s2n-codec", - "s2n-quic-core", - "s2n-quic-platform", - "s2n-quic-rustls", - "s2n-quic-transport", - "tokio", - "zerocopy", - "zerocopy-derive", - "zeroize", -] - -[[package]] -name = "s2n-quic-core" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ea949a1e82375bc7d05ac76372fd95992e8db42485c85228b3ce421d1a9af" -dependencies = [ - "byteorder", - "bytes", - "hex-literal", - "num-rational", - "num-traits", - "once_cell", - "s2n-codec", - "subtle", - "zerocopy", - "zerocopy-derive", -] - -[[package]] -name = "s2n-quic-crypto" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2a457eebfdd37f792fdfc9283893d90767cbb6fff623cbc7ec12531d29e1a7" -dependencies = [ - "cfg-if", - "lazy_static", - "ring", - "s2n-codec", - "s2n-quic-core", - "zeroize", -] - -[[package]] -name = "s2n-quic-platform" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8926e4b76c64a06ddb7e413d18d045c619f0daef8b93bcb04dd8f322ed293755" -dependencies = [ - "cfg-if", - "errno", - "futures", - "lazy_static", - "libc", - "pin-project", - "s2n-quic-core", - "socket2", - "tokio", - "zeroize", -] - -[[package]] -name = "s2n-quic-rustls" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec8decad668229de67f0b90e98ffbba8f844dc294084af3d7e86bfee4f6f040" -dependencies = [ - "bytes", - "rustls", - "rustls-pemfile", - "s2n-codec", - "s2n-quic-core", - "s2n-quic-crypto", -] - -[[package]] -name = "s2n-quic-transport" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b0506c95a703cf12dc21352275f699f9508485c89b257cf99c2b3c2f1cc25c9" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "hashbrown 0.12.3", - "intrusive-collections", - "once_cell", - "s2n-codec", - "s2n-quic-core", - "siphasher", - "smallvec", -] - [[package]] name = "safemem" version = "0.3.3" @@ -3809,7 +4026,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" dependencies = [ - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -3865,6 +4082,30 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +dependencies = [ + "serde", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "sendme" version = "0.1.0" @@ -3873,26 +4114,29 @@ dependencies = [ "bao", "base64 0.21.0", "blake3", - "bytes", + "bytes 1.2.1", "clap 4.1.4", "console", + "data-encoding", "der", "ed25519-dalek", - "futures", + "futures 0.3.26", "indicatif", + "multibase", "postcard", + "quinn", "rand 0.7.3", "rcgen", "ring", "rustls", - "s2n-quic", "serde", "ssh-key", "tempfile", "thiserror", - "tokio", + "tokio 1.25.0", "tokio-util", "tracing", + "tracing-futures", "tracing-subscriber", "webpki", "x509-parser", @@ -3975,7 +4219,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.5", ] @@ -3986,7 +4230,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.5", ] @@ -3998,7 +4242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -4010,7 +4254,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.5", ] @@ -4053,12 +4297,6 @@ dependencies = [ "rand_core 0.6.3", ] -[[package]] -name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - [[package]] name = "slab" version = "0.4.7" @@ -4068,6 +4306,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -4087,7 +4334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4148,7 +4395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af91f480ee899ab2d9f8435bfdfc14d08a5754bd9d3fef1f1a1c23336aad6c8b" dependencies = [ "async-channel", - "cfg-if", + "cfg-if 1.0.0", "futures-core", "pin-project-lite", ] @@ -4219,6 +4466,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sysinfo" +version = "0.26.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi 0.3.9", +] + [[package]] name = "tagger" version = "4.3.4" @@ -4231,12 +4492,12 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "remove_dir_all", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4248,6 +4509,20 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "testdir" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23029d5d16b0351859485c6849252f00bf0ebc98098a9efd954853c3533720c7" +dependencies = [ + "anyhow", + "backtrace", + "cargo_metadata", + "once_cell", + "sysinfo", + "whoami", +] + [[package]] name = "textwrap" version = "0.16.0" @@ -4296,7 +4571,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4351,6 +4626,30 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + [[package]] name = "tokio" version = "1.25.0" @@ -4358,12 +4657,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", - "bytes", + "bytes 1.2.1", "libc", "memchr", - "mio", + "mio 0.8.5", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", @@ -4371,6 +4670,59 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-io-timeout" version = "1.2.0" @@ -4378,7 +4730,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite", - "tokio", + "tokio 1.25.0", ] [[package]] @@ -4399,7 +4751,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio", + "tokio 1.25.0", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", ] [[package]] @@ -4410,7 +4781,17 @@ checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite", - "tokio", + "tokio 1.25.0", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", ] [[package]] @@ -4422,12 +4803,55 @@ dependencies = [ "filetime", "futures-core", "libc", - "redox_syscall", - "tokio", + "redox_syscall 0.2.16", + "tokio 1.25.0", "tokio-stream", "xattr", ] +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque 0.7.4", + "crossbeam-queue 0.2.3", + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "slab", + "tokio-executor", +] + [[package]] name = "tokio-tungstenite" version = "0.17.2" @@ -4436,7 +4860,7 @@ checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ "futures-util", "log", - "tokio", + "tokio 1.25.0", "tungstenite 0.17.3", ] @@ -4448,21 +4872,54 @@ checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", - "tokio", + "tokio 1.25.0", "tungstenite 0.18.0", ] +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + [[package]] name = "tokio-util" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ - "bytes", + "bytes 1.2.1", "futures-core", "futures-sink", "pin-project-lite", - "tokio", + "tokio 1.25.0", "tracing", ] @@ -4519,7 +4976,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio", + "tokio 1.25.0", "tower-layer", "tower-service", "tracing", @@ -4532,7 +4989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" dependencies = [ "bitflags", - "bytes", + "bytes 1.2.1", "futures-core", "futures-util", "http", @@ -4562,7 +5019,7 @@ version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -4590,6 +5047,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tokio 0.1.22", + "tracing", +] + [[package]] name = "tracing-log" version = "0.1.3" @@ -4612,7 +5080,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec", + "smallvec 1.10.0", "thread_local", "tracing", "tracing-core", @@ -4626,7 +5094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", - "cfg-if", + "cfg-if 1.0.0", "data-encoding", "enum-as-inner", "futures-channel", @@ -4636,10 +5104,10 @@ dependencies = [ "ipnet", "lazy_static", "rand 0.8.5", - "smallvec", + "smallvec 1.10.0", "thiserror", "tinyvec", - "tokio", + "tokio 1.25.0", "tracing", "url", ] @@ -4650,16 +5118,16 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "futures-util", "ipconfig", "lazy_static", "lru-cache", - "parking_lot", + "parking_lot 0.12.1", "resolv-conf", - "smallvec", + "smallvec 1.10.0", "thiserror", - "tokio", + "tokio 1.25.0", "tracing", "trust-dns-proto", ] @@ -4678,7 +5146,7 @@ checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" dependencies = [ "base64 0.13.1", "byteorder", - "bytes", + "bytes 1.2.1", "http", "httparse", "log", @@ -4697,7 +5165,7 @@ checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", - "bytes", + "bytes 1.2.1", "http", "httparse", "log", @@ -4881,7 +5349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", - "winapi", + "winapi 0.3.9", "winapi-util", ] @@ -4919,7 +5387,7 @@ version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -4944,7 +5412,7 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -5005,12 +5473,28 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" +[[package]] +name = "whoami" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45dbc71f0cdca27dc261a9bd37ddec174e4a0af2b900b890f378460f745426e3" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "widestring" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -5021,6 +5505,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -5033,7 +5523,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5148,7 +5638,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5157,7 +5647,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", ] [[package]] @@ -5218,12 +5718,12 @@ dependencies = [ "async-mutex", "async-trait", "axum 0.5.17", - "futures", + "futures 0.3.26", "futures-util", "log", "serde", "serde_json", - "tokio", + "tokio 1.25.0", "tracing", "typescript-type-def", "yerpc_derive", @@ -5242,27 +5742,6 @@ dependencies = [ "syn", ] -[[package]] -name = "zerocopy" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0f717764196a220d8c58500e3a3595e2c9054f95d66267f9fd5f6e74ad0fec" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0af017aca1fa6181f5dd7a802456fe6f7666ecdcc18d0910431f0fc89d474e51" -dependencies = [ - "proc-macro2", - "syn", - "synstructure", -] - [[package]] name = "zeroize" version = "1.5.7" diff --git a/Cargo.toml b/Cargo.toml index 06909ee96..5530dcfd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,6 +101,7 @@ log = "0.4" pretty_env_logger = "0.4" proptest = { version = "1", default-features = false, features = ["std"] } tempfile = "3" +testdir = "0.7.1" tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] } [workspace] diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index aaeb952d2..99031879a 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -72,7 +72,7 @@ impl BackupProvider { /// This will acquire the global "ongoing process" mutex. You must call /// [`BackupSender::join`] after creating this struct, otherwise this will not respect /// the possible cancellation of the "ongoing process". - pub async fn perpare(context: &Context, dir: &Path) -> Result { + pub async fn prepare(context: &Context, dir: &Path) -> Result { ensure!( // TODO: Should we worry about path normalisation? dir != context.get_blobdir(), @@ -109,7 +109,11 @@ impl BackupProvider { return Err(err); } }; - let handle = tokio::spawn(Self::watch_provider(provider, cancel_token)); + let handle = tokio::spawn(Self::watch_provider( + context.clone(), + provider, + cancel_token, + )); Ok(Self { handle, ticket }) } @@ -147,24 +151,28 @@ impl BackupProvider { /// /// The *cancel_token* is the handle for the ongoing process mutex, when this completes /// we must cancel this operation. - async fn watch_provider(mut provider: Provider, cancel_token: Receiver<()>) -> Result<()> { + async fn watch_provider( + context: Context, + mut provider: Provider, + cancel_token: Receiver<()>, + ) -> Result<()> { let mut events = provider.subscribe(); - loop { + let res = loop { tokio::select! { biased; res = &mut provider => { - return res.context("BackupSender failed"); + break res.context("BackupSender failed"); }, maybe_event = events.recv() => { match maybe_event { Ok(event) => { match event { Event::TransferCompleted { .. } => { - provider.abort(); + provider.shutdown(); } Event::TransferAborted { .. } => { - provider.abort(); - return Err(anyhow!("BackupSender transfer aborted")); + provider.shutdown(); + break Err(anyhow!("BackupSender transfer aborted")); } _ => (), } @@ -176,17 +184,19 @@ impl BackupProvider { Err(broadcast::error::RecvError::Lagged(_)) => { // We really shouldn't be lagging, if we did we may have missed // a completion event. - provider.abort(); - return Err(anyhow!("Missed events from BackupSender")); + provider.shutdown(); + break Err(anyhow!("Missed events from BackupSender")); } } }, _ = cancel_token.recv() => { - provider.abort(); - return Err(anyhow!("BackupSender cancelled")); + provider.shutdown(); + break Err(anyhow!("BackupSender cancelled")); }, } - } + }; + context.free_ongoing().await; + res } pub fn qr(&self) -> Qr { @@ -194,6 +204,15 @@ impl BackupProvider { ticket: self.ticket.clone(), } } + + /// Awaits the [`BackupSender`] until it is finished. + /// + /// This waits until someone connected to the sender and transferred a backup. If the + /// [`BackupSender`] task results in an error it will be returned here. + pub async fn join(self) -> Result<()> { + self.handle.await??; + Ok(()) + } } /// Contacts a backup provider and receives the backup from it. @@ -299,3 +318,62 @@ async fn on_blob( } Ok(reader) } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use testdir::testdir; + + use crate::chat::{get_chat_msgs, send_msg, ChatItem}; + use crate::message::{Message, Viewtype}; + use crate::test_utils::TestContextManager; + + use super::*; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_send_receive() { + let dir = testdir!(); + let mut tcm = TestContextManager::new(); + + // Create first device. + let ctx0 = tcm.alice().await; + + // Write a message in the self chat + let self_chat = ctx0.get_self_chat().await; + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("hi there".to_string())); + send_msg(&ctx0, self_chat.id, &mut msg).await.unwrap(); + + // Prepare to transfer backup. + ctx0.stop_io().await; + let provider = BackupProvider::prepare(&ctx0, &dir).await.unwrap(); + + // Set up second device. + let ctx1 = tcm.bob().await; + ctx1.stop_io().await; + ctx1.sql + .set_raw_config_bool("configured", false) + .await + .unwrap(); + get_backup(&ctx1, provider.qr()).await.unwrap(); + + // Make sure the provider finishes without an error. + tokio::time::timeout(Duration::from_secs(30), provider.join()) + .await + .expect("timed out") + .expect("error in provider"); + + // Check that we have the self message. + let self_chat = ctx1.get_self_chat().await; + let msgs = get_chat_msgs(&ctx1, self_chat.id, 0).await.unwrap(); + assert_eq!(msgs.len(), 1); + let msgid = match msgs.get(0).unwrap() { + ChatItem::Message { msg_id } => msg_id, + _ => panic!("wrong chat item"), + }; + let msg = Message::load_from_db(&ctx1, *msgid).await.unwrap(); + let text = msg.get_text().unwrap(); + assert_eq!(text, "hi there"); + } +} From 716504b833e485ebc436b3fa2845cf3f1d1bbe14 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Feb 2023 17:20:35 +0100 Subject: [PATCH 010/103] do not pull in sendme cli deps --- Cargo.lock | 127 +---------------------------------------------------- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 126 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d504893e..95dfe2b8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -418,12 +418,6 @@ dependencies = [ "blake3", ] -[[package]] -name = "base-x" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" - [[package]] name = "base16ct" version = "0.1.1" @@ -731,39 +725,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "bitflags", - "clap_lex 0.2.4", + "clap_lex", "indexmap", "textwrap", ] -[[package]] -name = "clap" -version = "4.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" -dependencies = [ - "bitflags", - "clap_derive", - "clap_lex 0.3.1", - "is-terminal", - "once_cell", - "strsim", - "termcolor", -] - -[[package]] -name = "clap_derive" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" -dependencies = [ - "heck", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "clap_lex" version = "0.2.4" @@ -773,15 +739,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "clap_lex" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "clipboard-win" version = "4.4.2" @@ -823,19 +780,6 @@ dependencies = [ "crossbeam-utils 0.8.11", ] -[[package]] -name = "console" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.42.0", -] - [[package]] name = "const-oid" version = "0.9.0" @@ -924,7 +868,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap", "criterion-plot", "futures 0.3.26", "itertools", @@ -1166,26 +1110,6 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" -[[package]] -name = "data-encoding-macro" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" -dependencies = [ - "data-encoding", - "data-encoding-macro-internal", -] - -[[package]] -name = "data-encoding-macro-internal" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" -dependencies = [ - "data-encoding", - "syn", -] - [[package]] name = "deltachat" version = "1.107.0" @@ -1553,12 +1477,6 @@ dependencies = [ "version_check 0.9.4", ] -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encoded-words" version = "0.2.0" @@ -2340,19 +2258,6 @@ dependencies = [ "hashbrown 0.12.3", ] -[[package]] -name = "indicatif" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" -dependencies = [ - "console", - "number_prefix", - "portable-atomic", - "tokio 1.25.0", - "unicode-width", -] - [[package]] name = "inout" version = "0.1.3" @@ -2760,17 +2665,6 @@ dependencies = [ "ws2_32-sys", ] -[[package]] -name = "multibase" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" -dependencies = [ - "base-x", - "data-encoding", - "data-encoding-macro", -] - [[package]] name = "mutate_once" version = "0.1.1" @@ -2956,12 +2850,6 @@ dependencies = [ "libc", ] -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - [[package]] name = "object" version = "0.30.0" @@ -3337,12 +3225,6 @@ dependencies = [ "miniz_oxide 0.6.2", ] -[[package]] -name = "portable-atomic" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" - [[package]] name = "postcard" version = "1.0.2" @@ -4115,14 +3997,9 @@ dependencies = [ "base64 0.21.0", "blake3", "bytes 1.2.1", - "clap 4.1.4", - "console", - "data-encoding", "der", "ed25519-dalek", "futures 0.3.26", - "indicatif", - "multibase", "postcard", "quinn", "rand 0.7.3", diff --git a/Cargo.toml b/Cargo.toml index 5530dcfd2..2b5c53991 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,7 @@ rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" rustyline = { version = "10", optional = true } sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "main" } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha-1 = "0.10" From 05ddc130548056c12b5082058016e4e027cdd82a Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Feb 2023 18:21:46 +0100 Subject: [PATCH 011/103] Use name prefixes so the db can not be spoofed by a blob --- src/imex/transfer.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 99031879a..2fe17a4e7 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -129,10 +129,15 @@ impl BackupProvider { .context("Database export failed")?; // Now we can be sure IO is not running. - let mut files = vec![DataSource::from(dbfile)]; + let mut files = vec![DataSource::with_name( + dbfile, + format!("db/{DBFILE_BACKUP_NAME}"), + )]; let blobdir = BlobDirContents::new(context).await?; for blob in blobdir.iter() { - files.push(blob.to_abs_path().into()); + let path = blob.to_abs_path(); + let name = format!("blob/P{}", blob.as_file_name()); + files.push(DataSource::with_name(path, name)); } // Start listening. @@ -290,15 +295,19 @@ async fn on_blob( name: String, ) -> Result { ensure!(!name.is_empty(), "Received a nameless blob"); - let path = if name == DBFILE_BACKUP_NAME { + let path = if name.starts_with("db/") { // We can only safely write to the blobdir. But the blobdir could have a file named // exactly like our special name. We solve this by using an uppercase extension // which is forbidden for normal blobs. - context.get_blobdir().join(format!("{name}.SPECIAL")) + context + .get_blobdir() + .join(format!("{DBFILE_BACKUP_NAME}.SPECIAL")) } else { - context.get_blobdir().join(&name) + ensure!(name.starts_with("blob/"), "malformatted blob name"); + let blobname = name.rsplit('/').next().context("malformatted blob name")?; + context.get_blobdir().join(blobname) }; - let _guard = if name == DBFILE_BACKUP_NAME { + let _guard = if name.starts_with("db/") { Some(DeleteOnDrop(path.clone())) } else { None @@ -306,7 +315,7 @@ async fn on_blob( let file = File::create(&path).await?; let mut file = BufWriter::with_capacity(128 * 1024, file); io::copy(&mut reader, &mut file).await?; - if name == DBFILE_BACKUP_NAME { + if name.starts_with("db/") { context .sql .import(&path, ticket.token.to_string()) From 3734fc25a72dd0965d20748a15235e8964f5e461 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 9 Feb 2023 10:02:18 +0100 Subject: [PATCH 012/103] update callback to take collection by ref --- src/imex/transfer.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 2fe17a4e7..a09d51c0b 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -31,7 +31,6 @@ use crate::qr::Qr; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures_lite::StreamExt; -use sendme::blobs::Collection; use sendme::get::{DataStream, Options}; use sendme::protocol::AuthToken; use sendme::provider::{DataSource, Event, Provider, Ticket}; @@ -251,7 +250,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { ticket.token, opts, on_connected, - on_collection, + |_collection| async { Ok(()) }, on_blob, ) .await @@ -280,12 +279,6 @@ async fn on_connected() -> Result<()> { Ok(()) } -/// Get callback when a collection is received from the provider. -#[allow(clippy::unused_async)] -async fn on_collection(_collection: Collection) -> Result<()> { - Ok(()) -} - /// Get callback when a blob is received from the provider. async fn on_blob( context: &Context, From 2129b2b7a0efd3fe422bcce79103b215c4c8f42c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 9 Feb 2023 18:09:16 +0100 Subject: [PATCH 013/103] Add a ton of code for receiver-side progress --- src/imex/transfer.rs | 194 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 180 insertions(+), 14 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index a09d51c0b..5e586354d 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -23,11 +23,15 @@ //! download to an impersonated getter. use std::path::Path; +use std::pin::Pin; +use std::sync::atomic::{AtomicU16, AtomicU64, Ordering}; +use std::sync::Arc; +use std::task::Poll; use crate::chat::delete_and_reset_all_device_msgs; use crate::context::Context; -use crate::e2ee; use crate::qr::Qr; +use crate::{e2ee, EventType}; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures_lite::StreamExt; @@ -36,8 +40,9 @@ use sendme::protocol::AuthToken; use sendme::provider::{DataSource, Event, Provider, Ticket}; use sendme::Hash; use tokio::fs::{self, File}; -use tokio::io::{self, BufWriter}; +use tokio::io::{self, AsyncRead, AsyncWriteExt, BufWriter}; use tokio::sync::broadcast; +use tokio::sync::broadcast::error::RecvError; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReadDirStream; @@ -244,19 +249,31 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { addr: ticket.addr, peer_id: Some(ticket.peer), }; - let on_blob = |hash, reader, name| on_blob(context, &ticket, hash, reader, name); - match sendme::get::run( + let progress = ProgressEmitter::new(0, 85); + spawn_progress_proxy(context.clone(), progress.subscribe()); + let on_connected = || { + context.emit_event(ReceiveProgress::Connected.into()); + async { Ok(()) } + }; + let on_blob = |hash, reader, name| on_blob(context, &progress, &ticket, hash, reader, name); + let res = sendme::get::run( ticket.hash, ticket.token, opts, on_connected, - |_collection| async { Ok(()) }, + |collection| { + context.emit_event(ReceiveProgress::CollectionRecieved.into()); + progress.set_total(collection.total_blobs_size()); + async { Ok(()) } + }, on_blob, ) - .await - { + .await; + drop(progress); + match res { Ok(_) => { delete_and_reset_all_device_msgs(context).await?; + context.emit_event(ReceiveProgress::Completed.into()); Ok(()) } Err(err) => { @@ -268,20 +285,16 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { fs::remove_file(dirent.path()).await.ok(); } } + context.emit_event(ReceiveProgress::Failed.into()); Err(err) } } } -/// Get callback when the connection is established with the provider. -#[allow(clippy::unused_async)] -async fn on_connected() -> Result<()> { - Ok(()) -} - /// Get callback when a blob is received from the provider. async fn on_blob( context: &Context, + progress: &ProgressEmitter, ticket: &Ticket, _hash: Hash, mut reader: DataStream, @@ -305,9 +318,13 @@ async fn on_blob( } else { None }; + + let mut wrapped_reader = progress.wrap_async_read(&mut reader); let file = File::create(&path).await?; let mut file = BufWriter::with_capacity(128 * 1024, file); - io::copy(&mut reader, &mut file).await?; + io::copy(&mut wrapped_reader, &mut file).await?; + file.flush().await?; + if name.starts_with("db/") { context .sql @@ -321,6 +338,155 @@ async fn on_blob( Ok(reader) } +/// Spawns a task proxying progress events. +/// +/// This spawns a tokio tasks which receives events from the [`ProgressEmitter`] and sends +/// them to the context. The task finishes when the emitter is dropped. +/// +/// This could be done directly in the emitter by making it less generic. +fn spawn_progress_proxy(context: Context, mut rx: broadcast::Receiver) { + tokio::spawn(async move { + loop { + match rx.recv().await { + Ok(step) => context.emit_event(ReceiveProgress::BlobProgress(step).into()), + Err(RecvError::Closed) => break, + Err(RecvError::Lagged(_)) => continue, + } + } + }); +} + +/// Create [`EventType::ImexProgress`] events using readable names. +/// +/// Plus you get warnings if you don't use all variants. +enum ReceiveProgress { + Connected, + CollectionRecieved, + /// A value between 0 and 85 as percentage + BlobProgress(u16), + Completed, + Failed, +} + +impl From for EventType { + fn from(source: ReceiveProgress) -> Self { + let val = match source { + ReceiveProgress::Connected => 50, + ReceiveProgress::CollectionRecieved => 100, + ReceiveProgress::BlobProgress(val) => 100 + 10 * val, + ReceiveProgress::Completed => 1000, + ReceiveProgress::Failed => 0, + }; + EventType::ImexProgress(val.into()) + } +} + +#[derive(Debug, Clone)] +struct ProgressEmitter { + inner: Arc, +} + +impl ProgressEmitter { + /// Creates a new emitter. + /// + /// The emitter expects to see *total* being added via [`ProgressEmitter::inc`] and will + /// emit *steps* updates. + fn new(total: u64, steps: u16) -> Self { + let (tx, _rx) = broadcast::channel(16); + Self { + inner: Arc::new(InnerProgressEmitter { + total: AtomicU64::new(total), + count: AtomicU64::new(0), + steps, + last_step: AtomicU16::new(0u16), + tx, + }), + } + } + + /// Sets a new total in case you did not now the total up front. + fn set_total(&self, value: u64) { + self.inner.set_total(value) + } + + /// Return a receiver that gets incremental values. + /// + /// The values yielded depend on *steps* passed to [`ProgressEmitter::new`]: it will go + /// from `1..steps`. + fn subscribe(&self) -> broadcast::Receiver { + self.inner.subscribe() + } + + /// Increments the progress by *amount*. + fn inc(&self, amount: u64) { + self.inner.inc(amount); + } + + fn wrap_async_read(&self, read: R) -> ProgressAsyncReader { + ProgressAsyncReader { + emitter: self.clone(), + inner: read, + } + } +} + +#[derive(Debug)] +struct InnerProgressEmitter { + total: AtomicU64, + count: AtomicU64, + steps: u16, + last_step: AtomicU16, + tx: broadcast::Sender, +} + +impl InnerProgressEmitter { + fn inc(&self, amount: u64) { + let prev_count = self.count.fetch_add(amount, Ordering::Relaxed); + let count = prev_count + amount; + let total = self.total.load(Ordering::Relaxed); + let step = (total * u64::from(self.steps) / std::cmp::min(count, total)) as u16; + let last_step = self.last_step.swap(step, Ordering::Relaxed); + if step > last_step { + self.tx.send(step).ok(); + } + } + + fn set_total(&self, value: u64) { + self.total.store(value, Ordering::Relaxed); + } + + fn subscribe(&self) -> broadcast::Receiver { + self.tx.subscribe() + } +} + +#[derive(Debug)] +struct ProgressAsyncReader { + emitter: ProgressEmitter, + inner: R, +} + +impl AsyncRead for ProgressAsyncReader +where + R: AsyncRead + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut io::ReadBuf<'_>, + ) -> Poll> { + let prev_len = buf.filled().len() as u64; + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Ready(val) => { + let new_len = buf.filled().len() as u64; + self.emitter.inc(new_len - prev_len); + Poll::Ready(val) + } + Poll::Pending => Poll::Pending, + } + } +} + #[cfg(test)] mod tests { use std::time::Duration; From 3f51a8ffc2f232532706336f374b946cdbe9d12c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Fri, 10 Feb 2023 10:48:10 +0100 Subject: [PATCH 014/103] Some more doc comments --- src/imex/transfer.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 5e586354d..0e68a3969 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -362,7 +362,9 @@ fn spawn_progress_proxy(context: Context, mut rx: broadcast::Receiver) { enum ReceiveProgress { Connected, CollectionRecieved, - /// A value between 0 and 85 as percentage + /// A value between 0 and 85 interpreted as a percentage. + /// + /// Other values are already used by the other variants of this enum. BlobProgress(u16), Completed, Failed, @@ -381,6 +383,15 @@ impl From for EventType { } } +/// A generic progress event emitter. +/// +/// It is created with a total value to reach and at which increments progress should be +/// emitted. E.g. when downloading a file of any size but you want percentage increments +/// you would create `ProgressEmitter::new(file_size_in_bytes, 100)` and +/// [`ProgressEmitter::subscribe`] will yield numbers `1..100` only. +/// +/// Progress is made by calling [`ProgressEmitter::inc`], which can be implicitly done by +/// [`ProgressEmitter::wrap_async_read`]. #[derive(Debug, Clone)] struct ProgressEmitter { inner: Arc, @@ -409,7 +420,7 @@ impl ProgressEmitter { self.inner.set_total(value) } - /// Return a receiver that gets incremental values. + /// Returns a receiver that gets incremental values. /// /// The values yielded depend on *steps* passed to [`ProgressEmitter::new`]: it will go /// from `1..steps`. @@ -422,6 +433,7 @@ impl ProgressEmitter { self.inner.inc(amount); } + /// Wraps an [`AsyncRead`] which implicitly calls [`ProgressEmitter::inc`]. fn wrap_async_read(&self, read: R) -> ProgressAsyncReader { ProgressAsyncReader { emitter: self.clone(), @@ -460,6 +472,10 @@ impl InnerProgressEmitter { } } +/// A wrapper around [`AsyncRead`] which increments a [`ProgressEmitter`]. +/// +/// This can be used just like the underlying [`AsyncRead`] but increments progress for each +/// byte read. Create this using [`ProgressEmitter::wrap_async_read`]. #[derive(Debug)] struct ProgressAsyncReader { emitter: ProgressEmitter, From ac048c154d3ce4cd5a208f798208485988ec20e4 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Fri, 10 Feb 2023 13:54:50 +0100 Subject: [PATCH 015/103] Add progress for provider Fix progress for getter. Maths. It's hard. Add test for progress. --- src/imex/transfer.rs | 57 +++++++++++++++++++++++++++++++++++++------- src/test_utils.rs | 5 ++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 0e68a3969..602eeca0c 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -127,10 +127,12 @@ impl BackupProvider { async fn prepare_inner(context: &Context, dir: &Path) -> Result<(Provider, Ticket)> { // Generate the token up front: we also use it to encrypt the database. let token = AuthToken::generate(); + context.emit_event(SendProgress::Started.into()); let dbfile = dir.join(DBFILE_BACKUP_NAME); export_database(context, &dbfile, token.to_string()) .await .context("Database export failed")?; + context.emit_event(SendProgress::DatabaseExported.into()); // Now we can be sure IO is not running. let mut files = vec![DataSource::with_name( @@ -146,7 +148,9 @@ impl BackupProvider { // Start listening. let (db, hash) = sendme::provider::create_collection(files).await?; + context.emit_event(SendProgress::CollectionCreated.into()); let provider = Provider::builder(db).auth_token(token).spawn()?; + context.emit_event(SendProgress::ProviderListening.into()); let ticket = provider.ticket(hash); Ok((provider, ticket)) } @@ -165,6 +169,7 @@ impl BackupProvider { mut provider: Provider, cancel_token: Receiver<()>, ) -> Result<()> { + context.emit_event(SendProgress::ProviderListening.into()); let mut events = provider.subscribe(); let res = loop { tokio::select! { @@ -176,14 +181,21 @@ impl BackupProvider { match maybe_event { Ok(event) => { match event { + Event::ClientConnected { ..} => { + context.emit_event(SendProgress::ClientConnected.into()); + } + Event::RequestReceived { .. } => { + context.emit_event(SendProgress::TransferStarted.into()); + } Event::TransferCompleted { .. } => { + context.emit_event(SendProgress::TransferFinished.into()); provider.shutdown(); } Event::TransferAborted { .. } => { + context.emit_event(SendProgress::Failed.into()); provider.shutdown(); break Err(anyhow!("BackupSender transfer aborted")); } - _ => (), } } Err(broadcast::error::RecvError::Closed) => { @@ -204,6 +216,7 @@ impl BackupProvider { }, } }; + context.emit_event(SendProgress::Completed.into()); context.free_ongoing().await; res } @@ -224,6 +237,29 @@ impl BackupProvider { } } +/// Create [`EventType::ImexProgress`] events using readable names. +/// +/// Plus you get warnings if you don't use all variants. +#[derive(Debug)] +#[repr(u16)] +enum SendProgress { + Failed = 0, + Started = 100, + DatabaseExported = 300, + CollectionCreated = 400, + ProviderListening = 500, + ClientConnected = 600, + TransferStarted = 650, + TransferFinished = 950, + Completed = 1000, +} + +impl From for EventType { + fn from(source: SendProgress) -> Self { + Self::ImexProgress((source as u16).into()) + } +} + /// Contacts a backup provider and receives the backup from it. /// /// This uses a QR code to contact another instance of deltachat which is providing a backup @@ -359,6 +395,7 @@ fn spawn_progress_proxy(context: Context, mut rx: broadcast::Receiver) { /// Create [`EventType::ImexProgress`] events using readable names. /// /// Plus you get warnings if you don't use all variants. +#[derive(Debug)] enum ReceiveProgress { Connected, CollectionRecieved, @@ -456,7 +493,7 @@ impl InnerProgressEmitter { let prev_count = self.count.fetch_add(amount, Ordering::Relaxed); let count = prev_count + amount; let total = self.total.load(Ordering::Relaxed); - let step = (total * u64::from(self.steps) / std::cmp::min(count, total)) as u16; + let step = (std::cmp::min(count, total) * u64::from(self.steps) / total) as u16; let last_step = self.last_step.swap(step, Ordering::Relaxed); if step > last_step { self.tx.send(step).ok(); @@ -530,16 +567,10 @@ mod tests { send_msg(&ctx0, self_chat.id, &mut msg).await.unwrap(); // Prepare to transfer backup. - ctx0.stop_io().await; let provider = BackupProvider::prepare(&ctx0, &dir).await.unwrap(); // Set up second device. - let ctx1 = tcm.bob().await; - ctx1.stop_io().await; - ctx1.sql - .set_raw_config_bool("configured", false) - .await - .unwrap(); + let ctx1 = tcm.unconfigured().await; get_backup(&ctx1, provider.qr()).await.unwrap(); // Make sure the provider finishes without an error. @@ -559,5 +590,13 @@ mod tests { let msg = Message::load_from_db(&ctx1, *msgid).await.unwrap(); let text = msg.get_text().unwrap(); assert_eq!(text, "hi there"); + + // Check that both received the ImexProgress events. + ctx0.evtracker + .get_matching(|ev| matches!(ev, EventType::ImexProgress(1000))) + .await; + ctx1.evtracker + .get_matching(|ev| matches!(ev, EventType::ImexProgress(1000))) + .await; } } diff --git a/src/test_utils.rs b/src/test_utils.rs index 0d74ffd76..92ffd3baf 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -40,6 +40,11 @@ pub const AVATAR_900x900_BYTES: &[u8] = include_bytes!("../test-data/image/avata static CONTEXT_NAMES: Lazy>> = Lazy::new(|| std::sync::RwLock::new(BTreeMap::new())); +/// Manage multiple [`TestContext`]s in one place. +/// +/// The main advantage is that the log records of the contexts will appear in the order they +/// occurred rather than grouped by context like would happen when you use separate +/// [`TestContext`]s without managing your own [`LogSink`]. pub struct TestContextManager { log_tx: Sender, _log_sink: LogSink, From 6a3fe3db922829b267ab08fae5bef8e6612aea2c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Fri, 10 Feb 2023 14:14:36 +0100 Subject: [PATCH 016/103] fixup doc comments --- src/imex/transfer.rs | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 602eeca0c..df28e07fc 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -70,12 +70,14 @@ impl BackupProvider { /// Prepares for sending a backup to a second device. /// /// Before calling this function all I/O must be stopped so that no changes to the blobs - /// or database are happening, this is done by calling the `dc_accounts_stop_io` or - /// `dc_stop_io` APIs first. TODO: Add the rust equivalents. + /// or database are happening, this is done by calling the [`Accounts::stop_io`] or + /// [`Context::stop_io`] APIs first. /// /// This will acquire the global "ongoing process" mutex. You must call - /// [`BackupSender::join`] after creating this struct, otherwise this will not respect + /// [`BackupProvider::join`] after creating this struct, otherwise this will not respect /// the possible cancellation of the "ongoing process". + /// + /// [`Accounts::stop_io`]: crate::accounts::Accounts::stop_io pub async fn prepare(context: &Context, dir: &Path) -> Result { ensure!( // TODO: Should we worry about path normalisation? @@ -121,7 +123,7 @@ impl BackupProvider { Ok(Self { handle, ticket }) } - /// Creates the provider and supervisor tasks. + /// Creates the provider task. /// /// Having this as a function makes it easier to cancel it when needed. async fn prepare_inner(context: &Context, dir: &Path) -> Result<(Provider, Ticket)> { @@ -155,9 +157,10 @@ impl BackupProvider { Ok((provider, ticket)) } - /// Supervises the sendme [`Provider`] terminating it when needed. + /// Supervises the sendme [`Provider`], terminating it when needed. /// /// This will watch the provider and terminate it when: + /// /// - A transfer is completed, successful or unsuccessful. /// - An event could not be observed to protect against not knowing of a completed event. /// - The ongoing process is cancelled. @@ -221,16 +224,19 @@ impl BackupProvider { res } + /// Returns a QR code that allows fetching this backup. + /// + /// This QR code can be passed to [`get_backup`] on a (different) device. pub fn qr(&self) -> Qr { Qr::Backup { ticket: self.ticket.clone(), } } - /// Awaits the [`BackupSender`] until it is finished. + /// Awaits the [`BackupProvider`] until it is finished. /// /// This waits until someone connected to the sender and transferred a backup. If the - /// [`BackupSender`] task results in an error it will be returned here. + /// [`BackupProvider`] task results in an error it will be returned here. pub async fn join(self) -> Result<()> { self.handle.await??; Ok(()) @@ -285,7 +291,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { addr: ticket.addr, peer_id: Some(ticket.peer), }; - let progress = ProgressEmitter::new(0, 85); + let progress = ProgressEmitter::new(0, ReceiveProgress::max_blob_progress()); spawn_progress_proxy(context.clone(), progress.subscribe()); let on_connected = || { context.emit_event(ReceiveProgress::Connected.into()); @@ -328,6 +334,9 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { } /// Get callback when a blob is received from the provider. +/// +/// This writes the blobs to the blobdir. If the blob is the database it will import it to +/// the database of the current [`Context`]. async fn on_blob( context: &Context, progress: &ProgressEmitter, @@ -407,6 +416,15 @@ enum ReceiveProgress { Failed, } +impl ReceiveProgress { + /// The maximum value for [`ReceiveProgress::BlobProgress`]. + /// + /// This only exists to keep this magic value local in this type. + fn max_blob_progress() -> u16 { + 85 + } +} + impl From for EventType { fn from(source: ReceiveProgress) -> Self { let val = match source { @@ -479,6 +497,12 @@ impl ProgressEmitter { } } +/// The actual implementation. +/// +/// This exists so it can be Arc'd into [`ProgressEmitter`] and we can easily have multiple +/// `Send + Sync` copies of it. This is used by the +/// [`ProgressEmitter::ProgressAsyncReader`] to update the progress without intertwining +/// lifetimes. #[derive(Debug)] struct InnerProgressEmitter { total: AtomicU64, From 3c32de185910230126ad9baee9ef2422fe64c81a Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Fri, 10 Feb 2023 18:16:01 +0100 Subject: [PATCH 017/103] Generate a QR code --- src/qr.rs | 2 +- src/qr_code_generator.rs | 66 ++++++++++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/src/qr.rs b/src/qr.rs index 349a18244..4baabf487 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -35,7 +35,7 @@ const VCARD_SCHEME: &str = "BEGIN:VCARD"; const SMTP_SCHEME: &str = "SMTP:"; const HTTP_SCHEME: &str = "http://"; const HTTPS_SCHEME: &str = "https://"; -const DCBACKUP_SCHEME: &str = "DCBACKUP:"; +pub(crate) const DCBACKUP_SCHEME: &str = "DCBACKUP:"; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Qr { diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index bbd2031ea..40d8d9778 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -1,6 +1,6 @@ #![allow(missing_docs)] -use anyhow::Result; +use anyhow::{bail, Result}; use base64::Engine as _; use qrcodegen::{QrCode, QrCodeEcc}; @@ -11,6 +11,7 @@ use crate::{ config::Config, contact::{Contact, ContactId}, context::Context, + qr::{Qr, DCBACKUP_SCHEME}, securejoin, stock_str, }; @@ -43,6 +44,36 @@ async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Resu } async fn generate_verification_qr(context: &Context) -> Result { + let (avatar, displayname, addr, color) = self_info(context).await?; + + inner_generate_secure_join_qr_code( + &stock_str::setup_contact_qr_description(context, &displayname, &addr).await, + &securejoin::get_securejoin_qr(context, None).await?, + &color, + avatar, + displayname.chars().next().unwrap_or('#'), + ) +} + +pub async fn generate_backup_qr(context: &Context, qr: Qr) -> Result { + let Qr::Backup { ticket } = qr else { + bail!("QR code not a backup"); + }; + let content = format!("{DCBACKUP_SCHEME}{}", ticket.to_string()); + let (avatar, displayname, _addr, color) = self_info(context).await?; + let description = "Scan to setup second device"; // TODO: translation! + + inner_generate_secure_join_qr_code( + description, + &content, + &color, + avatar, + displayname.chars().next().unwrap_or('#'), + ) +} + +/// Returns `(avatar, displayname, addr, color) of the configured account. +async fn self_info(context: &Context) -> Result<(Option>, String, String, String)> { let contact = Contact::get_by_id(context, ContactId::SELF).await?; let avatar = match contact.get_profile_image(context).await? { @@ -55,16 +86,11 @@ async fn generate_verification_qr(context: &Context) -> Result { let displayname = match context.get_config(Config::Displayname).await? { Some(name) => name, - None => contact.get_addr().to_owned(), + None => contact.get_addr().to_string(), }; - - inner_generate_secure_join_qr_code( - &stock_str::setup_contact_qr_description(context, &displayname, contact.get_addr()).await, - &securejoin::get_securejoin_qr(context, None).await?, - &color_int_to_hex_string(contact.get_color()), - avatar, - displayname.chars().next().unwrap_or('#'), - ) + let addr = contact.get_addr().to_string(); + let color = color_int_to_hex_string(contact.get_color()); + Ok((avatar, displayname, addr, color)) } fn inner_generate_secure_join_qr_code( @@ -268,6 +294,11 @@ fn inner_generate_secure_join_qr_code( #[cfg(test)] mod tests { + use testdir::testdir; + + use crate::imex::BackupProvider; + use crate::test_utils::TestContextManager; + use super::*; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -282,4 +313,19 @@ mod tests { .unwrap(); assert!(svg.contains("descr123 " < > &")) } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_generate_backup_qr() { + let dir = testdir!(); + let mut tcm = TestContextManager::new(); + let ctx = tcm.alice().await; + let provider = BackupProvider::prepare(&ctx, &dir).await.unwrap(); + let qr = provider.qr(); + + let rendered = generate_backup_qr(&ctx, qr).await.unwrap(); + tokio::fs::write(dir.join("qr.svg"), &rendered) + .await + .unwrap(); + assert_eq!(&rendered[..4], " Date: Fri, 10 Feb 2023 18:27:03 +0100 Subject: [PATCH 018/103] clippy --- src/qr_code_generator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index 40d8d9778..7f43cb2c2 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -59,7 +59,7 @@ pub async fn generate_backup_qr(context: &Context, qr: Qr) -> Result { let Qr::Backup { ticket } = qr else { bail!("QR code not a backup"); }; - let content = format!("{DCBACKUP_SCHEME}{}", ticket.to_string()); + let content = format!("{DCBACKUP_SCHEME}{ticket}"); let (avatar, displayname, _addr, color) = self_info(context).await?; let description = "Scan to setup second device"; // TODO: translation! @@ -326,6 +326,6 @@ mod tests { tokio::fs::write(dir.join("qr.svg"), &rendered) .await .unwrap(); - assert_eq!(&rendered[..4], " Date: Fri, 10 Feb 2023 18:35:47 +0100 Subject: [PATCH 019/103] Add to repl example --- examples/repl/cmdline.rs | 18 ++++++++++++++++++ examples/repl/main.rs | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/examples/repl/cmdline.rs b/examples/repl/cmdline.rs index cf99840ef..1d3d1edbb 100644 --- a/examples/repl/cmdline.rs +++ b/examples/repl/cmdline.rs @@ -336,6 +336,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu has-backup\n\ export-backup\n\ import-backup \n\ + send-backup\n\ + receive-backup \n\ export-keys\n\ import-keys\n\ export-setup\n\ @@ -486,6 +488,22 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu ) .await?; } + "send-backup" => { + let tdir = tempfile::TempDir::new()?; + let dir = tdir.path(); + let provider = BackupProvider::prepare(&context, dir).await?; + let qr = provider.qr(); + let rendered = deltachat::qr_code_generator::generate_backup_qr(&context, qr).await?; + let file = dir.join("qr.svg"); + tokio::fs::write(&file, rendered).await?; + println!("The QR code is at: {}", file.display()); + provider.join().await?; + } + "receive-backup" => { + ensure!(!arg1.is_empty(), "Argument is missing."); + let qr = check_qr(&context, arg1).await?; + deltachat::imex::get_backup(&context, qr).await?; + } "export-keys" => { let dir = dirs::home_dir().unwrap_or_default(); imex(&context, ImexMode::ExportSelfKeys, dir.as_ref(), None).await?; diff --git a/examples/repl/main.rs b/examples/repl/main.rs index 3798b9364..697cd1368 100644 --- a/examples/repl/main.rs +++ b/examples/repl/main.rs @@ -152,13 +152,15 @@ impl Completer for DcHelper { } } -const IMEX_COMMANDS: [&str; 12] = [ +const IMEX_COMMANDS: [&str; 14] = [ "initiate-key-transfer", "get-setupcodebegin", "continue-key-transfer", "has-backup", "export-backup", "import-backup", + "send-backup", + "receive-backup", "export-keys", "import-keys", "export-setup", From 852adbe514c4aaf44f7f2e83223b5d158f8e6771 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 13 Feb 2023 15:45:38 +0100 Subject: [PATCH 020/103] bits left over from master merge --- Cargo.lock | 1 + deltachat-repl/Cargo.toml | 1 + src/imex/transfer.rs | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 304027515..6073b8ec9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1233,6 +1233,7 @@ dependencies = [ "pretty_env_logger", "rusqlite", "rustyline", + "tempfile", "tokio 1.25.0", ] diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index b6df1f474..87e3c195f 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -12,6 +12,7 @@ log = "0.4.16" pretty_env_logger = "0.4" rusqlite = "0.27" rustyline = "10" +tempfile = "3" tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } [features] diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index df28e07fc..b0741285c 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -219,6 +219,7 @@ impl BackupProvider { }, } }; + // TODO: delete the database? context.emit_event(SendProgress::Completed.into()); context.free_ongoing().await; res @@ -605,7 +606,7 @@ mod tests { // Check that we have the self message. let self_chat = ctx1.get_self_chat().await; - let msgs = get_chat_msgs(&ctx1, self_chat.id, 0).await.unwrap(); + let msgs = get_chat_msgs(&ctx1, self_chat.id).await.unwrap(); assert_eq!(msgs.len(), 1); let msgid = match msgs.get(0).unwrap() { ChatItem::Message { msg_id } => msg_id, From 323535584bd637e3178f9d5b32502a966bedc434 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 13 Feb 2023 18:25:12 +0100 Subject: [PATCH 021/103] implement ffi and use public sendme --- Cargo.lock | 758 ++++++++------------------------------ Cargo.toml | 3 - deltachat-ffi/deltachat.h | 104 +++++- deltachat-ffi/src/lib.rs | 99 ++++- src/qr.rs | 13 + 5 files changed, 363 insertions(+), 614 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6073b8ec9..9d87cb5a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,7 +29,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", ] @@ -81,7 +81,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -171,7 +171,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -184,7 +184,7 @@ dependencies = [ "base64 0.13.1", "byte-pool", "chrono", - "futures 0.3.26", + "futures", "imap-proto", "log", "nom 7.1.1", @@ -193,7 +193,7 @@ dependencies = [ "pin-utils", "stop-token", "thiserror", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -213,7 +213,7 @@ checksum = "d57d4cec3c647232e1094dc013546c0b33ce785d8aeb251e1f20dfaf8a9a13fe" dependencies = [ "native-tls", "thiserror", - "tokio 1.25.0", + "tokio", "url", ] @@ -226,13 +226,13 @@ dependencies = [ "anyhow", "base64 0.13.1", "bufstream", - "futures 0.3.26", + "futures", "hostname", "log", "nom 7.1.1", "pin-project", "thiserror", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -252,7 +252,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b20cffc5590f4bf33f05f97a3ea587feba9c50d20325b401daa096b92ff7da0" dependencies = [ - "tokio 1.25.0", + "tokio", ] [[package]] @@ -266,7 +266,7 @@ dependencies = [ "chrono", "crc32fast", "thiserror", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -277,7 +277,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -296,7 +296,7 @@ dependencies = [ "axum-core 0.2.9", "base64 0.13.1", "bitflags", - "bytes 1.2.1", + "bytes", "futures-util", "http", "http-body", @@ -312,7 +312,7 @@ dependencies = [ "serde_urlencoded", "sha-1", "sync_wrapper", - "tokio 1.25.0", + "tokio", "tokio-tungstenite 0.17.2", "tower", "tower-http", @@ -330,7 +330,7 @@ dependencies = [ "axum-core 0.3.2", "base64 0.20.0", "bitflags", - "bytes 1.2.1", + "bytes", "futures-util", "http", "http-body", @@ -348,7 +348,7 @@ dependencies = [ "serde_urlencoded", "sha1", "sync_wrapper", - "tokio 1.25.0", + "tokio", "tokio-tungstenite 0.18.0", "tower", "tower-http", @@ -363,7 +363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e5939e02c56fecd5c017c37df4238c0a839fa76b7f97acdd7efb804fd181cc" dependencies = [ "async-trait", - "bytes 1.2.1", + "bytes", "futures-util", "http", "http-body", @@ -379,7 +379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34" dependencies = [ "async-trait", - "bytes 1.2.1", + "bytes", "futures-util", "http", "http-body", @@ -397,7 +397,7 @@ checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide", "object", @@ -478,7 +478,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if 1.0.0", + "cfg-if", "constant_time_eq", "digest 0.10.5", ] @@ -548,7 +548,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8c7230ddbb427b1094d477d821a99f3f54d36333178eeb806e279bcdcecf0ca" dependencies = [ - "crossbeam-queue 0.3.6", + "crossbeam-queue", "stable_deref_trait", ] @@ -564,16 +564,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.2.1" @@ -606,7 +596,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.16", + "semver", "serde", "serde_json", ] @@ -641,12 +631,6 @@ dependencies = [ "cipher", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -675,7 +659,7 @@ dependencies = [ "num-traits", "time 0.1.44", "wasm-bindgen", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -744,16 +728,7 @@ checksum = "c4ab1b92798304eedc095b53942963240037c0516452cb11aeba709d420b2219" dependencies = [ "error-code", "str-buf", - "winapi 0.3.9", -] - -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", + "winapi", ] [[package]] @@ -791,7 +766,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" dependencies = [ - "crossbeam-utils 0.8.11", + "crossbeam-utils", ] [[package]] @@ -869,7 +844,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -884,7 +859,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", - "futures 0.3.26", + "futures", "itertools", "lazy_static", "num-traits", @@ -896,7 +871,7 @@ dependencies = [ "serde_derive", "serde_json", "tinytemplate", - "tokio 1.25.0", + "tokio", "walkdir", ] @@ -916,19 +891,8 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.11", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "cfg-if", + "crossbeam-utils", ] [[package]] @@ -937,24 +901,9 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch 0.9.10", - "crossbeam-utils 0.8.11", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -964,43 +913,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" dependencies = [ "autocfg", - "cfg-if 1.0.0", - "crossbeam-utils 0.8.11", - "memoffset 0.6.5", + "cfg-if", + "crossbeam-utils", + "memoffset", "once_cell", "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - [[package]] name = "crossbeam-queue" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.11", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "cfg-if", + "crossbeam-utils", ] [[package]] @@ -1009,7 +936,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -1146,7 +1073,7 @@ dependencies = [ "escaper", "fast-socks5", "format-flowed", - "futures 0.3.26", + "futures", "futures-lite", "hex", "humansize", @@ -1181,7 +1108,7 @@ dependencies = [ "serde_json", "sha-1", "sha2 0.10.6", - "smallvec 1.10.0", + "smallvec", "strum", "strum_macros", "tagger", @@ -1189,7 +1116,7 @@ dependencies = [ "testdir", "textwrap", "thiserror", - "tokio 1.25.0", + "tokio", "tokio-io-timeout", "tokio-stream", "tokio-tar", @@ -1208,14 +1135,14 @@ dependencies = [ "axum 0.6.4", "deltachat", "env_logger 0.10.0", - "futures 0.3.26", + "futures", "log", "num-traits", "sanitize-filename", "serde", "serde_json", "tempfile", - "tokio 1.25.0", + "tokio", "typescript-type-def", "walkdir", "yerpc", @@ -1234,7 +1161,7 @@ dependencies = [ "rusqlite", "rustyline", "tempfile", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -1248,7 +1175,7 @@ dependencies = [ "log", "serde", "serde_json", - "tokio 1.25.0", + "tokio", "yerpc", ] @@ -1274,7 +1201,7 @@ dependencies = [ "rand 0.7.3", "serde_json", "thiserror", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -1391,7 +1318,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -1403,7 +1330,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1414,7 +1341,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1589,7 +1516,7 @@ version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1650,7 +1577,7 @@ checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" dependencies = [ "errno-dragonfly", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1709,7 +1636,7 @@ dependencies = [ "anyhow", "log", "thiserror", - "tokio 1.25.0", + "tokio", "tokio-stream", ] @@ -1737,7 +1664,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "rustix 0.35.7", "windows-sys 0.36.1", ] @@ -1758,9 +1685,9 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "windows-sys 0.36.1", ] @@ -1808,28 +1735,6 @@ dependencies = [ name = "format-flowed" version = "1.0.0" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.26" @@ -1950,7 +1855,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -1961,7 +1866,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] @@ -1999,7 +1904,7 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" dependencies = [ - "bytes 1.2.1", + "bytes", "fnv", "futures-core", "futures-sink", @@ -2007,7 +1912,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.25.0", + "tokio", "tokio-util", "tracing", ] @@ -2092,7 +1997,7 @@ checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" dependencies = [ "libc", "match_cfg", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2101,7 +2006,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ - "bytes 1.2.1", + "bytes", "fnv", "itoa", ] @@ -2112,7 +2017,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.2.1", + "bytes", "http", "pin-project-lite", ] @@ -2181,7 +2086,7 @@ version = "0.14.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" dependencies = [ - "bytes 1.2.1", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -2193,7 +2098,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio 1.25.0", + "tokio", "tower-service", "tracing", "want", @@ -2205,10 +2110,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.2.1", + "bytes", "hyper", "native-tls", - "tokio 1.25.0", + "tokio", "tokio-native-tls", ] @@ -2222,7 +2127,7 @@ dependencies = [ "core-foundation-sys", "js-sys", "wasm-bindgen", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2302,7 +2207,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2321,15 +2226,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipconfig" version = "0.3.0" @@ -2338,7 +2234,7 @@ checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" dependencies = [ "socket2", "widestring", - "winapi 0.3.9", + "winapi", "winreg 0.7.0", ] @@ -2405,16 +2301,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9b7d56ba4a8344d6be9729995e6b06f928af29998cdf79fe390cbf6b1fee838" -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2490,15 +2376,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" -[[package]] -name = "lock_api" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -dependencies = [ - "scopeguard", -] - [[package]] name = "lock_api" version = "0.4.7" @@ -2515,7 +2392,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2571,12 +2448,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md-5" version = "0.10.5" @@ -2592,15 +2463,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.5" @@ -2631,25 +2493,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.8.5" @@ -2662,29 +2505,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - [[package]] name = "mutate_once" version = "0.1.1" @@ -2709,24 +2529,13 @@ dependencies = [ "tempfile", ] -[[package]] -name = "net2" -version = "0.2.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d0df99cfcd2530b2e694f6e17e7f37b8e26bb23983ac530c0c97408837c631" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nibble_vec" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" dependencies = [ - "smallvec 1.10.0", + "smallvec", ] [[package]] @@ -2737,7 +2546,7 @@ checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags", - "cfg-if 1.0.0", + "cfg-if", "libc", ] @@ -2776,7 +2585,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc51db7b362b205941f71232e56c625156eb9a929f8cf74a428fd5bc094a4afc" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2804,7 +2613,7 @@ dependencies = [ "num-traits", "rand 0.8.5", "serde", - "smallvec 1.10.0", + "smallvec", "zeroize", ] @@ -2914,7 +2723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" dependencies = [ "bitflags", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -2970,7 +2779,7 @@ checksum = "d2cc1b4330bb29087e791ae2a5cf56be64fb8946a4ff5aec2ba11c6ca51f5d60" dependencies = [ "log", "serde", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3030,40 +2839,14 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" -[[package]] -name = "parking_lot" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.6.3", - "rustc_version", -] - [[package]] name = "parking_lot" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api 0.4.7", - "parking_lot_core 0.9.3", -] - -[[package]] -name = "parking_lot_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "rustc_version", - "smallvec 0.6.14", - "winapi 0.3.9", + "lock_api", + "parking_lot_core", ] [[package]] @@ -3072,10 +2855,10 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", - "redox_syscall 0.2.16", - "smallvec 1.10.0", + "redox_syscall", + "smallvec", "windows-sys 0.36.1", ] @@ -3141,7 +2924,7 @@ dependencies = [ "sha2 0.10.6", "sha3", "signature", - "smallvec 1.10.0", + "smallvec", "thiserror", "twofish", "x25519-dalek", @@ -3370,14 +3153,14 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "445cbfe2382fa023c4f2f3c7e1c95c03dcc1df2bf23cebcb2b13e1402c4394d1" dependencies = [ - "bytes 1.2.1", + "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash", "rustls", "thiserror", - "tokio 1.25.0", + "tokio", "tracing", "webpki", ] @@ -3388,7 +3171,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72ef4ced82a24bb281af338b9e8f94429b6eca01b4e66d899f40031f074e74c9" dependencies = [ - "bytes 1.2.1", + "bytes", "rand 0.8.5", "ring", "rustc-hash", @@ -3436,7 +3219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot 0.12.1", + "parking_lot", "scheduled-thread-pool", ] @@ -3551,7 +3334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ "autocfg", - "crossbeam-deque 0.8.2", + "crossbeam-deque", "either", "rayon-core", ] @@ -3563,8 +3346,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" dependencies = [ "crossbeam-channel", - "crossbeam-deque 0.8.2", - "crossbeam-utils 0.8.11", + "crossbeam-deque", + "crossbeam-utils", "num_cpus", ] @@ -3580,12 +3363,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.16" @@ -3602,7 +3379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom 0.2.7", - "redox_syscall 0.2.16", + "redox_syscall", "thiserror", ] @@ -3638,7 +3415,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3648,7 +3425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" dependencies = [ "base64 0.21.0", - "bytes 1.2.1", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -3668,7 +3445,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 1.25.0", + "tokio", "tokio-native-tls", "tower-service", "url", @@ -3711,7 +3488,7 @@ dependencies = [ "spin", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3739,7 +3516,7 @@ dependencies = [ "pkcs8", "rand_core 0.6.3", "signature", - "smallvec 1.10.0", + "smallvec", "subtle", "zeroize", ] @@ -3756,7 +3533,7 @@ dependencies = [ "hashlink", "libsqlite3-sys", "memchr", - "smallvec 1.10.0", + "smallvec", ] [[package]] @@ -3777,15 +3554,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rusticata-macros" version = "4.1.0" @@ -3868,7 +3636,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e83c32c3f3c33b08496e0d1df9ea8c64d39adb8eb36a1ebb1440c690697aef" dependencies = [ "bitflags", - "cfg-if 1.0.0", + "cfg-if", "clipboard-win", "dirs-next", "fd-lock", @@ -3881,7 +3649,7 @@ dependencies = [ "unicode-segmentation", "unicode-width", "utf8parse", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3931,7 +3699,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf" dependencies = [ - "parking_lot 0.12.1", + "parking_lot", ] [[package]] @@ -3987,15 +3755,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.16" @@ -4005,24 +3764,19 @@ dependencies = [ "serde", ] -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "sendme" version = "0.1.0" +source = "git+https://github.com/n0-computer/sendme?branch=main#5bd545d8eb800a48b36c8d7d2333b9f21afec49a" dependencies = [ "anyhow", "bao", "base64 0.21.0", "blake3", - "bytes 1.2.1", + "bytes", "der", "ed25519-dalek", - "futures 0.3.26", + "futures", "postcard", "quinn", "rand 0.7.3", @@ -4033,7 +3787,7 @@ dependencies = [ "ssh-key", "tempfile", "thiserror", - "tokio 1.25.0", + "tokio", "tokio-util", "tracing", "tracing-futures", @@ -4119,7 +3873,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.5", ] @@ -4130,7 +3884,7 @@ version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.5", ] @@ -4142,7 +3896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -4154,7 +3908,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.5", ] @@ -4206,15 +3960,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.10.0" @@ -4234,7 +3979,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4295,7 +4040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af91f480ee899ab2d9f8435bfdfc14d08a5754bd9d3fef1f1a1c23336aad6c8b" dependencies = [ "async-channel", - "cfg-if 1.0.0", + "cfg-if", "futures-core", "pin-project-lite", ] @@ -4372,12 +4117,12 @@ version = "0.26.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c18a6156d1f27a9592ee18c1a846ca8dd5c258b7179fc193ae87c74ebb666f5" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "core-foundation-sys", "libc", "ntapi", "once_cell", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4392,12 +4137,12 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "remove_dir_all", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4460,7 +4205,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -4472,7 +4217,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4527,30 +4272,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" -[[package]] -name = "tokio" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "num_cpus", - "tokio-codec", - "tokio-current-thread", - "tokio-executor", - "tokio-fs", - "tokio-io", - "tokio-reactor", - "tokio-sync", - "tokio-tcp", - "tokio-threadpool", - "tokio-timer", - "tokio-udp", - "tokio-uds", -] - [[package]] name = "tokio" version = "1.25.0" @@ -4558,12 +4279,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", - "bytes 1.2.1", + "bytes", "libc", "memchr", - "mio 0.8.5", + "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -4571,59 +4292,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "tokio-codec" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "tokio-io", -] - -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - -[[package]] -name = "tokio-executor" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", -] - -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-io-timeout" version = "1.2.0" @@ -4631,7 +4299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" dependencies = [ "pin-project-lite", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -4652,26 +4320,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.25.0", -] - -[[package]] -name = "tokio-reactor" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "mio 0.6.23", - "num_cpus", - "parking_lot 0.9.0", - "slab", - "tokio-executor", - "tokio-io", - "tokio-sync", + "tokio", ] [[package]] @@ -4682,17 +4331,7 @@ checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite", - "tokio 1.25.0", -] - -[[package]] -name = "tokio-sync" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" -dependencies = [ - "fnv", - "futures 0.1.31", + "tokio", ] [[package]] @@ -4704,55 +4343,12 @@ dependencies = [ "filetime", "futures-core", "libc", - "redox_syscall 0.2.16", - "tokio 1.25.0", + "redox_syscall", + "tokio", "tokio-stream", "xattr", ] -[[package]] -name = "tokio-tcp" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "mio 0.6.23", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque 0.7.4", - "crossbeam-queue 0.2.3", - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-timer" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "slab", - "tokio-executor", -] - [[package]] name = "tokio-tungstenite" version = "0.17.2" @@ -4761,7 +4357,7 @@ checksum = "f714dd15bead90401d77e04243611caec13726c2408afd5b31901dfcdcb3b181" dependencies = [ "futures-util", "log", - "tokio 1.25.0", + "tokio", "tungstenite 0.17.3", ] @@ -4773,54 +4369,21 @@ checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" dependencies = [ "futures-util", "log", - "tokio 1.25.0", + "tokio", "tungstenite 0.18.0", ] -[[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-uds" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "libc", - "log", - "mio 0.6.23", - "mio-uds", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - [[package]] name = "tokio-util" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" dependencies = [ - "bytes 1.2.1", + "bytes", "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.25.0", + "tokio", "tracing", ] @@ -4877,7 +4440,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.25.0", + "tokio", "tower-layer", "tower-service", "tracing", @@ -4890,7 +4453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" dependencies = [ "bitflags", - "bytes 1.2.1", + "bytes", "futures-core", "futures-util", "http", @@ -4920,7 +4483,7 @@ version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -4955,7 +4518,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ "pin-project", - "tokio 0.1.22", "tracing", ] @@ -4981,7 +4543,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec 1.10.0", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -4995,7 +4557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" dependencies = [ "async-trait", - "cfg-if 1.0.0", + "cfg-if", "data-encoding", "enum-as-inner", "futures-channel", @@ -5005,10 +4567,10 @@ dependencies = [ "ipnet", "lazy_static", "rand 0.8.5", - "smallvec 1.10.0", + "smallvec", "thiserror", "tinyvec", - "tokio 1.25.0", + "tokio", "tracing", "url", ] @@ -5019,16 +4581,16 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "futures-util", "ipconfig", "lazy_static", "lru-cache", - "parking_lot 0.12.1", + "parking_lot", "resolv-conf", - "smallvec 1.10.0", + "smallvec", "thiserror", - "tokio 1.25.0", + "tokio", "tracing", "trust-dns-proto", ] @@ -5047,7 +4609,7 @@ checksum = "e27992fd6a8c29ee7eef28fc78349aa244134e10ad447ce3b9f0ac0ed0fa4ce0" dependencies = [ "base64 0.13.1", "byteorder", - "bytes 1.2.1", + "bytes", "http", "httparse", "log", @@ -5066,7 +4628,7 @@ checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" dependencies = [ "base64 0.13.1", "byteorder", - "bytes 1.2.1", + "bytes", "http", "httparse", "log", @@ -5250,7 +4812,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" dependencies = [ "same-file", - "winapi 0.3.9", + "winapi", "winapi-util", ] @@ -5288,7 +4850,7 @@ version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -5313,7 +4875,7 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -5390,12 +4952,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -5406,12 +4962,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -5424,7 +4974,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -5539,7 +5089,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -5548,17 +5098,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] @@ -5619,12 +5159,12 @@ dependencies = [ "async-mutex", "async-trait", "axum 0.5.17", - "futures 0.3.26", + "futures", "futures-util", "log", "serde", "serde_json", - "tokio 1.25.0", + "tokio", "tracing", "typescript-type-def", "yerpc_derive", diff --git a/Cargo.toml b/Cargo.toml index dbf3503e6..4dd08702a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,6 +152,3 @@ vendored = [ "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored" ] - -[patch.'https://github.com/n0-computer/sendme'] -sendme = { path = "/home/flub/n0/sendme" } \ No newline at end of file diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index d3b1219f1..c320cea16 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -24,6 +24,7 @@ typedef struct _dc_provider dc_provider_t; typedef struct _dc_event dc_event_t; typedef struct _dc_event_emitter dc_event_emitter_t; typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t; +typedef struct _dc_backup_provider dc_backup_provider_t; // Alias for backwards compatibility, use dc_event_emitter_t instead. typedef struct _dc_event_emitter dc_accounts_event_emitter_t; @@ -2295,6 +2296,7 @@ void dc_stop_ongoing_process (dc_context_t* context); #define DC_QR_FPR_MISMATCH 220 // id=contact #define DC_QR_FPR_WITHOUT_ADDR 230 // test1=formatted fingerprint #define DC_QR_ACCOUNT 250 // text1=domain +#define DC_QR_BACKUP 251 #define DC_QR_WEBRTC_INSTANCE 260 // text1=domain, text2=instance pattern #define DC_QR_ADDR 320 // id=contact #define DC_QR_TEXT 330 // text1=text @@ -2634,8 +2636,108 @@ char* dc_get_last_error (dc_context_t* context); void dc_str_unref (char* str); -// TODO: add New stuff +/** + * @class dc_backup_provider_t + * + * Set up another device. + */ +/** + * Creates an object for sending a backup to another device. + * + * The backup is sent to through a peer-to-peer channel which is bootstrapped + * by a QR-code. The backup contains the entire state of the account + * including credentials. This can be used to setup a new device. + * + * Once this function returns the backup is being offered to remove devices. + * To wait until one device transferred the backup, use + * dc_backup_provider_done(). Alternatively abort the operation using + * dc_stop_ongoing_process(). + * + * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate + * state and progress. + * + * @memberof dc_backup_sender_t + * @param context The context. + * @param folder A Path to a temporary directory where the encrypted database + * export will be created. The directory is not automatically cleaned + * after the backup is sent. + * @return Opaque object for sending the backup. + * On errors, NULL is returned and dc_get_last_error()returns an error that + * should be shown to the user. + */ +dc_backup_provider_t* dc_provide_backup (dc_context_t* context, const chat* folder); + + +/** + * Returns the QR code text that will offer the backup to other devices. + * + * The QR code contains a ticket which will validate the backup and provide + * authentication for both the provider and the recipient. + * + * The scanning device should call the scanned text to dc_check_qr(). If + * dc_check_qr() returns DC_QR_BACKUP the backup transfer can be started using + * dc_get_backup(). + * + * @memberof dc_backup_provider_t + * @param context The context. + * @param backup_provider The backup provider object as created by + * dc_provide_backup(). + * @return The text that should be put in the QR code. + * On errors and empty QR code is returned, NULL is never returned. + * the returned string must be released using dc_str_unref() after usage. + */ +char* dc_backup_provider_qr (dc_context_t* context, const dc_backup_provider_t* backup_provider); + + +/** + * Returns the QR code SVG image that will offer the backup to other devices. + * + * This works like dc_backup_provider_qr() but returns the text of a rendered + * SVG image containing the QR code. + * + * @memberof dc_backup_provider_t + * @param context The context. + * @param backup_provider The backup provider object as created by + * dc_provide_backup(). + * @return The text that should be put in the QR code. + * On errors and empty QR code is returned, NULL is never returned. + * the returned string must be released using dc_str_unref() after usage. + */ +char * dc_backup_provider_qr_svg (dc_context_t* context, const dc_backup_provider_t* backup_provider); + +/** + * Waits for the sending to finish and frees the backup provider object. + * + * @memberof dc_backup_sender_t + * @param context The context. + * @param backup_sender The backup sender object as created by dc_send_backup(). + * If NULL is given nothing is done. + */ +void dc_backup_provider_done (dc_context_t* context, dc_backup_provider_t* backup_provider); + + +/** + * Gets a backup offered by a dc_backup_provider_t object on another device. + * + * This function is called on a device that scanned the QR code offered by + * dc_backup_sender_qr() or dc_backup_sender_qr_svg(). Typically this is a + * different device than that which provides the backup. + * + * While dc_receive_backup() returns immediately the started job make take a + * while. Use dc_stop_ongoing_process() to abort it early. + * + * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate + * state and progress. + * + * @param context The context. + * @param qr The qr code text, dc_check_qr() must have returned DC_QR_BACKUP + * on this text. + * @return 0=failure, 1=success. Success only means the progress was started + * correctly, final success or failure is via the #DC_EVENT_IMEX_PROGRESS + * events. + */ +int dc_receive_backup (dc_context_t* context, const char* qr); /** * @class dc_accounts_t diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index c7cd23bd2..74f79cf5d 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -28,9 +28,10 @@ use deltachat::constants::DC_MSG_ID_LAST_SPECIAL; use deltachat::contact::{Contact, ContactId, Origin}; use deltachat::context::Context; use deltachat::ephemeral::Timer as EphemeralTimer; +use deltachat::imex::BackupProvider; use deltachat::key::DcKey; use deltachat::message::MsgId; -use deltachat::qr_code_generator::get_securejoin_qr_svg; +use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg}; use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions}; use deltachat::stock_str::StockMessage; use deltachat::stock_str::StockStrings; @@ -4130,6 +4131,102 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) { libc::free(s as *mut _) } +pub type dc_backup_provider_t = BackupProvider; + +#[no_mangle] +pub unsafe extern "C" fn dc_provide_backup( + context: *mut dc_context_t, + folder: *const libc::c_char, +) -> *mut dc_backup_provider_t { + if context.is_null() { + eprintln!("ignoring careless call to dc_send_backup()"); + return ptr::null_mut(); + } + let ctx = &*context; + let dir = as_path(folder); + block_on(async move { + BackupProvider::prepare(ctx, dir) + .await + .map(|provider| Box::into_raw(Box::new(provider))) + .log_err(ctx, "BackupProvider failed") + .unwrap_or(ptr::null_mut()) + }) +} + +#[no_mangle] +pub unsafe extern "C" fn dc_backup_provider_qr( + _context: *mut dc_context_t, + provider: *const dc_backup_provider_t, +) -> *mut libc::c_char { + let provider = &*provider; + deltachat::qr::format_backup(provider.qr()) + .unwrap_or_default() + .strdup() +} + +#[no_mangle] +pub unsafe extern "C" fn dc_backup_provider_qr_svg( + context: *mut dc_context_t, + provider: *const dc_backup_provider_t, +) -> *mut libc::c_char { + if context.is_null() { + eprintln!("ignoring careless call to dc_send_backup()"); + return ptr::null_mut(); + } + let ctx = &*context; + let provider = &*provider; + block_on(async move { + generate_backup_qr(ctx, provider.qr()) + .await + .unwrap_or_default() + .strdup() + }) +} + +#[no_mangle] +pub unsafe extern "C" fn dc_backup_provider_done( + context: *mut dc_context_t, + provider: *mut dc_backup_provider_t, +) { + if context.is_null() { + eprintln!("ignoring careless call to dc_send_backup()"); + return; + } + let ctx = &*context; + let provider = Box::from_raw(provider); + block_on(async move { + provider + .join() + .await + .log_err(ctx, "Failed to join provider") + .ok(); + }); +} + +#[no_mangle] +pub unsafe extern "C" fn dc_receive_backup( + context: *mut dc_context_t, + qr: *const libc::c_char, +) -> libc::c_int { + if context.is_null() { + eprintln!("ignoring careless call to dc_send_backup()"); + return 0; + } + let ctx = &*context; + let qr_text = to_string_lossy(qr); + let qr = match block_on(qr::check_qr(ctx, &qr_text)).log_err(ctx, "Invalid QR code") { + Ok(qr) => qr, + Err(_) => return 0, + }; + spawn(async move { + imex::get_backup(ctx, qr) + .await + .log_err(ctx, "Get backup failed") + .ok(); + }); + 1 +} + trait ResultExt { /// Like `log_err()`, but: /// - returns the default value instead of an Err value. diff --git a/src/qr.rs b/src/qr.rs index 00e063513..3d4bc5a6f 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -161,6 +161,19 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result { Ok(qrcode) } +/// Formats the text of the [`Qr::Backup`] variant. +/// +/// This is the inverse of [`check_qr`] for that variant only. +/// +/// TODO: Refactor this so all variants have a correct [`Display`] and transform `check_qr` +/// into [`FromStr`]. +pub fn format_backup(qr: Qr) -> Result { + match qr { + Qr::Backup { ticket } => Ok(format!("{DCBACKUP_SCHEME}{ticket}")), + _ => Err(anyhow!("Not a backup QR code")), + } +} + /// scheme: `OPENPGP4FPR:FINGERPRINT#a=ADDR&n=NAME&i=INVITENUMBER&s=AUTH` /// or: `OPENPGP4FPR:FINGERPRINT#a=ADDR&g=GROUPNAME&x=GROUPID&i=INVITENUMBER&s=AUTH` /// or: `OPENPGP4FPR:FINGERPRINT#a=ADDR` From e1087b41450f12509e6dc5f9c67b4a6db0156a95 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 12:07:02 +0100 Subject: [PATCH 022/103] translate the string for qr code --- src/imex/transfer.rs | 8 ++------ src/qr.rs | 4 ++-- src/qr_code_generator.rs | 9 ++++++--- src/stock_str.rs | 19 +++++++++++++++++-- 4 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index b0741285c..67808b511 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -94,12 +94,7 @@ impl BackupProvider { biased; res = Self::prepare_inner(context, dir) => { match res { - Ok(slf) => { - // TODO: maybe this is the wrong place to log this - // TODO: Also needs to log progress somehow. - info!(context, "Waiting for remote to connect"); - Ok(slf) - }, + Ok(slf) => Ok(slf), Err(err) => { error!(context, "Failed to set up second device setup: {:#}", err); Err(err) @@ -153,6 +148,7 @@ impl BackupProvider { context.emit_event(SendProgress::CollectionCreated.into()); let provider = Provider::builder(db).auth_token(token).spawn()?; context.emit_event(SendProgress::ProviderListening.into()); + info!(context, "Waiting for remote to connect"); let ticket = provider.ticket(hash); Ok((provider, ticket)) } diff --git a/src/qr.rs b/src/qr.rs index 3d4bc5a6f..a5567d92b 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -167,9 +167,9 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result { /// /// TODO: Refactor this so all variants have a correct [`Display`] and transform `check_qr` /// into [`FromStr`]. -pub fn format_backup(qr: Qr) -> Result { +pub fn format_backup(qr: &Qr) -> Result { match qr { - Qr::Backup { ticket } => Ok(format!("{DCBACKUP_SCHEME}{ticket}")), + Qr::Backup { ref ticket } => Ok(format!("{DCBACKUP_SCHEME}{ticket}")), _ => Err(anyhow!("Not a backup QR code")), } } diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index 7f43cb2c2..c6f384cf5 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -12,7 +12,8 @@ use crate::{ contact::{Contact, ContactId}, context::Context, qr::{Qr, DCBACKUP_SCHEME}, - securejoin, stock_str, + securejoin, + stock_str::{self, backup_transfer_qr}, }; pub async fn get_securejoin_qr_svg(context: &Context, chat_id: Option) -> Result { @@ -61,10 +62,10 @@ pub async fn generate_backup_qr(context: &Context, qr: Qr) -> Result { }; let content = format!("{DCBACKUP_SCHEME}{ticket}"); let (avatar, displayname, _addr, color) = self_info(context).await?; - let description = "Scan to setup second device"; // TODO: translation! + let description = backup_transfer_qr(context).await?; inner_generate_secure_join_qr_code( - description, + &description, &content, &color, avatar, @@ -297,6 +298,7 @@ mod tests { use testdir::testdir; use crate::imex::BackupProvider; + use crate::qr::format_backup; use crate::test_utils::TestContextManager; use super::*; @@ -322,6 +324,7 @@ mod tests { let provider = BackupProvider::prepare(&ctx, &dir).await.unwrap(); let qr = provider.qr(); + println!("{}", format_backup(&qr).unwrap()); let rendered = generate_backup_qr(&ctx, qr).await.unwrap(); tokio::fs::write(dir.join("qr.svg"), &rendered) .await diff --git a/src/stock_str.rs b/src/stock_str.rs index f91b5a783..8b3b6162f 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -404,6 +404,9 @@ pub enum StockMessage { #[strum(props(fallback = "Chat protection disabled by %1$s."))] ProtectionDisabledBy = 161, + + #[strum(props(fallback = "Scan to set up second device for %1$s"))] + BackupTransferQr = 162, } impl StockMessage { @@ -740,14 +743,14 @@ pub(crate) async fn setup_contact_qr_description( display_name: &str, addr: &str, ) -> String { - let name = &if display_name == addr { + let name = if display_name == addr { addr.to_owned() } else { format!("{display_name} ({addr})") }; translated(context, StockMessage::SetupContactQRDescription) .await - .replace1(name) + .replace1(&name) } /// Stock string: `Scan to join %1$s`. @@ -1239,6 +1242,18 @@ pub(crate) async fn aeap_explanation_and_link( .replace2(new_addr) } +pub(crate) async fn backup_transfer_qr(context: &Context) -> Result { + let contact = Contact::get_by_id(context, ContactId::SELF).await?; + let addr = contact.get_addr(); + let full_name = match context.get_config(Config::Displayname).await? { + Some(name) if name != addr => format!("{name} ({addr})"), + _ => addr.to_string(), + }; + Ok(translated(context, StockMessage::BackupTransferQr) + .await + .replace1(&full_name)) +} + impl Context { /// Set the stock string for the [StockMessage]. /// From ff0d67508259397cfddd6d072a150f3f65164575 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 12:19:40 +0100 Subject: [PATCH 023/103] Make getting backup use the ongoing process --- deltachat-ffi/src/lib.rs | 2 +- src/imex/transfer.rs | 33 ++++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 74f79cf5d..3658eea17 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4159,7 +4159,7 @@ pub unsafe extern "C" fn dc_backup_provider_qr( provider: *const dc_backup_provider_t, ) -> *mut libc::c_char { let provider = &*provider; - deltachat::qr::format_backup(provider.qr()) + deltachat::qr::format_backup(&provider.qr()) .unwrap_or_default() .strdup() } diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 67808b511..09ec1b63b 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -28,10 +28,6 @@ use std::sync::atomic::{AtomicU16, AtomicU64, Ordering}; use std::sync::Arc; use std::task::Poll; -use crate::chat::delete_and_reset_all_device_msgs; -use crate::context::Context; -use crate::qr::Qr; -use crate::{e2ee, EventType}; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures_lite::StreamExt; @@ -46,6 +42,11 @@ use tokio::sync::broadcast::error::RecvError; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReadDirStream; +use crate::chat::delete_and_reset_all_device_msgs; +use crate::context::Context; +use crate::qr::Qr; +use crate::{e2ee, EventType}; + use super::{export_database, BlobDirContents, DeleteOnDrop, DBFILE_BACKUP_NAME}; /// Provide or send a backup of this device. @@ -273,9 +274,10 @@ impl From for EventType { /// does avoid having [`sendme::provider::Ticket`] in the primary API however, without /// having to revert to untyped bytes. pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { - let Qr::Backup { ticket } = qr else { - bail!("QR code for backup must be of type DCBACKUP"); - }; + ensure!( + matches!(qr, Qr::Backup { .. }), + "QR code for backup must be of type DCBACKUP" + ); ensure!( !context.is_configured().await?, "Cannot import backups to accounts in use." @@ -284,6 +286,23 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { context.scheduler.read().await.is_none(), "cannot import backup, IO is running" ); + + // Acquire global "ongoing" mutex. + let cancel_token = context.alloc_ongoing().await?; + tokio::select! { + biased; + res = get_backup_inner(context, qr) => { + context.free_ongoing().await; + res + } + _ = cancel_token.recv() => Err(format_err!("cancelled")), + } +} + +async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { + let Qr::Backup { ticket } = qr else { + bail!("QR code for backup must be of type DCBACKUP"); + }; let opts = Options { addr: ticket.addr, peer_id: Some(ticket.peer), From 44c1efe4e4070a2b02088b726187cb28899b5d71 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 13:05:54 +0100 Subject: [PATCH 024/103] Add jsonrpc support --- deltachat-ffi/src/lib.rs | 2 +- deltachat-jsonrpc/src/api/mod.rs | 37 ++++++++++++++++++++++++++- deltachat-jsonrpc/src/api/types/qr.rs | 4 +-- deltachat-repl/src/cmdline.rs | 2 +- src/qr_code_generator.rs | 4 +-- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 3658eea17..4a04380e0 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4176,7 +4176,7 @@ pub unsafe extern "C" fn dc_backup_provider_qr_svg( let ctx = &*context; let provider = &*provider; block_on(async move { - generate_backup_qr(ctx, provider.qr()) + generate_backup_qr(ctx, &provider.qr()) .await .unwrap_or_default() .strdup() diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 9df681cb3..a2a1aaf2e 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::path::Path; use std::sync::Arc; use std::{collections::HashMap, str::FromStr}; @@ -22,7 +23,7 @@ use deltachat::{ }, provider::get_provider_info, qr, - qr_code_generator::get_securejoin_qr_svg, + qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg}, reaction::send_reaction, securejoin, stock_str::StockMessage, @@ -1336,6 +1337,40 @@ impl CommandApi { .await } + /// Starts to provide the backup for remote devices to retrieve. + /// + /// Can be cancelled by stopping the ongoing process. Success or failure can be tracked + /// via the `ImexProgress` event which should either reach `1000` for success or `0` for + /// failure. + /// + /// This **stops IO**. After completion `start_io` must be called to restart IO. + /// + /// Returns the QR code as a rendered SVG image. + async fn provide_backup(&self, account_id: u32, path: String) -> Result { + let ctx = self.get_context(account_id).await?; + ctx.stop_io().await; + let provider = imex::BackupProvider::prepare(&ctx, Path::new(&path)).await?; + let qr = provider.qr(); + let svg = match generate_backup_qr(&ctx, &qr).await { + Ok(svg) => svg, + Err(err) => { + ctx.stop_ongoing().await; + return Err(err); + } + }; + Ok(svg) + } + + /// Gets a backup from a remote provider. + /// + /// Can be cancelled by stopping the ongoing process. + async fn get_backup(&self, account_id: u32, qr_text: String) -> Result<()> { + let ctx = self.get_context(account_id).await?; + let qr = qr::check_qr(&ctx, &qr_text).await?; + imex::get_backup(&ctx, qr).await?; + Ok(()) + } + // --------------------------------------------- // connectivity // --------------------------------------------- diff --git a/deltachat-jsonrpc/src/api/types/qr.rs b/deltachat-jsonrpc/src/api/types/qr.rs index 143792477..607b495ec 100644 --- a/deltachat-jsonrpc/src/api/types/qr.rs +++ b/deltachat-jsonrpc/src/api/types/qr.rs @@ -33,7 +33,7 @@ pub enum QrObject { domain: String, }, Backup { - ticket: Vec, + ticket: String, }, WebrtcInstance { domain: String, @@ -130,7 +130,7 @@ impl From for QrObject { Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint }, Qr::Account { domain } => QrObject::Account { domain }, Qr::Backup { ticket } => QrObject::Backup { - ticket: ticket.to_bytes(), + ticket: ticket.to_string(), }, Qr::WebrtcInstance { domain, diff --git a/deltachat-repl/src/cmdline.rs b/deltachat-repl/src/cmdline.rs index d94c04a47..42c7b0595 100644 --- a/deltachat-repl/src/cmdline.rs +++ b/deltachat-repl/src/cmdline.rs @@ -493,7 +493,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu let dir = tdir.path(); let provider = BackupProvider::prepare(&context, dir).await?; let qr = provider.qr(); - let rendered = deltachat::qr_code_generator::generate_backup_qr(&context, qr).await?; + let rendered = deltachat::qr_code_generator::generate_backup_qr(&context, &qr).await?; let file = dir.join("qr.svg"); tokio::fs::write(&file, rendered).await?; println!("The QR code is at: {}", file.display()); diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index c6f384cf5..cb1cc4ce5 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -56,7 +56,7 @@ async fn generate_verification_qr(context: &Context) -> Result { ) } -pub async fn generate_backup_qr(context: &Context, qr: Qr) -> Result { +pub async fn generate_backup_qr(context: &Context, qr: &Qr) -> Result { let Qr::Backup { ticket } = qr else { bail!("QR code not a backup"); }; @@ -325,7 +325,7 @@ mod tests { let qr = provider.qr(); println!("{}", format_backup(&qr).unwrap()); - let rendered = generate_backup_qr(&ctx, qr).await.unwrap(); + let rendered = generate_backup_qr(&ctx, &qr).await.unwrap(); tokio::fs::write(dir.join("qr.svg"), &rendered) .await .unwrap(); From c379a4e5a730b20f8939f5e8e972ad08d504729a Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 13:19:43 +0100 Subject: [PATCH 025/103] use ProgressEmitter from sendme --- Cargo.lock | 9 ++- src/imex/transfer.rs | 133 +------------------------------------------ 2 files changed, 10 insertions(+), 132 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d87cb5a6..9430e4a86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3031,6 +3031,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "portable-atomic" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c00c8683a03bd4fe7db7dd64ab4abee6b42166bc81231da983486ce96be51a" + [[package]] name = "postcard" version = "1.0.2" @@ -3767,7 +3773,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#5bd545d8eb800a48b36c8d7d2333b9f21afec49a" +source = "git+https://github.com/n0-computer/sendme?branch=main#d09a78636b6b280a4ca67c9948ba6325c4c75088" dependencies = [ "anyhow", "bao", @@ -3777,6 +3783,7 @@ dependencies = [ "der", "ed25519-dalek", "futures", + "portable-atomic", "postcard", "quinn", "rand 0.7.3", diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 09ec1b63b..e0888eaa3 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -23,20 +23,17 @@ //! download to an impersonated getter. use std::path::Path; -use std::pin::Pin; -use std::sync::atomic::{AtomicU16, AtomicU64, Ordering}; -use std::sync::Arc; -use std::task::Poll; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures_lite::StreamExt; use sendme::get::{DataStream, Options}; +use sendme::progress::ProgressEmitter; use sendme::protocol::AuthToken; use sendme::provider::{DataSource, Event, Provider, Ticket}; use sendme::Hash; use tokio::fs::{self, File}; -use tokio::io::{self, AsyncRead, AsyncWriteExt, BufWriter}; +use tokio::io::{self, AsyncWriteExt, BufWriter}; use tokio::sync::broadcast; use tokio::sync::broadcast::error::RecvError; use tokio::task::JoinHandle; @@ -454,132 +451,6 @@ impl From for EventType { } } -/// A generic progress event emitter. -/// -/// It is created with a total value to reach and at which increments progress should be -/// emitted. E.g. when downloading a file of any size but you want percentage increments -/// you would create `ProgressEmitter::new(file_size_in_bytes, 100)` and -/// [`ProgressEmitter::subscribe`] will yield numbers `1..100` only. -/// -/// Progress is made by calling [`ProgressEmitter::inc`], which can be implicitly done by -/// [`ProgressEmitter::wrap_async_read`]. -#[derive(Debug, Clone)] -struct ProgressEmitter { - inner: Arc, -} - -impl ProgressEmitter { - /// Creates a new emitter. - /// - /// The emitter expects to see *total* being added via [`ProgressEmitter::inc`] and will - /// emit *steps* updates. - fn new(total: u64, steps: u16) -> Self { - let (tx, _rx) = broadcast::channel(16); - Self { - inner: Arc::new(InnerProgressEmitter { - total: AtomicU64::new(total), - count: AtomicU64::new(0), - steps, - last_step: AtomicU16::new(0u16), - tx, - }), - } - } - - /// Sets a new total in case you did not now the total up front. - fn set_total(&self, value: u64) { - self.inner.set_total(value) - } - - /// Returns a receiver that gets incremental values. - /// - /// The values yielded depend on *steps* passed to [`ProgressEmitter::new`]: it will go - /// from `1..steps`. - fn subscribe(&self) -> broadcast::Receiver { - self.inner.subscribe() - } - - /// Increments the progress by *amount*. - fn inc(&self, amount: u64) { - self.inner.inc(amount); - } - - /// Wraps an [`AsyncRead`] which implicitly calls [`ProgressEmitter::inc`]. - fn wrap_async_read(&self, read: R) -> ProgressAsyncReader { - ProgressAsyncReader { - emitter: self.clone(), - inner: read, - } - } -} - -/// The actual implementation. -/// -/// This exists so it can be Arc'd into [`ProgressEmitter`] and we can easily have multiple -/// `Send + Sync` copies of it. This is used by the -/// [`ProgressEmitter::ProgressAsyncReader`] to update the progress without intertwining -/// lifetimes. -#[derive(Debug)] -struct InnerProgressEmitter { - total: AtomicU64, - count: AtomicU64, - steps: u16, - last_step: AtomicU16, - tx: broadcast::Sender, -} - -impl InnerProgressEmitter { - fn inc(&self, amount: u64) { - let prev_count = self.count.fetch_add(amount, Ordering::Relaxed); - let count = prev_count + amount; - let total = self.total.load(Ordering::Relaxed); - let step = (std::cmp::min(count, total) * u64::from(self.steps) / total) as u16; - let last_step = self.last_step.swap(step, Ordering::Relaxed); - if step > last_step { - self.tx.send(step).ok(); - } - } - - fn set_total(&self, value: u64) { - self.total.store(value, Ordering::Relaxed); - } - - fn subscribe(&self) -> broadcast::Receiver { - self.tx.subscribe() - } -} - -/// A wrapper around [`AsyncRead`] which increments a [`ProgressEmitter`]. -/// -/// This can be used just like the underlying [`AsyncRead`] but increments progress for each -/// byte read. Create this using [`ProgressEmitter::wrap_async_read`]. -#[derive(Debug)] -struct ProgressAsyncReader { - emitter: ProgressEmitter, - inner: R, -} - -impl AsyncRead for ProgressAsyncReader -where - R: AsyncRead + Unpin, -{ - fn poll_read( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &mut io::ReadBuf<'_>, - ) -> Poll> { - let prev_len = buf.filled().len() as u64; - match Pin::new(&mut self.inner).poll_read(cx, buf) { - Poll::Ready(val) => { - let new_len = buf.filled().len() as u64; - self.emitter.inc(new_len - prev_len); - Poll::Ready(val) - } - Poll::Pending => Poll::Pending, - } - } -} - #[cfg(test)] mod tests { use std::time::Duration; From d56e05a11a6bdaf0561e55ec025fc96e4fe24686 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 13:27:15 +0100 Subject: [PATCH 026/103] fixup doc links --- src/blob.rs | 5 +++-- src/imex.rs | 2 +- src/imex/transfer.rs | 2 +- src/qr.rs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/blob.rs b/src/blob.rs index f291fdde5..b6920f6ce 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -242,8 +242,9 @@ impl<'a> BlobObject<'a> { /// including the dot. E.g. "foo.txt" is returned as `("foo", /// ".txt")` while "bar" is returned as `("bar", "")`. /// - /// The extension part will always be lowercased. Note that [`imex::transfer`] relies - /// on this for safety, if uppercase extensions are ever allowed it needs to be adapted. + /// The extension part will always be lowercased. Note that [`crate::imex::get_backup`] + /// relies on this for safety, if uppercase extensions are ever allowed it needs to be + /// adapted. fn sanitise_name(name: &str) -> (String, String) { let mut name = name.to_string(); for part in name.rsplit('/') { diff --git a/src/imex.rs b/src/imex.rs index 34b4caa62..366092170 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -804,7 +804,7 @@ impl<'a> BlobDirContents<'a> { } } -/// A stream for [`Blob`]s. +/// A stream for [`BlobObject`]s. /// /// The stream emits [`EventType::ImexProgress`] events as it being consumed. /// diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index e0888eaa3..1f00d6dc4 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -398,7 +398,7 @@ async fn on_blob( /// Spawns a task proxying progress events. /// -/// This spawns a tokio tasks which receives events from the [`ProgressEmitter`] and sends +/// This spawns a tokio task which receives events from the [`ProgressEmitter`] and sends /// them to the context. The task finishes when the emitter is dropped. /// /// This could be done directly in the emitter by making it less generic. diff --git a/src/qr.rs b/src/qr.rs index a5567d92b..a59036eba 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -166,7 +166,7 @@ pub async fn check_qr(context: &Context, qr: &str) -> Result { /// This is the inverse of [`check_qr`] for that variant only. /// /// TODO: Refactor this so all variants have a correct [`Display`] and transform `check_qr` -/// into [`FromStr`]. +/// into `FromStr`. pub fn format_backup(qr: &Qr) -> Result { match qr { Qr::Backup { ref ticket } => Ok(format!("{DCBACKUP_SCHEME}{ticket}")), From 19f6f893123d623be80c736aa3a7ea5c22e312b3 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 13:29:55 +0100 Subject: [PATCH 027/103] no let else :( --- src/imex/transfer.rs | 5 +++-- src/qr_code_generator.rs | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 1f00d6dc4..105468631 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -297,8 +297,9 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { } async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { - let Qr::Backup { ticket } = qr else { - bail!("QR code for backup must be of type DCBACKUP"); + let ticket = match qr { + Qr::Backup { ticket } => ticket, + _ => bail!("QR code for backup must be of type DCBACKUP"), }; let opts = Options { addr: ticket.addr, diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index cb1cc4ce5..6aba36f6c 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -57,8 +57,9 @@ async fn generate_verification_qr(context: &Context) -> Result { } pub async fn generate_backup_qr(context: &Context, qr: &Qr) -> Result { - let Qr::Backup { ticket } = qr else { - bail!("QR code not a backup"); + let ticket = match qr { + Qr::Backup { ticket } => ticket, + _ => bail!("QR code not a backup"), }; let content = format!("{DCBACKUP_SCHEME}{ticket}"); let (avatar, displayname, _addr, color) = self_info(context).await?; From 020a9d33f65cb278308943d8d902d957e2a9a8a8 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 15:49:21 +0100 Subject: [PATCH 028/103] new sendme for lower msrv --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 9430e4a86..55f51fc4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3773,7 +3773,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#d09a78636b6b280a4ca67c9948ba6325c4c75088" +source = "git+https://github.com/n0-computer/sendme?branch=main#f3086a9576fdc0cdfbd6b0646745bec9e91f7d60" dependencies = [ "anyhow", "bao", From a34a69d8e4f7f35a207fc4dab77a61cc533259ca Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 17:15:52 +0100 Subject: [PATCH 029/103] yes, ci fun --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 4dd08702a..d742d777c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,3 +152,6 @@ vendored = [ "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored" ] + +[patch.'https://github.com/n0-computer/sendme'] +sendme = { branch = "flub/debug-ci" } From c73edd7e215e99b79879e767a3ee0e2d1463b9e0 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 17:20:25 +0100 Subject: [PATCH 030/103] oh --- Cargo.lock | 2 +- Cargo.toml | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 55f51fc4b..9dce43596 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3773,7 +3773,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#f3086a9576fdc0cdfbd6b0646745bec9e91f7d60" +source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#6a9b1cbd9c0beb0cbabd36785486cecc4529cff2" dependencies = [ "anyhow", "bao", diff --git a/Cargo.toml b/Cargo.toml index d742d777c..5ea016b71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "flub/debug-ci", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" @@ -152,6 +152,3 @@ vendored = [ "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored" ] - -[patch.'https://github.com/n0-computer/sendme'] -sendme = { branch = "flub/debug-ci" } From b413593c4333d3c9d9058ea3eac395e35a9dae28 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 17:39:58 +0100 Subject: [PATCH 031/103] hi --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 9dce43596..9bb714045 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3773,7 +3773,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#6a9b1cbd9c0beb0cbabd36785486cecc4529cff2" +source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#1ce30fcd21dc683dfb0e4c5e846594b9fccc04a3" dependencies = [ "anyhow", "bao", From 2a754744fe8fa9a5968c390de72b2df7fe26159c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 18:03:39 +0100 Subject: [PATCH 032/103] char, not chat --- deltachat-ffi/deltachat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index c320cea16..8828e388b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2666,7 +2666,7 @@ void dc_str_unref (char* str); * On errors, NULL is returned and dc_get_last_error()returns an error that * should be shown to the user. */ -dc_backup_provider_t* dc_provide_backup (dc_context_t* context, const chat* folder); +dc_backup_provider_t* dc_provide_backup (dc_context_t* context, const char* folder); /** From 1c2461974df6bec752da446a5eca8f4a92ca66d8 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 14 Feb 2023 18:29:15 +0100 Subject: [PATCH 033/103] better way --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 9bb714045..dd951723f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3773,7 +3773,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#1ce30fcd21dc683dfb0e4c5e846594b9fccc04a3" +source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#8d22c978ed9a0ac4e51fe67912f5e2ea5250c416" dependencies = [ "anyhow", "bao", From 7a0d61bbb00321f28a8998943e8f6051aef68a92 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 15 Feb 2023 13:50:39 +0100 Subject: [PATCH 034/103] hey --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd951723f..f58d338ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3773,7 +3773,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#8d22c978ed9a0ac4e51fe67912f5e2ea5250c416" +source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci2#b50020f0ebca744514b1c5270259a2aef8d6dad5" dependencies = [ "anyhow", "bao", diff --git a/Cargo.toml b/Cargo.toml index 5ea016b71..a28f5f5a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "flub/debug-ci", default-features = false } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "flub/debug-ci2", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" From ec5789997aa25995b4379bdc0f3bd2e2f8b92e82 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 15 Feb 2023 14:45:52 +0100 Subject: [PATCH 035/103] back to master --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a28f5f5a6..4b45d003e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "flub/debug-ci2", default-features = false } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "master", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" From 6f7a43804dc2536eca9007e85d1f67eff219c9cf Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 15 Feb 2023 14:48:17 +0100 Subject: [PATCH 036/103] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21bafce20..b8f54fe29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changes - deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042 +- ability to send backup over network and QR code to setup second device #4007 ### Fixes - deltachat-rpc-server: do not block stdin while processing the request. #4041 From daf56804a5aa080385de3adfac1979c9594b197f Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 15 Feb 2023 14:57:26 +0100 Subject: [PATCH 037/103] use correct branch --- Cargo.lock | 27 ++++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31f6bd7e0..2097d20aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,18 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "abao" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19aae7996efbfa033c10ce360f73c87e70af3401282e724cd04346973c36069" +dependencies = [ + "arrayref", + "arrayvec", + "blake3", + "tokio", +] + [[package]] name = "addr2line" version = "0.19.0" @@ -404,17 +416,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "bao" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3663af8a469bf3a2d818b8c1a7b62e4dc55c4547b4c14f5dab35fc9e707bef83" -dependencies = [ - "arrayref", - "arrayvec", - "blake3", -] - [[package]] name = "base16ct" version = "0.1.1" @@ -3773,10 +3774,10 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci2#b50020f0ebca744514b1c5270259a2aef8d6dad5" +source = "git+https://github.com/n0-computer/sendme?branch=main#21a5cfd47e5c6de968ca5882838e24b08572412d" dependencies = [ + "abao", "anyhow", - "bao", "base64 0.21.0", "blake3", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 2d8f812bb..62fc9b73a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "master", default-features = false } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" From 41319c85c7532239a0eb7a9a02dc2cf48db3b7a8 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 15 Feb 2023 15:12:14 +0100 Subject: [PATCH 038/103] patch in previous revision of sendme main broke rust 1.63 support :'( --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 2097d20aa..5458faca5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3774,7 +3774,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#21a5cfd47e5c6de968ca5882838e24b08572412d" +source = "git+https://github.com/n0-computer/sendme?branch=main#c0fafdbcbac6e1e5578a9dde5ee2061150e2486d" dependencies = [ "abao", "anyhow", From e565e19b4229d4dd7e4e458bfaaec3c843ea0194 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 15 Feb 2023 16:01:39 +0100 Subject: [PATCH 039/103] fix msrv in sendme --- Cargo.lock | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5458faca5..a0c93e2af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,13 +10,14 @@ checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" [[package]] name = "abao" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19aae7996efbfa033c10ce360f73c87e70af3401282e724cd04346973c36069" +checksum = "fa67f56b146a784d8feb9f5e4dc5e309fdbf7e3f6631a9c6d709348536abc628" dependencies = [ "arrayref", "arrayvec", "blake3", + "futures", "tokio", ] @@ -3774,7 +3775,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#c0fafdbcbac6e1e5578a9dde5ee2061150e2486d" +source = "git+https://github.com/n0-computer/sendme?branch=main#2edb528007dd773e7f00fd4dd460506c8abf1681" dependencies = [ "abao", "anyhow", From bb9a3d4b8ec4ebaffc3dd7bdb5a7e35629f3f1ed Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 09:00:27 +0100 Subject: [PATCH 040/103] more bug hunting: disable most ci, point to branch --- .github/workflows/ci.yml | 92 +++++++++---------- .github/workflows/dependabot.yml | 2 +- .../workflows/jsonrpc-client-npm-package.yml | 12 +-- .github/workflows/jsonrpc.yml | 10 +- .github/workflows/node-delete-preview.yml | 6 +- .github/workflows/node-docs.yml | 8 +- .github/workflows/node-package.yml | 12 +-- .github/workflows/node-tests.yml | 14 +-- .github/workflows/upload-docs.yml | 10 +- .github/workflows/upload-ffi-docs.yml | 10 +- Cargo.lock | 2 +- Cargo.toml | 2 +- 12 files changed, 90 insertions(+), 90 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4000e7296..4341078c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,58 +84,58 @@ jobs: - name: Tests run: cargo test --workspace - - name: Test cargo vendor - run: cargo vendor + # - name: Test cargo vendor + # run: cargo vendor - - name: Install python - if: ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} + # - name: Install python + # if: ${{ matrix.python }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python }} - - name: Install tox - if: ${{ matrix.python }} - run: pip install tox + # - name: Install tox + # if: ${{ matrix.python }} + # run: pip install tox - - name: Build C library - if: ${{ matrix.python }} - run: cargo build -p deltachat_ffi --features jsonrpc + # - name: Build C library + # if: ${{ matrix.python }} + # run: cargo build -p deltachat_ffi --features jsonrpc - - name: Run python tests - if: ${{ matrix.python }} - env: - DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} - DCC_RS_TARGET: debug - DCC_RS_DEV: ${{ github.workspace }} - working-directory: python - run: tox -e lint,mypy,doc,py3 + # - name: Run python tests + # if: ${{ matrix.python }} + # env: + # DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} + # DCC_RS_TARGET: debug + # DCC_RS_DEV: ${{ github.workspace }} + # working-directory: python + # run: tox -e lint,mypy,doc,py3 - - name: Build deltachat-rpc-server - if: ${{ matrix.python }} - run: cargo build -p deltachat-rpc-server + # - name: Build deltachat-rpc-server + # if: ${{ matrix.python }} + # run: cargo build -p deltachat-rpc-server - - name: Add deltachat-rpc-server to path - if: ${{ matrix.python }} - run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH + # - name: Add deltachat-rpc-server to path + # if: ${{ matrix.python }} + # run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH - - name: Run deltachat-rpc-client tests - if: ${{ matrix.python }} - env: - DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} - working-directory: deltachat-rpc-client - run: tox -e py3,lint + # - name: Run deltachat-rpc-client tests + # if: ${{ matrix.python }} + # env: + # DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} + # working-directory: deltachat-rpc-client + # run: tox -e py3,lint - - name: Install pypy - if: ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: 'pypy${{ matrix.python }}' + # - name: Install pypy + # if: ${{ matrix.python }} + # uses: actions/setup-python@v4 + # with: + # python-version: 'pypy${{ matrix.python }}' - - name: Run pypy tests - if: ${{ matrix.python }} - env: - DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} - DCC_RS_TARGET: debug - DCC_RS_DEV: ${{ github.workspace }} - working-directory: python - run: tox -e pypy3 + # - name: Run pypy tests + # if: ${{ matrix.python }} + # env: + # DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} + # DCC_RS_TARGET: debug + # DCC_RS_DEV: ${{ github.workspace }} + # working-directory: python + # run: tox -e pypy3 diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index 174f6096e..426d0a4db 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -1,5 +1,5 @@ name: Dependabot auto-approve -on: pull_request +# on: pull_request permissions: pull-requests: write diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index a1c3b4b82..e4577e60b 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -1,10 +1,10 @@ name: 'jsonrpc js client build' -on: - pull_request: - push: - tags: - - '*' - - '!py-*' +# on: +# pull_request: +# push: +# tags: +# - '*' +# - '!py-*' jobs: diff --git a/.github/workflows/jsonrpc.yml b/.github/workflows/jsonrpc.yml index de05f50d6..b422a01b7 100644 --- a/.github/workflows/jsonrpc.yml +++ b/.github/workflows/jsonrpc.yml @@ -1,10 +1,10 @@ name: JSON-RPC API Test -on: - push: - branches: [master] - pull_request: - branches: [master] +# on: +# push: +# branches: [master] +# pull_request: +# branches: [master] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/node-delete-preview.yml b/.github/workflows/node-delete-preview.yml index ba698c846..7a67b4448 100644 --- a/.github/workflows/node-delete-preview.yml +++ b/.github/workflows/node-delete-preview.yml @@ -1,9 +1,9 @@ # documentation: https://github.com/deltachat/sysadmin/tree/master/download.delta.chat name: Delete node PR previews -on: - pull_request: - types: [closed] +# on: +# pull_request: +# types: [closed] jobs: delete: diff --git a/.github/workflows/node-docs.yml b/.github/workflows/node-docs.yml index a7c07e0d9..2d790e046 100644 --- a/.github/workflows/node-docs.yml +++ b/.github/workflows/node-docs.yml @@ -1,9 +1,9 @@ name: Generate & upload node.js documentation -on: - push: - branches: - - master +# on: +# push: +# branches: +# - master jobs: generate: diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index f0cb1cc09..fd1ba6ff1 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -1,10 +1,10 @@ name: 'node.js build' -on: - pull_request: - push: - tags: - - '*' - - '!py-*' +# on: +# pull_request: +# push: +# tags: +# - '*' +# - '!py-*' jobs: diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index e0d557d42..7f59c2805 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,11 +1,11 @@ name: 'node.js tests' -on: - pull_request: - push: - branches: - - master - - staging - - trying +# on: +# pull_request: +# push: +# branches: +# - master +# - staging +# - trying jobs: tests: diff --git a/.github/workflows/upload-docs.yml b/.github/workflows/upload-docs.yml index c624cb081..3c7d7a31f 100644 --- a/.github/workflows/upload-docs.yml +++ b/.github/workflows/upload-docs.yml @@ -1,10 +1,10 @@ name: Build & Deploy Documentation on rs.delta.chat -on: - push: - branches: - - master - - docs-gh-action +# on: +# push: +# branches: +# - master +# - docs-gh-action jobs: build: diff --git a/.github/workflows/upload-ffi-docs.yml b/.github/workflows/upload-ffi-docs.yml index 73fa07cb1..1c7facf67 100644 --- a/.github/workflows/upload-ffi-docs.yml +++ b/.github/workflows/upload-ffi-docs.yml @@ -1,10 +1,10 @@ name: Build & Deploy Documentation on cffi.delta.chat -on: - push: - branches: - - master - - docs-gh-action +# on: +# push: +# branches: +# - master +# - docs-gh-action jobs: build: diff --git a/Cargo.lock b/Cargo.lock index a0c93e2af..73eb91e26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3775,7 +3775,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#2edb528007dd773e7f00fd4dd460506c8abf1681" +source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#fc368a5b3070029488c0380c37eec1fc92f972eb" dependencies = [ "abao", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 62fc9b73a..1e7a14788 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "flub/debug-ci", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" From b2266ffca10dab9a97d7502a57ae5b4119b1e0ad Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 09:06:40 +0100 Subject: [PATCH 041/103] make the have a valid on spec at least so gh doesn't complain too much --- .github/workflows/dependabot.yml | 5 ++++- .github/workflows/jsonrpc-client-npm-package.yml | 4 +++- .github/workflows/jsonrpc.yml | 3 ++- .github/workflows/node-delete-preview.yml | 3 ++- .github/workflows/node-docs.yml | 3 ++- .github/workflows/node-package.yml | 4 +++- .github/workflows/node-tests.yml | 4 +++- .github/workflows/upload-docs.yml | 3 ++- .github/workflows/upload-ffi-docs.yml | 3 ++- 9 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index 426d0a4db..9f724aaf4 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -1,5 +1,8 @@ name: Dependabot auto-approve -# on: pull_request + +on: + workflow_dispatch + pull_request permissions: pull-requests: write diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index e4577e60b..739a73843 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -1,5 +1,7 @@ name: 'jsonrpc js client build' -# on: + +on: + workflow_dispatch # pull_request: # push: # tags: diff --git a/.github/workflows/jsonrpc.yml b/.github/workflows/jsonrpc.yml index b422a01b7..828e3242f 100644 --- a/.github/workflows/jsonrpc.yml +++ b/.github/workflows/jsonrpc.yml @@ -1,6 +1,7 @@ name: JSON-RPC API Test -# on: +on: + workflow_dispatch # push: # branches: [master] # pull_request: diff --git a/.github/workflows/node-delete-preview.yml b/.github/workflows/node-delete-preview.yml index 7a67b4448..ba39f1295 100644 --- a/.github/workflows/node-delete-preview.yml +++ b/.github/workflows/node-delete-preview.yml @@ -1,7 +1,8 @@ # documentation: https://github.com/deltachat/sysadmin/tree/master/download.delta.chat name: Delete node PR previews -# on: +on: + workflow_dispatch # pull_request: # types: [closed] diff --git a/.github/workflows/node-docs.yml b/.github/workflows/node-docs.yml index 2d790e046..187c509db 100644 --- a/.github/workflows/node-docs.yml +++ b/.github/workflows/node-docs.yml @@ -1,6 +1,7 @@ name: Generate & upload node.js documentation -# on: +on: + workflow_dispatch # push: # branches: # - master diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index fd1ba6ff1..7371470e1 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -1,5 +1,7 @@ name: 'node.js build' -# on: + +on: + workflow_dispatch # pull_request: # push: # tags: diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 7f59c2805..35e6cd44a 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,5 +1,7 @@ name: 'node.js tests' -# on: + +on: + workflow_dispatch # pull_request: # push: # branches: diff --git a/.github/workflows/upload-docs.yml b/.github/workflows/upload-docs.yml index 3c7d7a31f..4199ae34c 100644 --- a/.github/workflows/upload-docs.yml +++ b/.github/workflows/upload-docs.yml @@ -1,6 +1,7 @@ name: Build & Deploy Documentation on rs.delta.chat -# on: +on: + workflow_dispatch # push: # branches: # - master diff --git a/.github/workflows/upload-ffi-docs.yml b/.github/workflows/upload-ffi-docs.yml index 1c7facf67..f93a5a3cf 100644 --- a/.github/workflows/upload-ffi-docs.yml +++ b/.github/workflows/upload-ffi-docs.yml @@ -1,6 +1,7 @@ name: Build & Deploy Documentation on cffi.delta.chat -# on: +on: + workflow_dispatch # push: # branches: # - master From 904990bf914c3223ad33f469b183115a9df47396 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 09:08:14 +0100 Subject: [PATCH 042/103] ugh, yaml syntax --- .github/workflows/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index 9f724aaf4..c0026d45e 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -1,8 +1,8 @@ name: Dependabot auto-approve on: - workflow_dispatch - pull_request + workflow_dispatch: + pull_request: permissions: pull-requests: write From f9f9bc3efb3eec62fcb23d8d966392ee6a9fa8c3 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 09:10:47 +0100 Subject: [PATCH 043/103] yaml --- .github/workflows/jsonrpc-client-npm-package.yml | 2 +- .github/workflows/jsonrpc.yml | 2 +- .github/workflows/node-delete-preview.yml | 2 +- .github/workflows/node-docs.yml | 2 +- .github/workflows/node-package.yml | 2 +- .github/workflows/node-tests.yml | 2 +- .github/workflows/upload-docs.yml | 2 +- .github/workflows/upload-ffi-docs.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index 739a73843..66e742390 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -1,7 +1,7 @@ name: 'jsonrpc js client build' on: - workflow_dispatch + workflow_dispatch: # pull_request: # push: # tags: diff --git a/.github/workflows/jsonrpc.yml b/.github/workflows/jsonrpc.yml index 828e3242f..1aad0a22c 100644 --- a/.github/workflows/jsonrpc.yml +++ b/.github/workflows/jsonrpc.yml @@ -1,7 +1,7 @@ name: JSON-RPC API Test on: - workflow_dispatch + workflow_dispatch: # push: # branches: [master] # pull_request: diff --git a/.github/workflows/node-delete-preview.yml b/.github/workflows/node-delete-preview.yml index ba39f1295..89023410f 100644 --- a/.github/workflows/node-delete-preview.yml +++ b/.github/workflows/node-delete-preview.yml @@ -2,7 +2,7 @@ name: Delete node PR previews on: - workflow_dispatch + workflow_dispatch: # pull_request: # types: [closed] diff --git a/.github/workflows/node-docs.yml b/.github/workflows/node-docs.yml index 187c509db..f1b6b261b 100644 --- a/.github/workflows/node-docs.yml +++ b/.github/workflows/node-docs.yml @@ -1,7 +1,7 @@ name: Generate & upload node.js documentation on: - workflow_dispatch + workflow_dispatch: # push: # branches: # - master diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index 7371470e1..f084c6981 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -1,7 +1,7 @@ name: 'node.js build' on: - workflow_dispatch + workflow_dispatch: # pull_request: # push: # tags: diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 35e6cd44a..b1c704891 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,7 +1,7 @@ name: 'node.js tests' on: - workflow_dispatch + workflow_dispatch: # pull_request: # push: # branches: diff --git a/.github/workflows/upload-docs.yml b/.github/workflows/upload-docs.yml index 4199ae34c..68a4fd3fa 100644 --- a/.github/workflows/upload-docs.yml +++ b/.github/workflows/upload-docs.yml @@ -1,7 +1,7 @@ name: Build & Deploy Documentation on rs.delta.chat on: - workflow_dispatch + workflow_dispatch: # push: # branches: # - master diff --git a/.github/workflows/upload-ffi-docs.yml b/.github/workflows/upload-ffi-docs.yml index f93a5a3cf..255532c79 100644 --- a/.github/workflows/upload-ffi-docs.yml +++ b/.github/workflows/upload-ffi-docs.yml @@ -1,7 +1,7 @@ name: Build & Deploy Documentation on cffi.delta.chat on: - workflow_dispatch + workflow_dispatch: # push: # branches: # - master From 7a9f497aa7bc985632ebdbdb663dc3c26cf043cc Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 14:49:04 +0100 Subject: [PATCH 044/103] why can't i see this action now? --- .github/workflows/ci.yml | 92 ++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4341078c5..4000e7296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,58 +84,58 @@ jobs: - name: Tests run: cargo test --workspace - # - name: Test cargo vendor - # run: cargo vendor + - name: Test cargo vendor + run: cargo vendor - # - name: Install python - # if: ${{ matrix.python }} - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ matrix.python }} + - name: Install python + if: ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} - # - name: Install tox - # if: ${{ matrix.python }} - # run: pip install tox + - name: Install tox + if: ${{ matrix.python }} + run: pip install tox - # - name: Build C library - # if: ${{ matrix.python }} - # run: cargo build -p deltachat_ffi --features jsonrpc + - name: Build C library + if: ${{ matrix.python }} + run: cargo build -p deltachat_ffi --features jsonrpc - # - name: Run python tests - # if: ${{ matrix.python }} - # env: - # DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} - # DCC_RS_TARGET: debug - # DCC_RS_DEV: ${{ github.workspace }} - # working-directory: python - # run: tox -e lint,mypy,doc,py3 + - name: Run python tests + if: ${{ matrix.python }} + env: + DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} + DCC_RS_TARGET: debug + DCC_RS_DEV: ${{ github.workspace }} + working-directory: python + run: tox -e lint,mypy,doc,py3 - # - name: Build deltachat-rpc-server - # if: ${{ matrix.python }} - # run: cargo build -p deltachat-rpc-server + - name: Build deltachat-rpc-server + if: ${{ matrix.python }} + run: cargo build -p deltachat-rpc-server - # - name: Add deltachat-rpc-server to path - # if: ${{ matrix.python }} - # run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH + - name: Add deltachat-rpc-server to path + if: ${{ matrix.python }} + run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH - # - name: Run deltachat-rpc-client tests - # if: ${{ matrix.python }} - # env: - # DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} - # working-directory: deltachat-rpc-client - # run: tox -e py3,lint + - name: Run deltachat-rpc-client tests + if: ${{ matrix.python }} + env: + DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} + working-directory: deltachat-rpc-client + run: tox -e py3,lint - # - name: Install pypy - # if: ${{ matrix.python }} - # uses: actions/setup-python@v4 - # with: - # python-version: 'pypy${{ matrix.python }}' + - name: Install pypy + if: ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: 'pypy${{ matrix.python }}' - # - name: Run pypy tests - # if: ${{ matrix.python }} - # env: - # DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} - # DCC_RS_TARGET: debug - # DCC_RS_DEV: ${{ github.workspace }} - # working-directory: python - # run: tox -e pypy3 + - name: Run pypy tests + if: ${{ matrix.python }} + env: + DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }} + DCC_RS_TARGET: debug + DCC_RS_DEV: ${{ github.workspace }} + working-directory: python + run: tox -e pypy3 From a53d30c4590bc8abde929beb43b0a87453e66f3e Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 14:49:48 +0100 Subject: [PATCH 045/103] fixed another bug, try main again --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73eb91e26..a1941f68a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3775,7 +3775,7 @@ dependencies = [ [[package]] name = "sendme" version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=flub/debug-ci#fc368a5b3070029488c0380c37eec1fc92f972eb" +source = "git+https://github.com/n0-computer/sendme?branch=main#153073a93740a214731181e42d69328652081cda" dependencies = [ "abao", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 1e7a14788..62fc9b73a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "flub/debug-ci", default-features = false } +sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" From bca0b256c9f81fbf7d1dfd7a720006f50fa1a1eb Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 14:52:05 +0100 Subject: [PATCH 046/103] goodness ci? --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4000e7296..4b552b4b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: Rust CI on: + workflow_dispatch: pull_request: push: branches: From 0bae3caaffcded030194f8b33bd9b8500e9ae28a Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 14:55:24 +0100 Subject: [PATCH 047/103] dear CI masters: i regret every trying to be clever --- .github/workflows/ci.yml | 1 - .github/workflows/dependabot.yml | 5 +---- .github/workflows/jsonrpc-client-npm-package.yml | 11 +++++------ .github/workflows/jsonrpc.yml | 9 ++++----- .github/workflows/node-delete-preview.yml | 5 ++--- .github/workflows/node-docs.yml | 7 +++---- .github/workflows/node-package.yml | 11 +++++------ .github/workflows/node-tests.yml | 13 ++++++------- .github/workflows/upload-docs.yml | 9 ++++----- .github/workflows/upload-ffi-docs.yml | 9 ++++----- 10 files changed, 34 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b552b4b7..4000e7296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,6 @@ name: Rust CI on: - workflow_dispatch: pull_request: push: branches: diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index c0026d45e..174f6096e 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -1,8 +1,5 @@ name: Dependabot auto-approve - -on: - workflow_dispatch: - pull_request: +on: pull_request permissions: pull-requests: write diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index 66e742390..8d7d39600 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -1,12 +1,11 @@ name: 'jsonrpc js client build' on: - workflow_dispatch: -# pull_request: -# push: -# tags: -# - '*' -# - '!py-*' + pull_request: + push: + tags: + - '*' + - '!py-*' jobs: diff --git a/.github/workflows/jsonrpc.yml b/.github/workflows/jsonrpc.yml index 1aad0a22c..de05f50d6 100644 --- a/.github/workflows/jsonrpc.yml +++ b/.github/workflows/jsonrpc.yml @@ -1,11 +1,10 @@ name: JSON-RPC API Test on: - workflow_dispatch: -# push: -# branches: [master] -# pull_request: -# branches: [master] + push: + branches: [master] + pull_request: + branches: [master] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/node-delete-preview.yml b/.github/workflows/node-delete-preview.yml index 89023410f..ba698c846 100644 --- a/.github/workflows/node-delete-preview.yml +++ b/.github/workflows/node-delete-preview.yml @@ -2,9 +2,8 @@ name: Delete node PR previews on: - workflow_dispatch: -# pull_request: -# types: [closed] + pull_request: + types: [closed] jobs: delete: diff --git a/.github/workflows/node-docs.yml b/.github/workflows/node-docs.yml index f1b6b261b..a7c07e0d9 100644 --- a/.github/workflows/node-docs.yml +++ b/.github/workflows/node-docs.yml @@ -1,10 +1,9 @@ name: Generate & upload node.js documentation on: - workflow_dispatch: -# push: -# branches: -# - master + push: + branches: + - master jobs: generate: diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index f084c6981..5b90ee974 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -1,12 +1,11 @@ name: 'node.js build' on: - workflow_dispatch: -# pull_request: -# push: -# tags: -# - '*' -# - '!py-*' + pull_request: + push: + tags: + - '*' + - '!py-*' jobs: diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index b1c704891..705d91152 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,13 +1,12 @@ name: 'node.js tests' on: - workflow_dispatch: -# pull_request: -# push: -# branches: -# - master -# - staging -# - trying + pull_request: + push: + branches: + - master + - staging + - trying jobs: tests: diff --git a/.github/workflows/upload-docs.yml b/.github/workflows/upload-docs.yml index 68a4fd3fa..c624cb081 100644 --- a/.github/workflows/upload-docs.yml +++ b/.github/workflows/upload-docs.yml @@ -1,11 +1,10 @@ name: Build & Deploy Documentation on rs.delta.chat on: - workflow_dispatch: -# push: -# branches: -# - master -# - docs-gh-action + push: + branches: + - master + - docs-gh-action jobs: build: diff --git a/.github/workflows/upload-ffi-docs.yml b/.github/workflows/upload-ffi-docs.yml index 255532c79..73fa07cb1 100644 --- a/.github/workflows/upload-ffi-docs.yml +++ b/.github/workflows/upload-ffi-docs.yml @@ -1,11 +1,10 @@ name: Build & Deploy Documentation on cffi.delta.chat on: - workflow_dispatch: -# push: -# branches: -# - master -# - docs-gh-action + push: + branches: + - master + - docs-gh-action jobs: build: From 7cf0820d2bab7085f596959b96bce5d9e09221f3 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 14:56:18 +0100 Subject: [PATCH 048/103] diff --- .github/workflows/jsonrpc-client-npm-package.yml | 1 - .github/workflows/node-package.yml | 1 - .github/workflows/node-tests.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/jsonrpc-client-npm-package.yml b/.github/workflows/jsonrpc-client-npm-package.yml index 8d7d39600..a1c3b4b82 100644 --- a/.github/workflows/jsonrpc-client-npm-package.yml +++ b/.github/workflows/jsonrpc-client-npm-package.yml @@ -1,5 +1,4 @@ name: 'jsonrpc js client build' - on: pull_request: push: diff --git a/.github/workflows/node-package.yml b/.github/workflows/node-package.yml index 5b90ee974..f0cb1cc09 100644 --- a/.github/workflows/node-package.yml +++ b/.github/workflows/node-package.yml @@ -1,5 +1,4 @@ name: 'node.js build' - on: pull_request: push: diff --git a/.github/workflows/node-tests.yml b/.github/workflows/node-tests.yml index 705d91152..e0d557d42 100644 --- a/.github/workflows/node-tests.yml +++ b/.github/workflows/node-tests.yml @@ -1,5 +1,4 @@ name: 'node.js tests' - on: pull_request: push: From dcce6ef50b5ab57b1d169b1b0ca602add75f0759 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 15:19:44 +0100 Subject: [PATCH 049/103] Some docs --- deltachat-jsonrpc/src/api/types/qr.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deltachat-jsonrpc/src/api/types/qr.rs b/deltachat-jsonrpc/src/api/types/qr.rs index 607b495ec..b606e33f4 100644 --- a/deltachat-jsonrpc/src/api/types/qr.rs +++ b/deltachat-jsonrpc/src/api/types/qr.rs @@ -129,7 +129,17 @@ impl From for QrObject { } Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint }, Qr::Account { domain } => QrObject::Account { domain }, + /// Provides a backup that can be retrieve. + /// + /// This contains all the data needed to connect to a device and download a + /// backup from it to configure the receiving device with the same account. Qr::Backup { ticket } => QrObject::Backup { + /// Printable version of the provider information. + /// + /// This is the printable version of a `sendme` ticket, which contains all + /// the information to connect to and authenticate a backup provider. + /// + /// The format is somewhat opaque, but `sendme` can deserialise this. ticket: ticket.to_string(), }, Qr::WebrtcInstance { From 490a14c5ef70d8ba911278d4194467c39ab1ae6c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 16:06:41 +0100 Subject: [PATCH 050/103] Remove the need for a directory for db export Plus on import use the context directory. We can actually write there just fine. --- deltachat-ffi/deltachat.h | 5 +-- deltachat-ffi/src/lib.rs | 4 +-- deltachat-jsonrpc/src/api/mod.rs | 5 ++- deltachat-jsonrpc/src/api/types/qr.rs | 10 ------ deltachat-repl/src/cmdline.rs | 6 ++-- src/blob.rs | 4 +-- src/imex/transfer.rs | 50 ++++++++++++++++----------- src/qr.rs | 10 ++++++ src/qr_code_generator.rs | 2 +- 9 files changed, 48 insertions(+), 48 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 1b1187bd7..987e0d3e4 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2659,14 +2659,11 @@ void dc_str_unref (char* str); * * @memberof dc_backup_sender_t * @param context The context. - * @param folder A Path to a temporary directory where the encrypted database - * export will be created. The directory is not automatically cleaned - * after the backup is sent. * @return Opaque object for sending the backup. * On errors, NULL is returned and dc_get_last_error()returns an error that * should be shown to the user. */ -dc_backup_provider_t* dc_provide_backup (dc_context_t* context, const char* folder); +dc_backup_provider_t* dc_provide_backup (dc_context_t* context); /** diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 4a04380e0..34a23b2f4 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4136,16 +4136,14 @@ pub type dc_backup_provider_t = BackupProvider; #[no_mangle] pub unsafe extern "C" fn dc_provide_backup( context: *mut dc_context_t, - folder: *const libc::c_char, ) -> *mut dc_backup_provider_t { if context.is_null() { eprintln!("ignoring careless call to dc_send_backup()"); return ptr::null_mut(); } let ctx = &*context; - let dir = as_path(folder); block_on(async move { - BackupProvider::prepare(ctx, dir) + BackupProvider::prepare(ctx) .await .map(|provider| Box::into_raw(Box::new(provider))) .log_err(ctx, "BackupProvider failed") diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index ddb4d193f..83e021afd 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1,5 +1,4 @@ use std::collections::BTreeMap; -use std::path::Path; use std::sync::Arc; use std::{collections::HashMap, str::FromStr}; @@ -1351,10 +1350,10 @@ impl CommandApi { /// This **stops IO**. After completion `start_io` must be called to restart IO. /// /// Returns the QR code as a rendered SVG image. - async fn provide_backup(&self, account_id: u32, path: String) -> Result { + async fn provide_backup(&self, account_id: u32) -> Result { let ctx = self.get_context(account_id).await?; ctx.stop_io().await; - let provider = imex::BackupProvider::prepare(&ctx, Path::new(&path)).await?; + let provider = imex::BackupProvider::prepare(&ctx).await?; let qr = provider.qr(); let svg = match generate_backup_qr(&ctx, &qr).await { Ok(svg) => svg, diff --git a/deltachat-jsonrpc/src/api/types/qr.rs b/deltachat-jsonrpc/src/api/types/qr.rs index b606e33f4..607b495ec 100644 --- a/deltachat-jsonrpc/src/api/types/qr.rs +++ b/deltachat-jsonrpc/src/api/types/qr.rs @@ -129,17 +129,7 @@ impl From for QrObject { } Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint }, Qr::Account { domain } => QrObject::Account { domain }, - /// Provides a backup that can be retrieve. - /// - /// This contains all the data needed to connect to a device and download a - /// backup from it to configure the receiving device with the same account. Qr::Backup { ticket } => QrObject::Backup { - /// Printable version of the provider information. - /// - /// This is the printable version of a `sendme` ticket, which contains all - /// the information to connect to and authenticate a backup provider. - /// - /// The format is somewhat opaque, but `sendme` can deserialise this. ticket: ticket.to_string(), }, Qr::WebrtcInstance { diff --git a/deltachat-repl/src/cmdline.rs b/deltachat-repl/src/cmdline.rs index 42c7b0595..5dd1b6783 100644 --- a/deltachat-repl/src/cmdline.rs +++ b/deltachat-repl/src/cmdline.rs @@ -489,11 +489,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu .await?; } "send-backup" => { - let tdir = tempfile::TempDir::new()?; - let dir = tdir.path(); - let provider = BackupProvider::prepare(&context, dir).await?; + let provider = BackupProvider::prepare(&context).await?; let qr = provider.qr(); let rendered = deltachat::qr_code_generator::generate_backup_qr(&context, &qr).await?; + let tdir = tempfile::TempDir::new()?; + let dir = tdir.path(); let file = dir.join("qr.svg"); tokio::fs::write(&file, rendered).await?; println!("The QR code is at: {}", file.display()); diff --git a/src/blob.rs b/src/blob.rs index b6920f6ce..ef2869bed 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -242,9 +242,7 @@ impl<'a> BlobObject<'a> { /// including the dot. E.g. "foo.txt" is returned as `("foo", /// ".txt")` while "bar" is returned as `("bar", "")`. /// - /// The extension part will always be lowercased. Note that [`crate::imex::get_backup`] - /// relies on this for safety, if uppercase extensions are ever allowed it needs to be - /// adapted. + /// The extension part will always be lowercased. fn sanitise_name(name: &str) -> (String, String) { let mut name = name.to_string(); for part in name.rsplit('/') { diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 105468631..24e2f6fd7 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -22,7 +22,7 @@ //! getter can not connect to an impersonated provider and the provider does not offer the //! download to an impersonated getter. -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; @@ -76,21 +76,25 @@ impl BackupProvider { /// the possible cancellation of the "ongoing process". /// /// [`Accounts::stop_io`]: crate::accounts::Accounts::stop_io - pub async fn prepare(context: &Context, dir: &Path) -> Result { - ensure!( - // TODO: Should we worry about path normalisation? - dir != context.get_blobdir(), - "Temporary database export directory should not be in blobdir" - ); + pub async fn prepare(context: &Context) -> Result { e2ee::ensure_secret_key_exists(context) .await .context("Private key not available, aborting backup export")?; // Acquire global "ongoing" mutex. let cancel_token = context.alloc_ongoing().await?; + let context_dir = context + .get_blobdir() + .parent() + .ok_or(anyhow!("Context dir not found"))?; + let dbfile = context_dir.join(DBFILE_BACKUP_NAME); + if fs::metadata(&dbfile).await.is_ok() { + fs::remove_file(&dbfile).await?; + warn!(context, "Previous database export deleted"); + } let res = tokio::select! { biased; - res = Self::prepare_inner(context, dir) => { + res = Self::prepare_inner(context, &dbfile) => { match res { Ok(slf) => Ok(slf), Err(err) => { @@ -112,6 +116,7 @@ impl BackupProvider { context.clone(), provider, cancel_token, + dbfile, )); Ok(Self { handle, ticket }) } @@ -119,11 +124,10 @@ impl BackupProvider { /// Creates the provider task. /// /// Having this as a function makes it easier to cancel it when needed. - async fn prepare_inner(context: &Context, dir: &Path) -> Result<(Provider, Ticket)> { + async fn prepare_inner(context: &Context, dbfile: &Path) -> Result<(Provider, Ticket)> { // Generate the token up front: we also use it to encrypt the database. let token = AuthToken::generate(); context.emit_event(SendProgress::Started.into()); - let dbfile = dir.join(DBFILE_BACKUP_NAME); export_database(context, &dbfile, token.to_string()) .await .context("Database export failed")?; @@ -131,7 +135,7 @@ impl BackupProvider { // Now we can be sure IO is not running. let mut files = vec![DataSource::with_name( - dbfile, + dbfile.to_owned(), format!("db/{DBFILE_BACKUP_NAME}"), )]; let blobdir = BlobDirContents::new(context).await?; @@ -165,6 +169,7 @@ impl BackupProvider { context: Context, mut provider: Provider, cancel_token: Receiver<()>, + dbfile: PathBuf, ) -> Result<()> { context.emit_event(SendProgress::ProviderListening.into()); let mut events = provider.subscribe(); @@ -213,7 +218,9 @@ impl BackupProvider { }, } }; - // TODO: delete the database? + if let Err(err) = fs::remove_file(&dbfile).await { + error!(context, "Failed to remove database export: {err:#}"); + } context.emit_event(SendProgress::Completed.into()); context.free_ongoing().await; res @@ -361,12 +368,16 @@ async fn on_blob( ) -> Result { ensure!(!name.is_empty(), "Received a nameless blob"); let path = if name.starts_with("db/") { - // We can only safely write to the blobdir. But the blobdir could have a file named - // exactly like our special name. We solve this by using an uppercase extension - // which is forbidden for normal blobs. - context + let context_dir = context .get_blobdir() - .join(format!("{DBFILE_BACKUP_NAME}.SPECIAL")) + .parent() + .ok_or(anyhow!("Context dir not found"))?; + let dbfile = context_dir.join(DBFILE_BACKUP_NAME); + if fs::metadata(&dbfile).await.is_ok() { + fs::remove_file(&dbfile).await?; + warn!(context, "Previous database export deleted"); + } + dbfile } else { ensure!(name.starts_with("blob/"), "malformatted blob name"); let blobname = name.rsplit('/').next().context("malformatted blob name")?; @@ -456,8 +467,6 @@ impl From for EventType { mod tests { use std::time::Duration; - use testdir::testdir; - use crate::chat::{get_chat_msgs, send_msg, ChatItem}; use crate::message::{Message, Viewtype}; use crate::test_utils::TestContextManager; @@ -466,7 +475,6 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_send_receive() { - let dir = testdir!(); let mut tcm = TestContextManager::new(); // Create first device. @@ -479,7 +487,7 @@ mod tests { send_msg(&ctx0, self_chat.id, &mut msg).await.unwrap(); // Prepare to transfer backup. - let provider = BackupProvider::prepare(&ctx0, &dir).await.unwrap(); + let provider = BackupProvider::prepare(&ctx0).await.unwrap(); // Set up second device. let ctx1 = tcm.unconfigured().await; diff --git a/src/qr.rs b/src/qr.rs index a59036eba..e3625b8e9 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -66,7 +66,17 @@ pub enum Qr { Account { domain: String, }, + /// Provides a backup that can be retrieve. + /// + /// This contains all the data needed to connect to a device and download a backup from + /// it to configure the receiving device with the same account. Backup { + /// Printable version of the provider information. + /// + /// This is the printable version of a `sendme` ticket, which contains all the + /// information to connect to and authenticate a backup provider. + /// + /// The format is somewhat opaque, but `sendme` can deserialise this. ticket: sendme::provider::Ticket, }, WebrtcInstance { diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index 6aba36f6c..3f4b2dba2 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -322,7 +322,7 @@ mod tests { let dir = testdir!(); let mut tcm = TestContextManager::new(); let ctx = tcm.alice().await; - let provider = BackupProvider::prepare(&ctx, &dir).await.unwrap(); + let provider = BackupProvider::prepare(&ctx).await.unwrap(); let qr = provider.qr(); println!("{}", format_backup(&qr).unwrap()); From c48c2af7a10c6f4ed3b0ac575b545ba81f68d7f9 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 16:49:20 +0100 Subject: [PATCH 051/103] Allow retrieval of backup QR on context This enables being able to get the QR code without needing to have access to the BackupProvider itself. This is useful for the JSON-RPC server. --- src/context.rs | 25 +++++++++++++++++++++++++ src/imex/transfer.rs | 19 ++++++++++++++----- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/context.rs b/src/context.rs index 7db8f7d56..5ad01c106 100644 --- a/src/context.rs +++ b/src/context.rs @@ -22,6 +22,7 @@ use crate::events::{Event, EventEmitter, EventType, Events}; use crate::key::{DcKey, SignedPublicKey}; use crate::login_param::LoginParam; use crate::message::{self, MessageState, MsgId}; +use crate::qr::Qr; use crate::quota::QuotaInfo; use crate::scheduler::Scheduler; use crate::sql::Sql; @@ -189,6 +190,10 @@ pub struct InnerContext { pub(crate) blobdir: PathBuf, pub(crate) sql: Sql, pub(crate) last_smeared_timestamp: RwLock, + /// The global "ongoing" process state. + /// + /// This is a global mutex-like state for operations which should be modal in the + /// clients. running_state: RwLock, /// Mutex to avoid generating the key for the user more than once. pub(crate) generating_key_mutex: Mutex<()>, @@ -228,6 +233,14 @@ pub struct InnerContext { /// If debug logging is enabled, this contains all neccesary information pub(crate) debug_logging: RwLock>, + + /// Qr code for currently running [`BackupProvider`]. + /// + /// This is only available if a backup export is currently running, it will also be + /// holding the ongoing process while running. + /// + /// [`BackupProvider`]: crate::imex::BackupProvider + pub(crate) export_provider: std::sync::Mutex>, } #[derive(Debug)] @@ -370,6 +383,7 @@ impl Context { last_full_folder_scan: Mutex::new(None), last_error: std::sync::RwLock::new("".to_string()), debug_logging: RwLock::new(None), + export_provider: std::sync::Mutex::new(None), }; let ctx = Context { @@ -563,6 +577,17 @@ impl Context { } } + /// Returns the QR-code of the currently running [`BackupProvider`]. + /// + /// [`BackupProvider`]: crate::imex::BackupProvider + pub fn backup_export_qr(&self) -> Option { + self.export_provider + .lock() + .expect("poisoned lock") + .as_ref() + .cloned() + } + /******************************************************************************* * UI chat/message related API ******************************************************************************/ diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 24e2f6fd7..d345c94a8 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -118,7 +118,10 @@ impl BackupProvider { cancel_token, dbfile, )); - Ok(Self { handle, ticket }) + let slf = Self { handle, ticket }; + let qr = slf.qr(); + *context.export_provider.lock().expect("poisoned lock") = Some(qr); + Ok(slf) } /// Creates the provider task. @@ -128,7 +131,7 @@ impl BackupProvider { // Generate the token up front: we also use it to encrypt the database. let token = AuthToken::generate(); context.emit_event(SendProgress::Started.into()); - export_database(context, &dbfile, token.to_string()) + export_database(context, dbfile, token.to_string()) .await .context("Database export failed")?; context.emit_event(SendProgress::DatabaseExported.into()); @@ -171,7 +174,6 @@ impl BackupProvider { cancel_token: Receiver<()>, dbfile: PathBuf, ) -> Result<()> { - context.emit_event(SendProgress::ProviderListening.into()); let mut events = provider.subscribe(); let res = loop { tokio::select! { @@ -194,7 +196,6 @@ impl BackupProvider { provider.shutdown(); } Event::TransferAborted { .. } => { - context.emit_event(SendProgress::Failed.into()); provider.shutdown(); break Err(anyhow!("BackupSender transfer aborted")); } @@ -221,7 +222,15 @@ impl BackupProvider { if let Err(err) = fs::remove_file(&dbfile).await { error!(context, "Failed to remove database export: {err:#}"); } - context.emit_event(SendProgress::Completed.into()); + context + .export_provider + .lock() + .expect("poisoned lock") + .take(); + match &res { + Ok(_) => context.emit_event(SendProgress::Completed.into()), + Err(_) => context.emit_event(SendProgress::Failed.into()), + } context.free_ongoing().await; res } From a75d2b1c8009dea231337f50822c7897393d4891 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 17:15:54 +0100 Subject: [PATCH 052/103] Create a blocking call for jsonrpc --- deltachat-jsonrpc/src/api/mod.rs | 37 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 83e021afd..e0baf9a3b 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1341,32 +1341,49 @@ impl CommandApi { .await } - /// Starts to provide the backup for remote devices to retrieve. + /// Offers a backup for remote devices to retrieve. /// /// Can be cancelled by stopping the ongoing process. Success or failure can be tracked /// via the `ImexProgress` event which should either reach `1000` for success or `0` for /// failure. /// - /// This **stops IO**. After completion `start_io` must be called to restart IO. + /// This **stops IO** while it is running. /// - /// Returns the QR code as a rendered SVG image. - async fn provide_backup(&self, account_id: u32) -> Result { + /// Returns once a remote device has retrieved the backup. + async fn provide_backup(&self, account_id: u32) -> Result<()> { let ctx = self.get_context(account_id).await?; ctx.stop_io().await; - let provider = imex::BackupProvider::prepare(&ctx).await?; - let qr = provider.qr(); - let svg = match generate_backup_qr(&ctx, &qr).await { - Ok(svg) => svg, + let provider = match imex::BackupProvider::prepare(&ctx).await { + Ok(provider) => provider, Err(err) => { - ctx.stop_ongoing().await; + ctx.start_io().await; return Err(err); } }; - Ok(svg) + let res = provider.join().await; + ctx.start_io().await; + res + } + + /// Returns the QR code for the running [`CommandApi::provide_backup`]. + /// + /// This QR code can be used in [`CommandApi::get_backup`] on a second device to + /// retrieve the backup and setup this second device. + /// + /// Returns the QR code rendered as an SVG image. + async fn get_backup_qr_svg(&self, account_id: u32) -> Result { + let ctx = self.get_context(account_id).await?; + let qr = ctx + .backup_export_qr() + .ok_or(anyhow!("no backup being exported"))?; + generate_backup_qr(&ctx, &qr).await } /// Gets a backup from a remote provider. /// + /// This retrieves the backup from a remote device over the network and imports it into + /// the current device. + /// /// Can be cancelled by stopping the ongoing process. async fn get_backup(&self, account_id: u32, qr_text: String) -> Result<()> { let ctx = self.get_context(account_id).await?; From 8072f7805803beb5438c871a8ec816c3d50940db Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 18:05:09 +0100 Subject: [PATCH 053/103] Do not emit ImexEvent From BlobDirIter We no longer need that in the transfer case, that would give very weird results. This also means there is nothing imex-specific about this anymore so move it to blobs.rs --- src/blob.rs | 91 +++++++++++++++++++++++++++++++++++ src/imex.rs | 112 ++++--------------------------------------- src/imex/transfer.rs | 3 +- 3 files changed, 102 insertions(+), 104 deletions(-) diff --git a/src/blob.rs b/src/blob.rs index ef2869bed..1fe51f140 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -4,13 +4,16 @@ use core::cmp::max; use std::ffi::OsStr; use std::fmt; use std::io::Cursor; +use std::iter::FusedIterator; use std::path::{Path, PathBuf}; use anyhow::{format_err, Context as _, Result}; +use futures::StreamExt; use image::{DynamicImage, ImageFormat}; use num_traits::FromPrimitive; use tokio::io::AsyncWriteExt; use tokio::{fs, io}; +use tokio_stream::wrappers::ReadDirStream; use crate::config::Config; use crate::constants::{ @@ -468,6 +471,94 @@ impl<'a> fmt::Display for BlobObject<'a> { } } +/// All files in the blobdir. +/// +/// This exists so we can have a [`BlobDirIter`] which needs something to own the data of +/// it's `&Path`. Use [`BlobDirContents::iter`] to create the iterator. +/// +/// Additionally pre-allocating this means we get a length for progress report. +pub(crate) struct BlobDirContents<'a> { + inner: Vec, + context: &'a Context, +} + +impl<'a> BlobDirContents<'a> { + pub(crate) async fn new(context: &'a Context) -> Result> { + let readdir = fs::read_dir(context.get_blobdir()).await?; + let inner = ReadDirStream::new(readdir) + .filter_map(|entry| async move { + match entry { + Ok(entry) => Some(entry), + Err(err) => { + error!(context, "Failed to read blob file: {err}"); + None + } + } + }) + .filter_map(|entry| async move { + match entry.file_type().await.ok()?.is_file() { + true => Some(entry.path()), + false => { + warn!( + context, + "Export: Found blob dir entry {} that is not a file, ignoring", + entry.path().display() + ); + None + } + } + }) + .collect() + .await; + Ok(Self { inner, context }) + } + + pub(crate) fn iter(&self) -> BlobDirIter<'_> { + BlobDirIter::new(self.context, &self.inner) + } + + pub(crate) fn len(&self) -> usize { + self.inner.len() + } +} + +/// A iterator over all the [`BlobObject`]s in the blobdir. +pub(crate) struct BlobDirIter<'a> { + paths: &'a [PathBuf], + offset: usize, + context: &'a Context, +} + +impl<'a> BlobDirIter<'a> { + fn new(context: &'a Context, paths: &'a [PathBuf]) -> BlobDirIter<'a> { + Self { + paths, + offset: 0, + context, + } + } +} + +impl<'a> Iterator for BlobDirIter<'a> { + type Item = BlobObject<'a>; + + fn next(&mut self) -> Option { + while let Some(path) = self.paths.get(self.offset) { + self.offset += 1; + + // In theory this can error but we'd have corrupted filenames in the blobdir, so + // silently skipping them is fine. + match BlobObject::from_path(self.context, path) { + Ok(blob) => return Some(blob), + Err(err) => warn!(self.context, "{err}"), + } + } + None + } +} + +impl FusedIterator for BlobDirIter<'_> {} + fn encode_img(img: &DynamicImage, encoded: &mut Vec) -> anyhow::Result<()> { encoded.clear(); let mut buf = Cursor::new(encoded); diff --git a/src/imex.rs b/src/imex.rs index c6187a208..df28fae63 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -2,7 +2,6 @@ use std::any::Any; use std::ffi::OsStr; -use std::iter::FusedIterator; use std::path::{Path, PathBuf}; use ::pgp::types::KeyTrait; @@ -11,10 +10,9 @@ use futures::StreamExt; use futures_lite::FutureExt; use rand::{thread_rng, Rng}; use tokio::fs::{self, File}; -use tokio_stream::wrappers::ReadDirStream; use tokio_tar::Archive; -use crate::blob::BlobObject; +use crate::blob::{BlobDirContents, BlobObject}; use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId}; use crate::config::Config; use crate::contact::ContactId; @@ -570,10 +568,17 @@ async fn export_backup_inner( .await?; let blobdir = BlobDirContents::new(context).await?; - for blob in blobdir.iter() { + let mut last_progress = 0; + + for (i, blob) in blobdir.iter().enumerate() { let mut file = File::open(blob.to_abs_path()).await?; let path_in_archive = PathBuf::from(BLOBS_BACKUP_NAME).join(blob.as_name()); builder.append_file(path_in_archive, &mut file).await?; + let progress = 1000 * i / blobdir.len(); + if progress != last_progress && progress > 10 && progress < 1000 { + context.emit_event(EventType::ImexProgress(progress)); + last_progress = progress; + } } builder.finish().await?; @@ -761,105 +766,6 @@ async fn export_database(context: &Context, dest: &Path, passphrase: String) -> Ok(()) } -/// All files in the blobdir. -/// -/// This exists so we can have a [`BlobDirIter`] which needs something to own the data of -/// it's `&Path`. Use [`BlobDirContents::iter`] to create the iterator. -/// -/// Additionally pre-allocating this means we get a length for progress report. -struct BlobDirContents<'a> { - inner: Vec, - context: &'a Context, -} - -impl<'a> BlobDirContents<'a> { - async fn new(context: &'a Context) -> Result> { - let readdir = fs::read_dir(context.get_blobdir()).await?; - let inner = ReadDirStream::new(readdir) - .filter_map(|entry| async move { - match entry { - Ok(entry) => Some(entry), - Err(err) => { - error!(context, "Failed to read blob file: {err}"); - None - } - } - }) - .filter_map(|entry| async move { - match entry.file_type().await.ok()?.is_file() { - true => Some(entry.path()), - false => { - warn!( - context, - "Export: Found blob dir entry {} that is not a file, ignoring", - entry.path().display() - ); - None - } - } - }) - .collect() - .await; - Ok(Self { inner, context }) - } - - fn iter(&self) -> BlobDirIter<'_> { - BlobDirIter::new(self.context, &self.inner) - } -} - -/// A stream for [`BlobObject`]s. -/// -/// The stream emits [`EventType::ImexProgress`] events as it being consumed. -/// -/// Because we like to know the total number of blobs to emit progress all the blobs are -/// read up front and stored. Luckily this also makes our life easier, since we now only -/// need to implement `Iterator` and not `Stream`. -struct BlobDirIter<'a> { - paths: &'a [PathBuf], - offset: usize, - last_progress: usize, - context: &'a Context, -} - -impl<'a> BlobDirIter<'a> { - fn new(context: &'a Context, paths: &'a [PathBuf]) -> BlobDirIter<'a> { - Self { - paths, - offset: 0, - last_progress: 0, - context, - } - } -} - -impl<'a> Iterator for BlobDirIter<'a> { - type Item = BlobObject<'a>; - - fn next(&mut self) -> Option { - while let Some(path) = self.paths.get(self.offset) { - self.offset += 1; - - // In theory this can error but we'd have corrupted filenames in the blobdir, so - // silently skipping them is fine. - match BlobObject::from_path(self.context, path) { - Ok(blob) => { - let progress = 1000 * self.offset / self.paths.len(); - if progress != self.last_progress && progress > 10 && progress < 1000 { - self.context.emit_event(EventType::ImexProgress(progress)); - self.last_progress = progress; - } - return Some(blob); - } - Err(err) => warn!(self.context, "{err}"), - } - } - None - } -} - -impl FusedIterator for BlobDirIter<'_> {} - #[cfg(test)] mod tests { use ::pgp::armor::BlockType; diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index d345c94a8..af7e644f9 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -39,12 +39,13 @@ use tokio::sync::broadcast::error::RecvError; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReadDirStream; +use crate::blob::BlobDirContents; use crate::chat::delete_and_reset_all_device_msgs; use crate::context::Context; use crate::qr::Qr; use crate::{e2ee, EventType}; -use super::{export_database, BlobDirContents, DeleteOnDrop, DBFILE_BACKUP_NAME}; +use super::{export_database, DeleteOnDrop, DBFILE_BACKUP_NAME}; /// Provide or send a backup of this device. /// From f2570945c63032b76708ce0e4010371368dc1000 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 16 Feb 2023 18:18:18 +0100 Subject: [PATCH 054/103] Don't reimplement qr::format_backup --- src/qr_code_generator.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/qr_code_generator.rs b/src/qr_code_generator.rs index fddcce826..c8ca9c9da 100644 --- a/src/qr_code_generator.rs +++ b/src/qr_code_generator.rs @@ -1,6 +1,6 @@ //! # QR code generation module. -use anyhow::{bail, Result}; +use anyhow::Result; use base64::Engine as _; use qrcodegen::{QrCode, QrCodeEcc}; @@ -11,7 +11,7 @@ use crate::{ config::Config, contact::{Contact, ContactId}, context::Context, - qr::{Qr, DCBACKUP_SCHEME}, + qr::{self, Qr}, securejoin, stock_str::{self, backup_transfer_qr}, }; @@ -62,11 +62,7 @@ async fn generate_verification_qr(context: &Context) -> Result { /// Renders a [`Qr::Backup`] QR code as an SVG image. pub async fn generate_backup_qr(context: &Context, qr: &Qr) -> Result { - let ticket = match qr { - Qr::Backup { ticket } => ticket, - _ => bail!("QR code not a backup"), - }; - let content = format!("{DCBACKUP_SCHEME}{ticket}"); + let content = qr::format_backup(qr)?; let (avatar, displayname, _addr, color) = self_info(context).await?; let description = backup_transfer_qr(context).await?; From ada858f439a747d17d082edbf72137904641a74a Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 12:48:43 +0100 Subject: [PATCH 055/103] Improve comments, mostly ffi. and some renames --- deltachat-ffi/deltachat.h | 34 ++++++++++++++++++---------------- deltachat-ffi/src/lib.rs | 4 ++-- src/imex/transfer.rs | 2 ++ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 2a28020b1..c07fa624d 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2649,9 +2649,9 @@ void dc_str_unref (char* str); * by a QR-code. The backup contains the entire state of the account * including credentials. This can be used to setup a new device. * - * Once this function returns the backup is being offered to remove devices. - * To wait until one device transferred the backup, use - * dc_backup_provider_done(). Alternatively abort the operation using + * Once this function returns, the backup is being offered to remove devices. + * To wait until one device received the backup, use + * dc_backup_provider_wait(). Alternatively abort the operation using * dc_stop_ongoing_process(). * * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate @@ -2663,7 +2663,7 @@ void dc_str_unref (char* str); * On errors, NULL is returned and dc_get_last_error()returns an error that * should be shown to the user. */ -dc_backup_provider_t* dc_provide_backup (dc_context_t* context); +dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); /** @@ -2679,7 +2679,7 @@ dc_backup_provider_t* dc_provide_backup (dc_context_t* context); * @memberof dc_backup_provider_t * @param context The context. * @param backup_provider The backup provider object as created by - * dc_provide_backup(). + * dc_backup_provider_new(). * @return The text that should be put in the QR code. * On errors and empty QR code is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. @@ -2696,8 +2696,8 @@ char* dc_backup_provider_qr (dc_context_t* context, const dc_backup_provider_t* * @memberof dc_backup_provider_t * @param context The context. * @param backup_provider The backup provider object as created by - * dc_provide_backup(). - * @return The text that should be put in the QR code. + * dc_backup_provider_new(). + * @return The QR code rendered as SVG. * On errors and empty QR code is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ @@ -2708,10 +2708,10 @@ char * dc_backup_provider_qr_svg (dc_context_t* context, const dc_backup_provide * * @memberof dc_backup_sender_t * @param context The context. - * @param backup_sender The backup sender object as created by dc_send_backup(). - * If NULL is given nothing is done. + * @param backup_provider The backup provider object as created by + * dc_backup_provider_new(). If NULL is given nothing is done. */ -void dc_backup_provider_done (dc_context_t* context, dc_backup_provider_t* backup_provider); +void dc_backup_provider_wait (dc_context_t* context, dc_backup_provider_t* backup_provider); /** @@ -2721,18 +2721,20 @@ void dc_backup_provider_done (dc_context_t* context, dc_backup_provider_t* backu * dc_backup_sender_qr() or dc_backup_sender_qr_svg(). Typically this is a * different device than that which provides the backup. * - * While dc_receive_backup() returns immediately the started job make take a - * while. Use dc_stop_ongoing_process() to abort it early. + * This call will block while the backup is being transferred and only + * complete on success or failure. Use dc_stop_ongoing_process() to abort it + * early. * * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate - * state and progress. + * state and progress. The process is finished when the event emits either 0 + * or 1000, 0 means it failed and 1000 means it succeeded. These events are + * for showing progress and informational only, success and failure is also + * shown in the return code of this function. * * @param context The context. * @param qr The qr code text, dc_check_qr() must have returned DC_QR_BACKUP * on this text. - * @return 0=failure, 1=success. Success only means the progress was started - * correctly, final success or failure is via the #DC_EVENT_IMEX_PROGRESS - * events. + * @return 0=failure, 1=success. */ int dc_receive_backup (dc_context_t* context, const char* qr); diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 34a23b2f4..40b45bbef 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4134,7 +4134,7 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) { pub type dc_backup_provider_t = BackupProvider; #[no_mangle] -pub unsafe extern "C" fn dc_provide_backup( +pub unsafe extern "C" fn dc_backup_provider_new( context: *mut dc_context_t, ) -> *mut dc_backup_provider_t { if context.is_null() { @@ -4182,7 +4182,7 @@ pub unsafe extern "C" fn dc_backup_provider_qr_svg( } #[no_mangle] -pub unsafe extern "C" fn dc_backup_provider_done( +pub unsafe extern "C" fn dc_backup_provider_wait( context: *mut dc_context_t, provider: *mut dc_backup_provider_t, ) { diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index af7e644f9..412b302a4 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -284,6 +284,8 @@ impl From for EventType { /// using the [`BackupProvider`]. Once connected it will authenticate using the secrets in /// the QR code and retrieve the backup. /// +/// This is a long running operation which will only when completed. +/// /// Using [`Qr`] as argument is a bit odd as it only accepts one specific variant of it. It /// does avoid having [`sendme::provider::Ticket`] in the primary API however, without /// having to revert to untyped bytes. From 3c4ffc3550371545fa0e75aed144bbb7dbf082db Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 12:58:23 +0100 Subject: [PATCH 056/103] Some fixes --- deltachat-ffi/src/lib.rs | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 40b45bbef..f7534d679 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4142,13 +4142,10 @@ pub unsafe extern "C" fn dc_backup_provider_new( return ptr::null_mut(); } let ctx = &*context; - block_on(async move { - BackupProvider::prepare(ctx) - .await - .map(|provider| Box::into_raw(Box::new(provider))) - .log_err(ctx, "BackupProvider failed") - .unwrap_or(ptr::null_mut()) - }) + block_on(BackupProvider::prepare(ctx)) + .map(|provider| Box::into_raw(Box::new(provider))) + .log_err(ctx, "BackupProvider failed") + .unwrap_or(ptr::null_mut()) } #[no_mangle] @@ -4168,17 +4165,14 @@ pub unsafe extern "C" fn dc_backup_provider_qr_svg( provider: *const dc_backup_provider_t, ) -> *mut libc::c_char { if context.is_null() { - eprintln!("ignoring careless call to dc_send_backup()"); + eprintln!("ignoring careless call to dc_backup_provider_qr_svg()"); return ptr::null_mut(); } let ctx = &*context; let provider = &*provider; - block_on(async move { - generate_backup_qr(ctx, &provider.qr()) - .await - .unwrap_or_default() - .strdup() - }) + block_on(generate_backup_qr(ctx, &provider.qr())) + .unwrap_or_default() + .strdup() } #[no_mangle] @@ -4187,18 +4181,14 @@ pub unsafe extern "C" fn dc_backup_provider_wait( provider: *mut dc_backup_provider_t, ) { if context.is_null() { - eprintln!("ignoring careless call to dc_send_backup()"); + eprintln!("ignoring careless call to dc_backup_provider_wait()"); return; } let ctx = &*context; let provider = Box::from_raw(provider); - block_on(async move { - provider - .join() - .await - .log_err(ctx, "Failed to join provider") - .ok(); - }); + block_on(provider.join()) + .log_err(ctx, "Failed to join provider") + .ok(); } #[no_mangle] @@ -4207,7 +4197,7 @@ pub unsafe extern "C" fn dc_receive_backup( qr: *const libc::c_char, ) -> libc::c_int { if context.is_null() { - eprintln!("ignoring careless call to dc_send_backup()"); + eprintln!("ignoring careless call to dc_receive_backup()"); return 0; } let ctx = &*context; From 788d3125a3767a197ecef0c38f9501d9f1df9f19 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 13:02:16 +0100 Subject: [PATCH 057/103] Do not save svg to file, just print qr text --- Cargo.lock | 1 - deltachat-repl/Cargo.toml | 1 - deltachat-repl/src/cmdline.rs | 7 +------ 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1941f68a..f3a00cc69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1162,7 +1162,6 @@ dependencies = [ "pretty_env_logger", "rusqlite", "rustyline", - "tempfile", "tokio", ] diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index 678d74db1..9dca2a2ae 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -12,7 +12,6 @@ log = "0.4.16" pretty_env_logger = "0.4" rusqlite = "0.27" rustyline = "10" -tempfile = "3" tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] } [features] diff --git a/deltachat-repl/src/cmdline.rs b/deltachat-repl/src/cmdline.rs index 5dd1b6783..0a41da87c 100644 --- a/deltachat-repl/src/cmdline.rs +++ b/deltachat-repl/src/cmdline.rs @@ -491,12 +491,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu "send-backup" => { let provider = BackupProvider::prepare(&context).await?; let qr = provider.qr(); - let rendered = deltachat::qr_code_generator::generate_backup_qr(&context, &qr).await?; - let tdir = tempfile::TempDir::new()?; - let dir = tdir.path(); - let file = dir.join("qr.svg"); - tokio::fs::write(&file, rendered).await?; - println!("The QR code is at: {}", file.display()); + println!("QR code: {}", format_backup(&qr)?); provider.join().await?; } "receive-backup" => { From d0638c1542c99e076c07846a64ccf898f54af4d8 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 13:05:11 +0100 Subject: [PATCH 058/103] typo --- src/context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/context.rs b/src/context.rs index 5ad01c106..28ddae26e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -234,7 +234,7 @@ pub struct InnerContext { /// If debug logging is enabled, this contains all neccesary information pub(crate) debug_logging: RwLock>, - /// Qr code for currently running [`BackupProvider`]. + /// QR code for currently running [`BackupProvider`]. /// /// This is only available if a backup export is currently running, it will also be /// holding the ongoing process while running. From 4637a28bf681c39975db8638d5a131887f581aad Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 13:08:43 +0100 Subject: [PATCH 059/103] doc comment --- src/stock_str.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/stock_str.rs b/src/stock_str.rs index 580ffc6f8..6401ed1e1 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -1243,6 +1243,10 @@ pub(crate) async fn aeap_explanation_and_link( .replace2(new_addr) } +/// Text to put in the [`Qr::Backup`] rendered SVG image. +/// +/// The default is "Scan to set up second device for ". The +/// account name and address are looked up from the context. pub(crate) async fn backup_transfer_qr(context: &Context) -> Result { let contact = Contact::get_by_id(context, ContactId::SELF).await?; let addr = contact.get_addr(); From 73b90eee3e6ba5243f4bad22d24ffddb197ebaae Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 13:10:29 +0100 Subject: [PATCH 060/103] improve docs --- src/imex/transfer.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 412b302a4..aa32ba3c4 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -247,8 +247,9 @@ impl BackupProvider { /// Awaits the [`BackupProvider`] until it is finished. /// - /// This waits until someone connected to the sender and transferred a backup. If the - /// [`BackupProvider`] task results in an error it will be returned here. + /// This waits until someone connected to the sender and finished transferring a backup. + /// A failed transfer also counts as a finished transfer. If the [`BackupProvider`] + /// task results in an error it will be returned here. pub async fn join(self) -> Result<()> { self.handle.await??; Ok(()) From b920db12c7db792d4c5d37c6f06a49d06e4b07eb Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Mon, 20 Feb 2023 15:56:05 +0100 Subject: [PATCH 061/103] Split _wait and _unref This also removes BackupProvider::join in favour of implementing Future directly. I wondered about implementing a FusedFutre to make this a little safer but it would introduce a dependency on the futures crate in deltachat-ffi which did not exist yet, so I didn't do that. --- deltachat-ffi/deltachat.h | 10 +++++++++- deltachat-ffi/src/lib.rs | 9 +++++++-- deltachat-jsonrpc/src/api/mod.rs | 2 +- deltachat-repl/src/cmdline.rs | 2 +- src/imex/transfer.rs | 22 +++++++++++++--------- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index c07fa624d..980702a3e 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2706,13 +2706,21 @@ char * dc_backup_provider_qr_svg (dc_context_t* context, const dc_backup_provide /** * Waits for the sending to finish and frees the backup provider object. * - * @memberof dc_backup_sender_t + * This should only be called once. + * + * @memberof dc_backup_provider_t * @param context The context. * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). If NULL is given nothing is done. */ void dc_backup_provider_wait (dc_context_t* context, dc_backup_provider_t* backup_provider); +/** + * Frees a dc_backup_provider_t object. + * + * @memberof dc_backup_provider_t + */ +void dc_backup_prvider_unref (dc_backup_provider_t* backup_provider); /** * Gets a backup offered by a dc_backup_provider_t object on another device. diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index f7534d679..17104d238 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4185,12 +4185,17 @@ pub unsafe extern "C" fn dc_backup_provider_wait( return; } let ctx = &*context; - let provider = Box::from_raw(provider); - block_on(provider.join()) + let provider = &mut *provider; + block_on(provider) .log_err(ctx, "Failed to join provider") .ok(); } +#[no_mangle] +pub unsafe extern "C" fn dc_backup_provider_unref(provider: *mut dc_backup_provider_t) { + drop(Box::from_raw(provider)); +} + #[no_mangle] pub unsafe extern "C" fn dc_receive_backup( context: *mut dc_context_t, diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 047c285cc..6ff1a371d 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1371,7 +1371,7 @@ impl CommandApi { return Err(err); } }; - let res = provider.join().await; + let res = provider.await; ctx.start_io().await; res } diff --git a/deltachat-repl/src/cmdline.rs b/deltachat-repl/src/cmdline.rs index 0a41da87c..a90577009 100644 --- a/deltachat-repl/src/cmdline.rs +++ b/deltachat-repl/src/cmdline.rs @@ -492,7 +492,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu let provider = BackupProvider::prepare(&context).await?; let qr = provider.qr(); println!("QR code: {}", format_backup(&qr)?); - provider.join().await?; + provider.await?; } "receive-backup" => { ensure!(!arg1.is_empty(), "Argument is missing."); diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index aa32ba3c4..51c2b891c 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -22,7 +22,10 @@ //! getter can not connect to an impersonated provider and the provider does not offer the //! download to an impersonated getter. +use std::future::Future; use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::Poll; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; @@ -57,6 +60,9 @@ use super::{export_database, DeleteOnDrop, DBFILE_BACKUP_NAME}; /// /// This starts a task which acquires the global "ongoing" mutex. If you need to stop the /// task use the [`Context::stop_ongoing`] mechanism. +/// +/// The task implements [`Future`] and awaiting it will complete once a transfer has been +/// either completed or aborted. #[derive(Debug)] pub struct BackupProvider { /// The supervisor task, run by [`BackupProvider::watch_provider`]. @@ -244,15 +250,13 @@ impl BackupProvider { ticket: self.ticket.clone(), } } +} - /// Awaits the [`BackupProvider`] until it is finished. - /// - /// This waits until someone connected to the sender and finished transferring a backup. - /// A failed transfer also counts as a finished transfer. If the [`BackupProvider`] - /// task results in an error it will be returned here. - pub async fn join(self) -> Result<()> { - self.handle.await??; - Ok(()) +impl Future for BackupProvider { + type Output = Result<()>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + Pin::new(&mut self.handle).poll(cx)? } } @@ -507,7 +511,7 @@ mod tests { get_backup(&ctx1, provider.qr()).await.unwrap(); // Make sure the provider finishes without an error. - tokio::time::timeout(Duration::from_secs(30), provider.join()) + tokio::time::timeout(Duration::from_secs(30), provider) .await .expect("timed out") .expect("error in provider"); From fc64c3336896439a93f7d6d2a79d91e41771f0f8 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 22 Feb 2023 16:05:24 +0100 Subject: [PATCH 062/103] Use released version of sendme^Wiroh This switches to a released version. It has been renamed from sendme to iroh. --- Cargo.lock | 71 ++++++++++++++++++++++---------------------- Cargo.toml | 2 +- src/imex/transfer.rs | 15 +++++----- src/qr.rs | 4 +-- 4 files changed, 47 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44a858626..1fb8a944b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1021,6 +1021,7 @@ dependencies = [ "hex", "humansize", "image", + "iroh", "kamadak-exif", "lettre_email", "libc", @@ -1044,7 +1045,6 @@ dependencies = [ "rusqlite", "rust-hsluv", "sanitize-filename", - "sendme", "serde", "serde_json", "sha-1", @@ -2175,6 +2175,41 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +[[package]] +name = "iroh" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7849b9b19f7f05a21aa4ed1d2b344f9154be5ac1c7de119c9263091bc7505d" +dependencies = [ + "abao", + "anyhow", + "base64 0.21.0", + "blake3", + "bytes", + "der", + "ed25519-dalek", + "futures", + "portable-atomic", + "postcard", + "quinn", + "rand 0.7.3", + "rcgen", + "ring", + "rustls", + "serde", + "ssh-key", + "tempfile", + "thiserror", + "tokio", + "tokio-util", + "tracing", + "tracing-futures", + "tracing-subscriber", + "webpki", + "x509-parser", + "zeroize", +] + [[package]] name = "is-terminal" version = "0.4.1" @@ -3664,40 +3699,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sendme" -version = "0.1.0" -source = "git+https://github.com/n0-computer/sendme?branch=main#153073a93740a214731181e42d69328652081cda" -dependencies = [ - "abao", - "anyhow", - "base64 0.21.0", - "blake3", - "bytes", - "der", - "ed25519-dalek", - "futures", - "portable-atomic", - "postcard", - "quinn", - "rand 0.7.3", - "rcgen", - "ring", - "rustls", - "serde", - "ssh-key", - "tempfile", - "thiserror", - "tokio", - "tokio-util", - "tracing", - "tracing-futures", - "tracing-subscriber", - "webpki", - "x509-parser", - "zeroize", -] - [[package]] name = "serde" version = "1.0.152" diff --git a/Cargo.toml b/Cargo.toml index 9e5729574..e9777fb93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ futures-lite = "1.12.0" hex = "0.4.0" humansize = "2" image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] } +iroh = { version = "0.3.0", default-features = false } kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } libc = "0.2" @@ -67,7 +68,6 @@ reqwest = { version = "0.11.14", features = ["json"] } rusqlite = { version = "0.28", features = ["sqlcipher"] } rust-hsluv = "0.1" sanitize-filename = "0.4" -sendme = { git = "https://github.com/n0-computer/sendme", branch = "main", default-features = false } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } sha-1 = "0.10" diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 51c2b891c..21bdd814d 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -30,11 +30,11 @@ use std::task::Poll; use anyhow::{anyhow, bail, ensure, format_err, Context as _, Result}; use async_channel::Receiver; use futures_lite::StreamExt; -use sendme::get::{DataStream, Options}; -use sendme::progress::ProgressEmitter; -use sendme::protocol::AuthToken; -use sendme::provider::{DataSource, Event, Provider, Ticket}; -use sendme::Hash; +use iroh::get::{DataStream, Options}; +use iroh::progress::ProgressEmitter; +use iroh::protocol::AuthToken; +use iroh::provider::{DataSource, Event, Provider, Ticket}; +use iroh::Hash; use tokio::fs::{self, File}; use tokio::io::{self, AsyncWriteExt, BufWriter}; use tokio::sync::broadcast; @@ -156,7 +156,7 @@ impl BackupProvider { } // Start listening. - let (db, hash) = sendme::provider::create_collection(files).await?; + let (db, hash) = iroh::provider::create_collection(files).await?; context.emit_event(SendProgress::CollectionCreated.into()); let provider = Provider::builder(db).auth_token(token).spawn()?; context.emit_event(SendProgress::ProviderListening.into()); @@ -328,6 +328,7 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { let opts = Options { addr: ticket.addr, peer_id: Some(ticket.peer), + keylog: false, }; let progress = ProgressEmitter::new(0, ReceiveProgress::max_blob_progress()); spawn_progress_proxy(context.clone(), progress.subscribe()); @@ -336,7 +337,7 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { async { Ok(()) } }; let on_blob = |hash, reader, name| on_blob(context, &progress, &ticket, hash, reader, name); - let res = sendme::get::run( + let res = iroh::get::run( ticket.hash, ticket.token, opts, diff --git a/src/qr.rs b/src/qr.rs index dd5c84e75..aa83bdac5 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -114,7 +114,7 @@ pub enum Qr { /// information to connect to and authenticate a backup provider. /// /// The format is somewhat opaque, but `sendme` can deserialise this. - ticket: sendme::provider::Ticket, + ticket: iroh::provider::Ticket, }, /// Ask the user if they want to use the given service for video chats. @@ -509,7 +509,7 @@ fn decode_backup(qr: &str) -> Result { let payload = qr .strip_prefix(DCBACKUP_SCHEME) .ok_or(anyhow!("invalid DCBACKUP scheme"))?; - let ticket: sendme::provider::Ticket = payload.parse().context("invalid DCBACKUP payload")?; + let ticket: iroh::provider::Ticket = payload.parse().context("invalid DCBACKUP payload")?; Ok(Qr::Backup { ticket }) } From 5833a9b347128bc4d36b82dd2608b9ec38a62d90 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 22 Feb 2023 18:50:32 +0100 Subject: [PATCH 063/103] fix doc comments --- src/imex/transfer.rs | 7 +++---- src/qr.rs | 2 +- src/stock_str.rs | 2 ++ 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 21bdd814d..94aeba639 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -78,9 +78,8 @@ impl BackupProvider { /// or database are happening, this is done by calling the [`Accounts::stop_io`] or /// [`Context::stop_io`] APIs first. /// - /// This will acquire the global "ongoing process" mutex. You must call - /// [`BackupProvider::join`] after creating this struct, otherwise this will not respect - /// the possible cancellation of the "ongoing process". + /// This will acquire the global "ongoing process" mutex, which can be used to cancel + /// the process. /// /// [`Accounts::stop_io`]: crate::accounts::Accounts::stop_io pub async fn prepare(context: &Context) -> Result { @@ -292,7 +291,7 @@ impl From for EventType { /// This is a long running operation which will only when completed. /// /// Using [`Qr`] as argument is a bit odd as it only accepts one specific variant of it. It -/// does avoid having [`sendme::provider::Ticket`] in the primary API however, without +/// does avoid having [`iroh::provider::Ticket`] in the primary API however, without /// having to revert to untyped bytes. pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { ensure!( diff --git a/src/qr.rs b/src/qr.rs index aa83bdac5..54855b5c8 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -504,7 +504,7 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result { /// Decodes a [`DCBACKUP_SCHEME`] QR code. /// /// The format of this scheme is `DCBACKUP:`. The encoding is the -/// [`sendme::provider::Ticket`]'s `Display` impl. +/// [`iroh::provider::Ticket`]'s `Display` impl. fn decode_backup(qr: &str) -> Result { let payload = qr .strip_prefix(DCBACKUP_SCHEME) diff --git a/src/stock_str.rs b/src/stock_str.rs index 6401ed1e1..b4b2fd0c9 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -1247,6 +1247,8 @@ pub(crate) async fn aeap_explanation_and_link( /// /// The default is "Scan to set up second device for ". The /// account name and address are looked up from the context. +/// +/// [`Qr::Backup`]: crate::qr::Qr::Backup pub(crate) async fn backup_transfer_qr(context: &Context) -> Result { let contact = Contact::get_by_id(context, ContactId::SELF).await?; let addr = contact.get_addr(); From 0e06da22df832a76823a9ddb44460a270eafd84c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 22 Feb 2023 18:51:17 +0100 Subject: [PATCH 064/103] fix symbol name --- deltachat-ffi/deltachat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 1e9998488..c3299a217 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2720,7 +2720,7 @@ void dc_backup_provider_wait (dc_context_t* context, dc_backup_provider_t* backu * * @memberof dc_backup_provider_t */ -void dc_backup_prvider_unref (dc_backup_provider_t* backup_provider); +void dc_backup_provider_unref (dc_backup_provider_t* backup_provider); /** * Gets a backup offered by a dc_backup_provider_t object on another device. From 5762fbb9a7a8bcd77efdd45baa3907d5a9237cef Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Wed, 1 Mar 2023 11:27:21 +0100 Subject: [PATCH 065/103] Allow JSON-RPC to get text of QR code as well Desktop does use this as it allows reading QR codes as text from the clipboard as well as copying the QR text to the clipboard instead of showing the QR code. --- deltachat-jsonrpc/src/api/mod.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 6ff1a371d..a9e214bee 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1376,7 +1376,19 @@ impl CommandApi { res } - /// Returns the QR code for the running [`CommandApi::provide_backup`]. + /// Returns the text of the QR code for the running [`CommandApi::provide_backup`]. + /// + /// This QR code text can be used in [`CommandApi::get_backup`] on a second device to + /// retrieve the backup and setup this second device. + async fn get_backup_qr(&self, account_id: u32) -> Result { + let ctx = self.get_context(account_id).await?; + let qr = ctx + .backup_export_qr() + .ok_or(anyhow!("no backup being exported"))?; + qr::format_backup(&qr) + } + + /// Returns the rendered QR code for the running [`CommandApi::provide_backup`]. /// /// This QR code can be used in [`CommandApi::get_backup`] on a second device to /// retrieve the backup and setup this second device. From 2d2081265256dd72f4d68597a70cc66ac16fbc38 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 09:39:50 +0100 Subject: [PATCH 066/103] some typos --- deltachat-ffi/deltachat.h | 2 +- deltachat-ffi/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index c3299a217..b98dfba0c 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2649,7 +2649,7 @@ void dc_str_unref (char* str); * by a QR-code. The backup contains the entire state of the account * including credentials. This can be used to setup a new device. * - * Once this function returns, the backup is being offered to remove devices. + * Once this function returns, the backup is being offered to remote devices. * To wait until one device received the backup, use * dc_backup_provider_wait(). Alternatively abort the operation using * dc_stop_ongoing_process(). diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index f4923e7ed..43a82f959 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4146,7 +4146,7 @@ pub unsafe extern "C" fn dc_backup_provider_new( context: *mut dc_context_t, ) -> *mut dc_backup_provider_t { if context.is_null() { - eprintln!("ignoring careless call to dc_send_backup()"); + eprintln!("ignoring careless call to dc_backup_provider_new()"); return ptr::null_mut(); } let ctx = &*context; From 6f24874eb8a36af927d4aba854ca17aec28d941d Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 10:58:39 +0100 Subject: [PATCH 067/103] Use a RAII guard to remove the db export --- src/imex/transfer.rs | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 94aeba639..cc951d18a 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -23,6 +23,7 @@ //! download to an impersonated getter. use std::future::Future; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::task::Poll; @@ -98,6 +99,7 @@ impl BackupProvider { fs::remove_file(&dbfile).await?; warn!(context, "Previous database export deleted"); } + let dbfile = TempPathGuard::new(dbfile); let res = tokio::select! { biased; res = Self::prepare_inner(context, &dbfile) => { @@ -178,8 +180,9 @@ impl BackupProvider { context: Context, mut provider: Provider, cancel_token: Receiver<()>, - dbfile: PathBuf, + _dbfile: TempPathGuard, ) -> Result<()> { + // _dbfile exists so we can clean up the file once it is no longer needed let mut events = provider.subscribe(); let res = loop { tokio::select! { @@ -225,9 +228,6 @@ impl BackupProvider { }, } }; - if let Err(err) = fs::remove_file(&dbfile).await { - error!(context, "Failed to remove database export: {err:#}"); - } context .export_provider .lock() @@ -259,6 +259,37 @@ impl Future for BackupProvider { } } +/// A guard which will remove the path when dropped. +/// +/// It implements [`Deref`] it it can be used as a `&Path`. +#[derive(Debug)] +struct TempPathGuard { + path: PathBuf, +} + +impl TempPathGuard { + fn new(path: PathBuf) -> Self { + Self { path } + } +} + +impl Drop for TempPathGuard { + fn drop(&mut self) { + let path = self.path.clone(); + tokio::spawn(async move { + fs::remove_file(&path).await.ok(); + }); + } +} + +impl Deref for TempPathGuard { + type Target = Path; + + fn deref(&self) -> &Self::Target { + &self.path + } +} + /// Create [`EventType::ImexProgress`] events using readable names. /// /// Plus you get warnings if you don't use all variants. From 38d4ea8514cdc11660993ac0f08d56868ec9fd25 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 11:15:02 +0100 Subject: [PATCH 068/103] Use std::slice::Iter instead of manually tracking the offset --- src/blob.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/blob.rs b/src/blob.rs index 1fe51f140..074ed3a76 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -514,7 +514,7 @@ impl<'a> BlobDirContents<'a> { } pub(crate) fn iter(&self) -> BlobDirIter<'_> { - BlobDirIter::new(self.context, &self.inner) + BlobDirIter::new(self.context, self.inner.iter()) } pub(crate) fn len(&self) -> usize { @@ -524,18 +524,13 @@ impl<'a> BlobDirContents<'a> { /// A iterator over all the [`BlobObject`]s in the blobdir. pub(crate) struct BlobDirIter<'a> { - paths: &'a [PathBuf], - offset: usize, + iter: std::slice::Iter<'a, PathBuf>, context: &'a Context, } impl<'a> BlobDirIter<'a> { - fn new(context: &'a Context, paths: &'a [PathBuf]) -> BlobDirIter<'a> { - Self { - paths, - offset: 0, - context, - } + fn new(context: &'a Context, iter: std::slice::Iter<'a, PathBuf>) -> BlobDirIter<'a> { + Self { iter, context } } } @@ -543,9 +538,7 @@ impl<'a> Iterator for BlobDirIter<'a> { type Item = BlobObject<'a>; fn next(&mut self) -> Option { - while let Some(path) = self.paths.get(self.offset) { - self.offset += 1; - + while let Some(path) = self.iter.next() { // In theory this can error but we'd have corrupted filenames in the blobdir, so // silently skipping them is fine. match BlobObject::from_path(self.context, path) { From 155dff2813535da4d0e94c317e4d9ca8b6e41879 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 11:18:30 +0100 Subject: [PATCH 069/103] renaming of upstream --- src/imex/transfer.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index cc951d18a..d8eb5821d 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -1,14 +1,14 @@ //! Transfer a backup to an other device. //! -//! This module provides support for using n0's sendme tool to initiate transfer of a backup +//! This module provides support for using n0's iroh tool to initiate transfer of a backup //! to another device using a QR code. //! -//! Using the sendme terminology there are two parties to this: +//! Using the iroh terminology there are two parties to this: //! //! - The *Provider*, which starts a server and listens for connections. //! - The *Getter*, which connects to the server and retrieves the data. //! -//! Sendme is designed around the idea of verifying hashes, the downloads are verified as +//! Iroh is designed around the idea of verifying hashes, the downloads are verified as //! they are retrieved. The entire transfer is initiated by requesting the data of a single //! root hash. //! @@ -166,7 +166,7 @@ impl BackupProvider { Ok((provider, ticket)) } - /// Supervises the sendme [`Provider`], terminating it when needed. + /// Supervises the iroh [`Provider`], terminating it when needed. /// /// This will watch the provider and terminate it when: /// From 9d88ef069e97f0f1fd3a61eecf91322cf66f6fa5 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 11:21:05 +0100 Subject: [PATCH 070/103] log some more --- src/imex/transfer.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index d8eb5821d..057e609c8 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -188,7 +188,7 @@ impl BackupProvider { tokio::select! { biased; res = &mut provider => { - break res.context("BackupSender failed"); + break res.context("BackupProvider failed"); }, maybe_event = events.recv() => { match maybe_event { @@ -206,7 +206,7 @@ impl BackupProvider { } Event::TransferAborted { .. } => { provider.shutdown(); - break Err(anyhow!("BackupSender transfer aborted")); + break Err(anyhow!("BackupProvider transfer aborted")); } } } @@ -218,7 +218,7 @@ impl BackupProvider { // We really shouldn't be lagging, if we did we may have missed // a completion event. provider.shutdown(); - break Err(anyhow!("Missed events from BackupSender")); + break Err(anyhow!("Missed events from BackupProvider")); } } }, @@ -235,7 +235,10 @@ impl BackupProvider { .take(); match &res { Ok(_) => context.emit_event(SendProgress::Completed.into()), - Err(_) => context.emit_event(SendProgress::Failed.into()), + Err(err) => { + error!(context, "Backup transfer failure: {err:#}"); + context.emit_event(SendProgress::Failed.into()) + } } context.free_ongoing().await; res From d108f9b3e38d7d02619bb33ecb8982088e2ab8c6 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 11:47:03 +0100 Subject: [PATCH 071/103] clippy --- src/blob.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blob.rs b/src/blob.rs index 074ed3a76..ca88217ea 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -538,7 +538,7 @@ impl<'a> Iterator for BlobDirIter<'a> { type Item = BlobObject<'a>; fn next(&mut self) -> Option { - while let Some(path) = self.iter.next() { + for path in self.iter.as_ref() { // In theory this can error but we'd have corrupted filenames in the blobdir, so // silently skipping them is fine. match BlobObject::from_path(self.context, path) { From da85c2412e56af69a85f5e0e1ef00fcd2873b6b0 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 21:48:14 +0100 Subject: [PATCH 072/103] fix iterator --- src/blob.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blob.rs b/src/blob.rs index ca88217ea..cdc8ffc94 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -538,7 +538,7 @@ impl<'a> Iterator for BlobDirIter<'a> { type Item = BlobObject<'a>; fn next(&mut self) -> Option { - for path in self.iter.as_ref() { + for path in self.iter.by_ref() { // In theory this can error but we'd have corrupted filenames in the blobdir, so // silently skipping them is fine. match BlobObject::from_path(self.context, path) { From c1a2df91acd0fe35880cba0c62485d78b3b446de Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 2 Mar 2023 21:53:13 +0100 Subject: [PATCH 073/103] Fix typo in blob names This is now tested properly too. --- src/imex/transfer.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 057e609c8..357e8b8f1 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -152,7 +152,7 @@ impl BackupProvider { let blobdir = BlobDirContents::new(context).await?; for blob in blobdir.iter() { let path = blob.to_abs_path(); - let name = format!("blob/P{}", blob.as_file_name()); + let name = format!("blob/{}", blob.as_file_name()); files.push(DataSource::with_name(path, name)); } @@ -537,6 +537,13 @@ mod tests { msg.set_text(Some("hi there".to_string())); send_msg(&ctx0, self_chat.id, &mut msg).await.unwrap(); + // Send an attachment in the self chat + let file = ctx0.get_blobdir().join("hello.txt"); + fs::write(&file, "i am attachment").await.unwrap(); + let mut msg = Message::new(Viewtype::File); + msg.set_file(file.to_str().unwrap(), Some("text/plain")); + send_msg(&ctx0, self_chat.id, &mut msg).await.unwrap(); + // Prepare to transfer backup. let provider = BackupProvider::prepare(&ctx0).await.unwrap(); @@ -553,7 +560,7 @@ mod tests { // Check that we have the self message. let self_chat = ctx1.get_self_chat().await; let msgs = get_chat_msgs(&ctx1, self_chat.id).await.unwrap(); - assert_eq!(msgs.len(), 1); + assert_eq!(msgs.len(), 2); let msgid = match msgs.get(0).unwrap() { ChatItem::Message { msg_id } => msg_id, _ => panic!("wrong chat item"), @@ -561,6 +568,14 @@ mod tests { let msg = Message::load_from_db(&ctx1, *msgid).await.unwrap(); let text = msg.get_text().unwrap(); assert_eq!(text, "hi there"); + let msgid = match msgs.get(1).unwrap() { + ChatItem::Message { msg_id } => msg_id, + _ => panic!("wrong chat item"), + }; + let msg = Message::load_from_db(&ctx1, *msgid).await.unwrap(); + let path = msg.get_file(&ctx1).unwrap(); + let text = fs::read_to_string(&path).await.unwrap(); + assert_eq!(text, "i am attachment"); // Check that both received the ImexProgress events. ctx0.evtracker From 731e90f0d5c7cbe9c391ad4805e3f490fb1e0da7 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Fri, 3 Mar 2023 12:53:43 +0100 Subject: [PATCH 074/103] update cargo-deny --- deny.toml | 57 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/deny.toml b/deny.toml index 0d1e04259..02ef8bce7 100644 --- a/deny.toml +++ b/deny.toml @@ -12,27 +12,28 @@ ignore = [ # becoming empty. Adding versions forces us to revisit this at least # when upgrading. skip = [ - { name = "windows-sys", version = "<0.45" }, - { name = "wasi", version = "<0.11" }, - { name = "version_check", version = "<0.9" }, - { name = "uuid", version = "<1.3" }, - { name = "sha2", version = "<0.10" }, - { name = "rand_core", version = "<0.6" }, - { name = "rand_chacha", version = "<0.3" }, - { name = "rand", version = "<0.8" }, - { name = "nom", version = "<7.1" }, - { name = "idna", version = "<0.3" }, - { name = "humantime", version = "<2.1" }, - { name = "hermit-abi", version = "<0.3" }, - { name = "getrandom", version = "<0.2" }, - { name = "quick-error", version = "<2.0" }, - { name = "env_logger", version = "<0.10" }, - { name = "digest", version = "<0.10" }, - { name = "darling_macro", version = "<0.14" }, - { name = "darling_core", version = "<0.14" }, - { name = "darling", version = "<0.14" }, - { name = "block-buffer", version = "<0.10" }, { name = "base64", version = "<0.21" }, + { name = "block-buffer", version = "<0.10" }, + { name = "darling", version = "<0.14" }, + { name = "darling_core", version = "<0.14" }, + { name = "darling_macro", version = "<0.14" }, + { name = "digest", version = "<0.10" }, + { name = "env_logger", version = "<0.10" }, + { name = "getrandom", version = "<0.2" }, + { name = "hermit-abi", version = "<0.3" }, + { name = "humantime", version = "<2.1" }, + { name = "idna", version = "<0.3" }, + { name = "nom", version = "<7.1" }, + { name = "quick-error", version = "<2.0" }, + { name = "rand", version = "<0.8" }, + { name = "rand_chacha", version = "<0.3" }, + { name = "rand_core", version = "<0.6" }, + { name = "sha2", version = "<0.10" }, + { name = "time", version = "<0.3" }, + { name = "uuid", version = "<1.3" }, + { name = "version_check", version = "<0.9" }, + { name = "wasi", version = "<0.11" }, + { name = "windows-sys", version = "<0.45" }, ] @@ -42,11 +43,21 @@ allow = [ "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", - "CC0-1.0", - "MIT", "BSL-1.0", # Boost Software License 1.0 - "Unicode-DFS-2016", + "CC0-1.0", + "ISC", + "MIT", "MPL-2.0", + "OpenSSL", + "Unicode-DFS-2016", + "Zlib", +] + +[[licenses.clarify]] +name = "ring" +expression = "MIT AND ISC AND OpenSSL" +license-files = [ + { path = "LICENSE", hash = 0xbd0eed23 }, ] [sources.allow-org] From d73d56c399f2824a4d01fda54c66efb3d81b8d4c Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Fri, 3 Mar 2023 13:13:58 +0100 Subject: [PATCH 075/103] bump testdir for windows bug workaround --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a27a67fcc..6d551dbcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,7 +95,7 @@ log = "0.4" pretty_env_logger = "0.4" proptest = { version = "1", default-features = false, features = ["std"] } tempfile = "3" -testdir = "0.7.1" +testdir = "0.7.2" tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] } [workspace] From b6c24932a7630f98ff9608bdb2435f60e73ea902 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 12:23:30 +0100 Subject: [PATCH 076/103] Apply typos from code review Co-authored-by: Hocuri --- deltachat-ffi/deltachat.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index b98dfba0c..4a998028a 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2660,7 +2660,7 @@ void dc_str_unref (char* str); * @memberof dc_backup_sender_t * @param context The context. * @return Opaque object for sending the backup. - * On errors, NULL is returned and dc_get_last_error()returns an error that + * On errors, NULL is returned and dc_get_last_error() returns an error that * should be shown to the user. */ dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); @@ -2673,7 +2673,7 @@ dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); * authentication for both the provider and the recipient. * * The scanning device should call the scanned text to dc_check_qr(). If - * dc_check_qr() returns DC_QR_BACKUP the backup transfer can be started using + * dc_check_qr() returns DC_QR_BACKUP, the backup transfer can be started using * dc_get_backup(). * * @memberof dc_backup_provider_t @@ -2681,7 +2681,7 @@ dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). * @return The text that should be put in the QR code. - * On errors and empty QR code is returned, NULL is never returned. + * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ char* dc_backup_provider_qr (dc_context_t* context, const dc_backup_provider_t* backup_provider); @@ -2698,7 +2698,7 @@ char* dc_backup_provider_qr (dc_context_t* context, const dc_backup_provider_t* * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). * @return The QR code rendered as SVG. - * On errors and empty QR code is returned, NULL is never returned. + * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ char * dc_backup_provider_qr_svg (dc_context_t* context, const dc_backup_provider_t* backup_provider); From a5e6bd3e8e852f446c3be19e7aaa4f6ca083215e Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 12:49:42 +0100 Subject: [PATCH 077/103] Do not require context for non-context methods This follows the ffi style better. --- Cargo.lock | 4 ++-- deltachat-ffi/src/lib.rs | 42 +++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77544a7bf..db57d49d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4136,9 +4136,9 @@ dependencies = [ [[package]] name = "testdir" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23029d5d16b0351859485c6849252f00bf0ebc98098a9efd954853c3533720c7" +checksum = "c31eb500f7513b559ed7e0652894268dbe8ef27b6241b783ce274f4741eae137" dependencies = [ "anyhow", "backtrace", diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index f3f14b2af..79f18de96 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4139,7 +4139,12 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) { libc::free(s as *mut _) } -pub type dc_backup_provider_t = BackupProvider; +pub struct BackupProviderWrapper { + context: *const dc_context_t, + provider: BackupProvider, +} + +pub type dc_backup_provider_t = BackupProviderWrapper; #[no_mangle] pub unsafe extern "C" fn dc_backup_provider_new( @@ -4151,49 +4156,54 @@ pub unsafe extern "C" fn dc_backup_provider_new( } let ctx = &*context; block_on(BackupProvider::prepare(ctx)) - .map(|provider| Box::into_raw(Box::new(provider))) + .map(|provider| BackupProviderWrapper { + context: ctx, + provider, + }) + .map(|ffi_provider| Box::into_raw(Box::new(ffi_provider))) .log_err(ctx, "BackupProvider failed") .unwrap_or(ptr::null_mut()) } #[no_mangle] pub unsafe extern "C" fn dc_backup_provider_qr( - _context: *mut dc_context_t, provider: *const dc_backup_provider_t, ) -> *mut libc::c_char { - let provider = &*provider; - deltachat::qr::format_backup(&provider.qr()) + if provider.is_null() { + eprintln!("ignoring careless call to dc_backup_provider_qr"); + return ptr::null_mut(); + } + let ffi_provider = &*provider; + deltachat::qr::format_backup(&ffi_provider.provider.qr()) .unwrap_or_default() .strdup() } #[no_mangle] pub unsafe extern "C" fn dc_backup_provider_qr_svg( - context: *mut dc_context_t, provider: *const dc_backup_provider_t, ) -> *mut libc::c_char { - if context.is_null() { + if provider.is_null() { eprintln!("ignoring careless call to dc_backup_provider_qr_svg()"); return ptr::null_mut(); } - let ctx = &*context; - let provider = &*provider; + let ffi_provider = &*provider; + let ctx = &*ffi_provider.context; + let provider = &ffi_provider.provider; block_on(generate_backup_qr(ctx, &provider.qr())) .unwrap_or_default() .strdup() } #[no_mangle] -pub unsafe extern "C" fn dc_backup_provider_wait( - context: *mut dc_context_t, - provider: *mut dc_backup_provider_t, -) { - if context.is_null() { +pub unsafe extern "C" fn dc_backup_provider_wait(provider: *mut dc_backup_provider_t) { + if provider.is_null() { eprintln!("ignoring careless call to dc_backup_provider_wait()"); return; } - let ctx = &*context; - let provider = &mut *provider; + let ffi_provider = &mut *provider; + let ctx = &*ffi_provider.context; + let provider = &mut ffi_provider.provider; block_on(provider) .log_err(ctx, "Failed to join provider") .ok(); From 9767f51c3d6a471348262feed518d093862bffe2 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 14:13:42 +0100 Subject: [PATCH 078/103] update .h file too --- deltachat-ffi/deltachat.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 0f0a204bd..8cfbba3f7 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2684,7 +2684,7 @@ dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ -char* dc_backup_provider_qr (dc_context_t* context, const dc_backup_provider_t* backup_provider); +char* dc_backup_provider_qr (const dc_backup_provider_t* backup_provider); /** @@ -2701,7 +2701,7 @@ char* dc_backup_provider_qr (dc_context_t* context, const dc_backup_provider_t* * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ -char * dc_backup_provider_qr_svg (dc_context_t* context, const dc_backup_provider_t* backup_provider); +char * dc_backup_provider_qr_svg (const dc_backup_provider_t* backup_provider); /** * Waits for the sending to finish and frees the backup provider object. @@ -2713,7 +2713,7 @@ char * dc_backup_provider_qr_svg (dc_context_t* context, const dc_backup_provide * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). If NULL is given nothing is done. */ -void dc_backup_provider_wait (dc_context_t* context, dc_backup_provider_t* backup_provider); +void dc_backup_provider_wait (dc_backup_provider_t* backup_provider); /** * Frees a dc_backup_provider_t object. From 52a188693743464a0763a5f4e04aad5aac44d9f9 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 14:40:01 +0100 Subject: [PATCH 079/103] naming conventions! they're hard --- deltachat-ffi/deltachat.h | 4 ++-- deltachat-ffi/src/lib.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 8cfbba3f7..562efa183 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2684,7 +2684,7 @@ dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ -char* dc_backup_provider_qr (const dc_backup_provider_t* backup_provider); +char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider); /** @@ -2701,7 +2701,7 @@ char* dc_backup_provider_qr (const dc_backup_provider_t* backup_provider); * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ -char * dc_backup_provider_qr_svg (const dc_backup_provider_t* backup_provider); +char * dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider); /** * Waits for the sending to finish and frees the backup provider object. diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 66e321928..87bc517e2 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4166,7 +4166,7 @@ pub unsafe extern "C" fn dc_backup_provider_new( } #[no_mangle] -pub unsafe extern "C" fn dc_backup_provider_qr( +pub unsafe extern "C" fn dc_backup_provider_get_qr( provider: *const dc_backup_provider_t, ) -> *mut libc::c_char { if provider.is_null() { @@ -4180,7 +4180,7 @@ pub unsafe extern "C" fn dc_backup_provider_qr( } #[no_mangle] -pub unsafe extern "C" fn dc_backup_provider_qr_svg( +pub unsafe extern "C" fn dc_backup_provider_get_qr_svg( provider: *const dc_backup_provider_t, ) -> *mut libc::c_char { if provider.is_null() { From 4b94eadf5e81ac762ce05bf25f83cf2ad7f42a42 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 14:40:51 +0100 Subject: [PATCH 080/103] typo --- deltachat-ffi/deltachat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 562efa183..d00d9c0d7 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2701,7 +2701,7 @@ char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider); * On errors an empty string is returned, NULL is never returned. * the returned string must be released using dc_str_unref() after usage. */ -char * dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider); +char* dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider); /** * Waits for the sending to finish and frees the backup provider object. From eec5ae96e89d3349440a46d34880470c429a0f6f Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 15:36:33 +0100 Subject: [PATCH 081/103] Update docs and fix string allocation The docs say you should always unref the string and NULL is never returned. The implementation should follow that. --- deltachat-ffi/deltachat.h | 13 ++++++++++--- deltachat-ffi/src/lib.rs | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index d00d9c0d7..bdf9bd025 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2645,12 +2645,17 @@ void dc_str_unref (char* str); /** * Creates an object for sending a backup to another device. * + * Before calling this function IO must be stopped using dc_accounts_stop_io() + * or dc_stop_io(). IO should only be restarted once + * dc_backup_provider_wait() has returned. + * * The backup is sent to through a peer-to-peer channel which is bootstrapped * by a QR-code. The backup contains the entire state of the account * including credentials. This can be used to setup a new device. * - * Once this function returns, the backup is being offered to remote devices. - * To wait until one device received the backup, use + * This is a blocking call as some preparations are made like e.g. exporting + * the database. Once this function returns, the backup is being offered to + * remote devices. To wait until one device received the backup, use * dc_backup_provider_wait(). Alternatively abort the operation using * dc_stop_ongoing_process(). * @@ -2706,7 +2711,9 @@ char* dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider /** * Waits for the sending to finish and frees the backup provider object. * - * This should only be called once. + * This is a blocking call and should only be called once. Once this function + * returns IO can be started again using dc_accounts_start_io() or + * dc_start_io(). * * @memberof dc_backup_provider_t * @param context The context. diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 87bc517e2..9e6edb7c4 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -4171,7 +4171,7 @@ pub unsafe extern "C" fn dc_backup_provider_get_qr( ) -> *mut libc::c_char { if provider.is_null() { eprintln!("ignoring careless call to dc_backup_provider_qr"); - return ptr::null_mut(); + return "".strdup(); } let ffi_provider = &*provider; deltachat::qr::format_backup(&ffi_provider.provider.qr()) @@ -4185,7 +4185,7 @@ pub unsafe extern "C" fn dc_backup_provider_get_qr_svg( ) -> *mut libc::c_char { if provider.is_null() { eprintln!("ignoring careless call to dc_backup_provider_qr_svg()"); - return ptr::null_mut(); + return "".strdup(); } let ffi_provider = &*provider; let ctx = &*ffi_provider.context; From c9b8c5079b3fede16d380e25ee876984b516c020 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Tue, 7 Mar 2023 15:45:18 +0100 Subject: [PATCH 082/103] wording --- deltachat-ffi/deltachat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index bdf9bd025..5bfc9d54b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2646,8 +2646,8 @@ void dc_str_unref (char* str); * Creates an object for sending a backup to another device. * * Before calling this function IO must be stopped using dc_accounts_stop_io() - * or dc_stop_io(). IO should only be restarted once - * dc_backup_provider_wait() has returned. + * or dc_stop_io() so that no changes to the blobs or database are happening. + * IO should only be restarted once dc_backup_provider_wait() has returned. * * The backup is sent to through a peer-to-peer channel which is bootstrapped * by a QR-code. The backup contains the entire state of the account From c7de4f66e710a6f79e71e70297c67f6bee8f81d5 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 9 Mar 2023 15:34:15 +0100 Subject: [PATCH 083/103] Bind to 0.0.0.0 --- src/imex/transfer.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 357e8b8f1..be468b877 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -23,6 +23,7 @@ //! download to an impersonated getter. use std::future::Future; +use std::net::Ipv4Addr; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::pin::Pin; @@ -159,7 +160,10 @@ impl BackupProvider { // Start listening. let (db, hash) = iroh::provider::create_collection(files).await?; context.emit_event(SendProgress::CollectionCreated.into()); - let provider = Provider::builder(db).auth_token(token).spawn()?; + let provider = Provider::builder(db) + .bind_addr((Ipv4Addr::UNSPECIFIED, 0).into()) + .auth_token(token) + .spawn()?; context.emit_event(SendProgress::ProviderListening.into()); info!(context, "Waiting for remote to connect"); let ticket = provider.ticket(hash); From 02afacf9895b87b2fa0b9307f03cf1d8053a3b63 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 9 Mar 2023 16:12:33 +0100 Subject: [PATCH 084/103] clarify docs --- deltachat-ffi/deltachat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 5bfc9d54b..4aab53276 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2709,7 +2709,7 @@ char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider); char* dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider); /** - * Waits for the sending to finish and frees the backup provider object. + * Waits for the sending to finish. * * This is a blocking call and should only be called once. Once this function * returns IO can be started again using dc_accounts_start_io() or From 05e50ea787a251ee07667875c47a551404ac3876 Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Thu, 9 Mar 2023 16:49:34 +0100 Subject: [PATCH 085/103] Connect to all addresses the provider has This uses a branch directly from iroh repo again --- Cargo.lock | 382 +++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 +- src/imex/transfer.rs | 78 ++++++--- 3 files changed, 419 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d89ff7e23..7b6dd698b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,6 +379,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + [[package]] name = "base16ct" version = "0.1.1" @@ -415,6 +421,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitfield" version = "0.14.0" @@ -664,11 +679,39 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "bitflags", - "clap_lex", + "clap_lex 0.2.4", "indexmap", "textwrap", ] +[[package]] +name = "clap" +version = "4.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" +dependencies = [ + "bitflags", + "clap_derive", + "clap_lex 0.3.2", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "0.2.4" @@ -678,6 +721,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "clipboard-win" version = "4.5.0" @@ -737,6 +789,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.42.0", +] + [[package]] name = "const-oid" version = "0.9.2" @@ -769,6 +834,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "convert_case" version = "0.5.0" @@ -825,7 +896,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap", + "clap 3.2.23", "criterion-plot", "futures", "itertools", @@ -1061,6 +1132,38 @@ version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" +[[package]] +name = "data-encoding-macro" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86927b7cd2fe88fa698b87404b287ab98d1a0063a34071d92e575b72d3029aca" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "default-net" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e349ed1e06fb344a7dd8b5a676375cf671b31e8900075dd2be816efc063a63" +dependencies = [ + "libc", + "memalloc", + "system-configuration", + "windows", +] + [[package]] name = "deltachat" version = "1.111.0" @@ -1281,6 +1384,19 @@ dependencies = [ "syn", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case 0.4.0", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "des" version = "0.8.1" @@ -1400,6 +1516,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "educe" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0188e3c3ba8df5753894d54461f0e39bc91741dc5b22e1c46999ec2c71f4e4" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "either" version = "1.8.1" @@ -1440,6 +1568,12 @@ dependencies = [ "version_check 0.9.4", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoded-words" version = "0.2.0" @@ -1551,6 +1685,20 @@ dependencies = [ "syn", ] +[[package]] +name = "enum-ordinalize" +version = "3.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -1709,6 +1857,19 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin 0.9.5", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1875,8 +2036,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -2208,6 +2371,19 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.17.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +dependencies = [ + "console", + "number_prefix", + "portable-atomic 0.3.19", + "tokio", + "unicode-width", +] + [[package]] name = "inout" version = "0.1.3" @@ -2257,25 +2433,33 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "iroh" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7849b9b19f7f05a21aa4ed1d2b344f9154be5ac1c7de119c9263091bc7505d" +source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#b7d3e3ece526d06e39ead6a23761785e18c8dcac" dependencies = [ "abao", "anyhow", "base64 0.21.0", "blake3", "bytes", + "clap 4.1.8", + "console", + "data-encoding", + "default-net", "der", + "derive_more", "ed25519-dalek", "futures", - "portable-atomic", + "indicatif", + "multibase", + "portable-atomic 1.0.1", "postcard", + "quic-rpc", "quinn", "rand 0.7.3", "rcgen", "ring", "rustls", "serde", + "serde-error", "ssh-key", "tempfile", "thiserror", @@ -2355,7 +2539,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" dependencies = [ - "spin", + "spin 0.5.2", ] [[package]] @@ -2502,6 +2686,12 @@ dependencies = [ "digest 0.10.6", ] +[[package]] +name = "memalloc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df39d232f5c40b0891c10216992c2f250c054105cb1e56f0fc9032db6203ecc1" + [[package]] name = "memchr" version = "2.5.0" @@ -2550,12 +2740,32 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "multibase" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b3539ec3c1f04ac9748a260728e855f261b4977f5c3406612c884564f329404" +dependencies = [ + "base-x", + "data-encoding", + "data-encoding-macro", +] + [[package]] name = "mutate_once" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16cf681a23b4d0a43fc35024c176437f9dcd818db34e0f42ab456a0ee5ad497b" +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.8", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -2726,6 +2936,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.30.3" @@ -3083,6 +3299,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "portable-atomic" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" + [[package]] name = "portable-atomic" version = "1.0.1" @@ -3185,6 +3407,24 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4339fc7a1021c9c1621d87f5e3505f2805c8c105420ba2f2a4df86814590c142" +[[package]] +name = "quic-rpc" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d453504fc3e456160ae3b9ebe4d83c1f6477af167aa9b67e2d7bf11a096f179d" +dependencies = [ + "bincode", + "flume", + "futures", + "pin-project", + "quinn", + "serde", + "tokio", + "tokio-serde", + "tokio-util", + "tracing", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3512,7 +3752,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -3580,6 +3820,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -3781,6 +4030,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-error" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e988182713aeed6a619a88bca186f6d6407483485ffe44c869ee264f8eabd13f" +dependencies = [ + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.9" @@ -3963,6 +4221,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.6.0" @@ -4106,6 +4373,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "system-configuration" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagger" version = "4.3.4" @@ -4303,6 +4591,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-serde" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", +] + [[package]] name = "tokio-stream" version = "0.1.12" @@ -4929,6 +5232,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b749ebd2304aa012c5992d11a25d07b406bdbe5f79d371cb7a918ce501a19eb0" +dependencies = [ + "windows_aarch64_msvc 0.30.0", + "windows_i686_gnu 0.30.0", + "windows_i686_msvc 0.30.0", + "windows_x86_64_gnu 0.30.0", + "windows_x86_64_msvc 0.30.0", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -4936,12 +5252,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_x86_64_msvc 0.42.1", ] [[package]] @@ -4960,12 +5276,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_x86_64_msvc 0.42.1", ] [[package]] @@ -4974,24 +5290,48 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_i686_gnu" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_x86_64_gnu" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" @@ -5004,6 +5344,12 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_msvc" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" @@ -5103,7 +5449,7 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd53ff9053698697b92c2535bf7ecb983fd5d546d690b7c725e5070d6d9a620" dependencies = [ - "convert_case", + "convert_case 0.5.0", "darling 0.14.3", "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 683ebcd04..70587f969 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,8 @@ futures-lite = "1.12.0" hex = "0.4.0" humansize = "2" image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] } -iroh = { version = "0.3.0", default-features = false } +# iroh = { version = "0.3.0", default-features = false } +iroh = { git = 'https://github.com/n0-computer/iroh', branch = "flub/ticket-multiple-addrs" } kamadak-exif = "0.5" lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" } libc = "0.2" diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index be468b877..a5d1317c7 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -362,18 +362,61 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { Qr::Backup { ticket } => ticket, _ => bail!("QR code for backup must be of type DCBACKUP"), }; - let opts = Options { - addr: ticket.addr, - peer_id: Some(ticket.peer), - keylog: false, - }; + for addr in ticket.addrs.iter() { + let opts = Options { + addr: *addr, + peer_id: Some(ticket.peer), + keylog: false, + }; + match transfer_from_provider(context, &ticket, opts).await { + Ok(_) => { + delete_and_reset_all_device_msgs(context).await?; + context.emit_event(ReceiveProgress::Completed.into()); + return Ok(()); + } + Err(TransferError::ConnectionError(_)) => continue, + Err(TransferError::Other(err)) => { + // Clean up any blobs we already wrote. + let readdir = fs::read_dir(context.get_blobdir()).await?; + let mut readdir = ReadDirStream::new(readdir); + while let Some(dirent) = readdir.next().await { + if let Ok(dirent) = dirent { + fs::remove_file(dirent.path()).await.ok(); + } + } + context.emit_event(ReceiveProgress::Failed.into()); + return Err(err); + } + } + } + Err(anyhow!("failed to contact provider")) +} + +/// Error during a single transfer attempt. +/// +/// Mostly exists to distinguish between `ConnectionError` and any other errors. +#[derive(Debug, thiserror::Error)] +enum TransferError { + #[error("connection error")] + ConnectionError(#[source] anyhow::Error), + #[error("other")] + Other(#[source] anyhow::Error), +} + +async fn transfer_from_provider( + context: &Context, + ticket: &Ticket, + opts: Options, +) -> Result<(), TransferError> { let progress = ProgressEmitter::new(0, ReceiveProgress::max_blob_progress()); spawn_progress_proxy(context.clone(), progress.subscribe()); + let mut connected = false; let on_connected = || { context.emit_event(ReceiveProgress::Connected.into()); + connected = true; async { Ok(()) } }; - let on_blob = |hash, reader, name| on_blob(context, &progress, &ticket, hash, reader, name); + let on_blob = |hash, reader, name| on_blob(context, &progress, ticket, hash, reader, name); let res = iroh::get::run( ticket.hash, ticket.token, @@ -388,25 +431,10 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { ) .await; drop(progress); - match res { - Ok(_) => { - delete_and_reset_all_device_msgs(context).await?; - context.emit_event(ReceiveProgress::Completed.into()); - Ok(()) - } - Err(err) => { - // Clean up any blobs we already wrote. - let readdir = fs::read_dir(context.get_blobdir()).await?; - let mut readdir = ReadDirStream::new(readdir); - while let Some(dirent) = readdir.next().await { - if let Ok(dirent) = dirent { - fs::remove_file(dirent.path()).await.ok(); - } - } - context.emit_event(ReceiveProgress::Failed.into()); - Err(err) - } - } + res.map(|_| ()).map_err(|err| match connected { + true => TransferError::Other(err), + false => TransferError::ConnectionError(err), + }) } /// Get callback when a blob is received from the provider. From 28d9bec0b4ef9dcc502ea6ae43af96878713af8e Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 10 Mar 2023 17:14:01 +0000 Subject: [PATCH 086/103] Patch default-net --- Cargo.lock | 5 +++++ Cargo.toml | 3 +++ 2 files changed, 8 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 7b6dd698b..7e246040a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5476,3 +5476,8 @@ dependencies = [ "syn", "synstructure", ] + +[[patch.unused]] +name = "default-net" +version = "0.13.0" +source = "git+https://github.com/dignifiedquire/default-net.git?branch=feat-android#5c5fd8f0d425af673405428f77203ee370d4d6cf" diff --git a/Cargo.toml b/Cargo.toml index 70587f969..41cae54b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,9 @@ lto = true panic = 'abort' opt-level = "z" +[patch.crates-io] +default-net = { git = "https://github.com/dignifiedquire/default-net.git", branch="feat-android" } + [dependencies] deltachat_derive = { path = "./deltachat_derive" } format-flowed = { path = "./format-flowed" } From 5e4807b7ac10c9c44dccc30483039c3244579523 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 13 Mar 2023 12:51:54 +0100 Subject: [PATCH 087/103] update patched default-net --- Cargo.lock | 104 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e246040a..0b7486295 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1154,12 +1154,14 @@ dependencies = [ [[package]] name = "default-net" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e349ed1e06fb344a7dd8b5a676375cf671b31e8900075dd2be816efc063a63" +version = "0.13.1" +source = "git+https://github.com/dignifiedquire/default-net.git?branch=feat-android#64e265889aefec26d3351690c590289f892df0cf" dependencies = [ "libc", "memalloc", + "netlink-packet-core", + "netlink-packet-route", + "netlink-sys", "system-configuration", "windows", ] @@ -2433,7 +2435,7 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "iroh" version = "0.3.0" -source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#b7d3e3ece526d06e39ead6a23761785e18c8dcac" +source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#2aa7f6c240e143979fbf337f02086300df884a37" dependencies = [ "abao", "anyhow", @@ -2784,6 +2786,55 @@ dependencies = [ "tempfile", ] +[[package]] +name = "netlink-packet-core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e5cf0b54effda4b91615c40ff0fd12d0d4c9a6e0f5116874f03941792ff535a" +dependencies = [ + "anyhow", + "byteorder", + "libc", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-route" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea993e32c77d87f01236c38f572ecb6c311d592e56a06262a007fd2a6e31253c" +dependencies = [ + "anyhow", + "bitflags", + "byteorder", + "libc", + "netlink-packet-core", + "netlink-packet-utils", +] + +[[package]] +name = "netlink-packet-utils" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34" +dependencies = [ + "anyhow", + "byteorder", + "paste", + "thiserror", +] + +[[package]] +name = "netlink-sys" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" +dependencies = [ + "bytes", + "libc", + "log", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3130,6 +3181,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "paste" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" + [[package]] name = "pem" version = "1.1.1" @@ -5234,15 +5291,15 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b749ebd2304aa012c5992d11a25d07b406bdbe5f79d371cb7a918ce501a19eb0" +checksum = "fbedf6db9096bc2364adce0ae0aa636dcd89f3c3f2cd67947062aaf0ca2a10ec" dependencies = [ - "windows_aarch64_msvc 0.30.0", - "windows_i686_gnu 0.30.0", - "windows_i686_msvc 0.30.0", - "windows_x86_64_gnu 0.30.0", - "windows_x86_64_msvc 0.30.0", + "windows_aarch64_msvc 0.32.0", + "windows_i686_gnu 0.32.0", + "windows_i686_msvc 0.32.0", + "windows_x86_64_gnu 0.32.0", + "windows_x86_64_msvc 0.32.0", ] [[package]] @@ -5292,9 +5349,9 @@ checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" [[package]] name = "windows_aarch64_msvc" @@ -5304,9 +5361,9 @@ checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" [[package]] name = "windows_i686_gnu" @@ -5316,9 +5373,9 @@ checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" [[package]] name = "windows_i686_msvc" @@ -5328,9 +5385,9 @@ checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" [[package]] name = "windows_x86_64_gnu" @@ -5346,9 +5403,9 @@ checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" [[package]] name = "windows_x86_64_msvc" @@ -5476,8 +5533,3 @@ dependencies = [ "syn", "synstructure", ] - -[[patch.unused]] -name = "default-net" -version = "0.13.0" -source = "git+https://github.com/dignifiedquire/default-net.git?branch=feat-android#5c5fd8f0d425af673405428f77203ee370d4d6cf" From 14521cfc2dabcba120ae88a7b19d2755c0c6680b Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 14 Mar 2023 15:38:28 +0100 Subject: [PATCH 088/103] improve address handling --- Cargo.lock | 296 ++++++++++++++++++++++++++----------------- src/imex/transfer.rs | 4 + 2 files changed, 182 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b7486295..5af501a89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,9 +146,9 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", "synstructure", ] @@ -158,9 +158,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -266,9 +266,9 @@ version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -707,9 +707,9 @@ checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" dependencies = [ "heck", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -823,9 +823,9 @@ version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.51", + "quote 1.0.23", + "unicode-xid 0.2.4", ] [[package]] @@ -1033,10 +1033,10 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "scratch", - "syn", + "syn 1.0.109", ] [[package]] @@ -1051,9 +1051,9 @@ version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1084,10 +1084,10 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "strsim", - "syn", + "syn 1.0.109", ] [[package]] @@ -1098,10 +1098,10 @@ checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" dependencies = [ "fnv", "ident_case", - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "strsim", - "syn", + "syn 1.0.109", ] [[package]] @@ -1111,8 +1111,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote", - "syn", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1122,8 +1122,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" dependencies = [ "darling_core 0.14.3", - "quote", - "syn", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1149,19 +1149,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5bbed42daaa95e780b60a50546aa345b8413a1e46f9a40a12907d3598f038db" dependencies = [ "data-encoding", - "syn", + "syn 1.0.109", ] [[package]] name = "default-net" version = "0.13.1" -source = "git+https://github.com/dignifiedquire/default-net.git?branch=feat-android#64e265889aefec26d3351690c590289f892df0cf" +source = "git+https://github.com/dignifiedquire/default-net.git?branch=feat-android#7a257095bac009c4be0b93c2979801624fdd337b" dependencies = [ + "dlopen", "libc", "memalloc", "netlink-packet-core", "netlink-packet-route", "netlink-sys", + "once_cell", "system-configuration", "windows", ] @@ -1296,8 +1298,8 @@ dependencies = [ name = "deltachat_derive" version = "2.0.0" dependencies = [ - "quote", - "syn", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1350,9 +1352,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ef71ddb5b3a1f53dee24817c8f70dfa1cb29e804c18d88c228d4bc9c86ee3b9" dependencies = [ "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1371,9 +1373,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" dependencies = [ "darling 0.14.3", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1383,7 +1385,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" dependencies = [ "derive_builder_core", - "syn", + "syn 1.0.109", ] [[package]] @@ -1393,10 +1395,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -1476,9 +1478,32 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", +] + +[[package]] +name = "dlopen" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e80ad39f814a9abe68583cd50a2d45c8a67561c3361ab8da240587dda80937" +dependencies = [ + "dlopen_derive", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "dlopen_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f236d9e1b1fbd81cea0f9cbdc8dcc7e8ebcd80e6659cd7cb2ad5f6c05946c581" +dependencies = [ + "libc", + "quote 0.6.13", + "syn 0.15.44", ] [[package]] @@ -1525,9 +1550,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0188e3c3ba8df5753894d54461f0e39bc91741dc5b22e1c46999ec2c71f4e4" dependencies = [ "enum-ordinalize", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1682,9 +1707,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ "heck", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -1695,10 +1720,10 @@ checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a" dependencies = [ "num-bigint", "num-traits", - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -1975,9 +2000,9 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -2930,9 +2955,9 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -3050,9 +3075,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -3119,9 +3144,9 @@ checksum = "5f7d21ccd03305a674437ee1248f3ab5d4b1db095cf1caf49f1713ddf61956b7" dependencies = [ "Inflector", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -3271,9 +3296,9 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -3386,9 +3411,9 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc4b01218787dd4420daf63875163a787a78294ad48a24e9f6fa8c6507759a79" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -3414,9 +3439,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", "version_check 0.9.4", ] @@ -3426,11 +3451,20 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "version_check 0.9.4", ] +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.51" @@ -3553,13 +3587,22 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.51", ] [[package]] @@ -4111,9 +4154,9 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -4375,10 +4418,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "rustversion", - "syn", + "syn 1.0.109", ] [[package]] @@ -4387,14 +4430,25 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + [[package]] name = "syn" version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.51", + "quote 1.0.23", "unicode-ident", ] @@ -4410,10 +4464,10 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", + "unicode-xid 0.2.4", ] [[package]] @@ -4519,9 +4573,9 @@ version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -4633,9 +4687,9 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -4815,9 +4869,9 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -4973,9 +5027,9 @@ dependencies = [ "darling 0.13.4", "ident_case", "proc-macro-error", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -5027,6 +5081,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.4" @@ -5169,9 +5229,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -5193,7 +5253,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ - "quote", + "quote 1.0.23", "wasm-bindgen-macro-support", ] @@ -5203,9 +5263,9 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5508,9 +5568,9 @@ checksum = "6bd53ff9053698697b92c2535bf7ecb983fd5d546d690b7c725e5070d6d9a620" dependencies = [ "convert_case 0.5.0", "darling 0.14.3", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", ] [[package]] @@ -5528,8 +5588,8 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", "synstructure", ] diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index a5d1317c7..f240bc12f 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -362,12 +362,16 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { Qr::Backup { ticket } => ticket, _ => bail!("QR code for backup must be of type DCBACKUP"), }; + if ticket.addrs.is_empty() { + bail!("ticket is missing addresses to dial"); + } for addr in ticket.addrs.iter() { let opts = Options { addr: *addr, peer_id: Some(ticket.peer), keylog: false, }; + info!(context, "attempting to contact {}", addr); match transfer_from_provider(context, &ticket, opts).await { Ok(_) => { delete_and_reset_all_device_msgs(context).await?; From cbc18ee5a4d616f7005b30e77775f7b09ef85ecf Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 14 Mar 2023 18:49:14 +0000 Subject: [PATCH 089/103] Log connection errors --- src/imex/transfer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index f240bc12f..9524f43bb 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -378,7 +378,10 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { context.emit_event(ReceiveProgress::Completed.into()); return Ok(()); } - Err(TransferError::ConnectionError(_)) => continue, + Err(TransferError::ConnectionError(err)) => { + warn!(context, "Connection error: {err:#}."); + continue; + }, Err(TransferError::Other(err)) => { // Clean up any blobs we already wrote. let readdir = fs::read_dir(context.get_blobdir()).await?; From 56f6d6849e355216982a6356db971a6b93347dad Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 15 Mar 2023 12:45:58 +0000 Subject: [PATCH 090/103] Patch quinn to work on android --- Cargo.lock | 8 +++----- Cargo.toml | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5af501a89..b2c2e3f5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3558,8 +3558,7 @@ dependencies = [ [[package]] name = "quinn-proto" version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4ced82a24bb281af338b9e8f94429b6eca01b4e66d899f40031f074e74c9" +source = "git+https://github.com/link2xt/quinn?branch=android-no-tos#e67136966a4010564bd85861177029765a9ea811" dependencies = [ "bytes", "rand 0.8.5", @@ -3577,14 +3576,13 @@ dependencies = [ [[package]] name = "quinn-udp" version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "641538578b21f5e5c8ea733b736895576d0fe329bb883b937db6f4d163dbaaf4" +source = "git+https://github.com/link2xt/quinn?branch=android-no-tos#e67136966a4010564bd85861177029765a9ea811" dependencies = [ "libc", "quinn-proto", "socket2", "tracing", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 41cae54b5..2cdf1b08c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,8 @@ opt-level = "z" [patch.crates-io] default-net = { git = "https://github.com/dignifiedquire/default-net.git", branch="feat-android" } +quinn-udp = { git = "https://github.com/link2xt/quinn", branch="android-no-tos" } +quinn-proto = { git = "https://github.com/link2xt/quinn", branch="android-no-tos" } [dependencies] deltachat_derive = { path = "./deltachat_derive" } From 9a7ff9d2b170ab9ed5a856697ff713156971074f Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 16 Mar 2023 16:25:40 +0000 Subject: [PATCH 091/103] Switch quinn to `ecn-einval-fallback` branch. --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2c2e3f5b..5be90be3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3558,7 +3558,7 @@ dependencies = [ [[package]] name = "quinn-proto" version = "0.9.2" -source = "git+https://github.com/link2xt/quinn?branch=android-no-tos#e67136966a4010564bd85861177029765a9ea811" +source = "git+https://github.com/link2xt/quinn?branch=ecn-einval-fallback#b60a5474ae63fcc0cec0c7b104079a20b79c9ede" dependencies = [ "bytes", "rand 0.8.5", @@ -3576,7 +3576,7 @@ dependencies = [ [[package]] name = "quinn-udp" version = "0.3.2" -source = "git+https://github.com/link2xt/quinn?branch=android-no-tos#e67136966a4010564bd85861177029765a9ea811" +source = "git+https://github.com/link2xt/quinn?branch=ecn-einval-fallback#b60a5474ae63fcc0cec0c7b104079a20b79c9ede" dependencies = [ "libc", "quinn-proto", diff --git a/Cargo.toml b/Cargo.toml index 2cdf1b08c..d7f3d7249 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,8 @@ opt-level = "z" [patch.crates-io] default-net = { git = "https://github.com/dignifiedquire/default-net.git", branch="feat-android" } -quinn-udp = { git = "https://github.com/link2xt/quinn", branch="android-no-tos" } -quinn-proto = { git = "https://github.com/link2xt/quinn", branch="android-no-tos" } +quinn-udp = { git = "https://github.com/link2xt/quinn", branch="ecn-einval-fallback" } +quinn-proto = { git = "https://github.com/link2xt/quinn", branch="ecn-einval-fallback" } [dependencies] deltachat_derive = { path = "./deltachat_derive" } From c02686b56e09f8b75de7b1d10c746eadba1e19a6 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 16 Mar 2023 17:38:15 +0100 Subject: [PATCH 092/103] update iroh --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5be90be3d..682243050 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" [[package]] name = "abao" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa67f56b146a784d8feb9f5e4dc5e309fdbf7e3f6631a9c6d709348536abc628" +checksum = "2daa0989489b05a455a9707adbbbc17443edf7bbc902ce499cd3b84148d68a40" dependencies = [ "arrayref", "arrayvec", @@ -2460,7 +2460,7 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "iroh" version = "0.3.0" -source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#2aa7f6c240e143979fbf337f02086300df884a37" +source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#2149bc8c6819b044833f5af0b8fdac567ef93650" dependencies = [ "abao", "anyhow", From 7a49e9401ff73e6833ae9473e12bc5d307ac7482 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Thu, 16 Mar 2023 17:53:27 +0100 Subject: [PATCH 093/103] fixup clippy & fmt --- src/imex/transfer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 9524f43bb..e789048f4 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -365,7 +365,7 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { if ticket.addrs.is_empty() { bail!("ticket is missing addresses to dial"); } - for addr in ticket.addrs.iter() { + for addr in &ticket.addrs { let opts = Options { addr: *addr, peer_id: Some(ticket.peer), @@ -381,7 +381,7 @@ async fn get_backup_inner(context: &Context, qr: Qr) -> Result<()> { Err(TransferError::ConnectionError(err)) => { warn!(context, "Connection error: {err:#}."); continue; - }, + } Err(TransferError::Other(err)) => { // Clean up any blobs we already wrote. let readdir = fs::read_dir(context.get_blobdir()).await?; From 961e3ad7e2262ba806de00eac43171bc2f0bf1d7 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 17 Mar 2023 00:00:37 +0000 Subject: [PATCH 094/103] Update spin 0.9.5->0.9.6 --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 682243050..951934b36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1894,7 +1894,7 @@ dependencies = [ "futures-sink", "nanorand", "pin-project", - "spin 0.9.5", + "spin 0.9.6", ] [[package]] @@ -4321,9 +4321,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spin" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" +checksum = "b5d6e0250b93c8427a177b849d144a96d5acc57006149479403d7861ab721e34" dependencies = [ "lock_api", ] From 518bd19e96e0891a970b229498ad78fffa211432 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 17 Mar 2023 11:29:27 +0100 Subject: [PATCH 095/103] fix: do not block transfer on db import --- src/imex/transfer.rs | 45 +++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index e789048f4..664e8ca89 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -39,9 +39,9 @@ use iroh::provider::{DataSource, Event, Provider, Ticket}; use iroh::Hash; use tokio::fs::{self, File}; use tokio::io::{self, AsyncWriteExt, BufWriter}; -use tokio::sync::broadcast; use tokio::sync::broadcast::error::RecvError; -use tokio::task::JoinHandle; +use tokio::sync::{broadcast, Mutex}; +use tokio::task::{JoinHandle, JoinSet}; use tokio_stream::wrappers::ReadDirStream; use crate::blob::BlobDirContents; @@ -50,7 +50,7 @@ use crate::context::Context; use crate::qr::Qr; use crate::{e2ee, EventType}; -use super::{export_database, DeleteOnDrop, DBFILE_BACKUP_NAME}; +use super::{export_database, DBFILE_BACKUP_NAME}; /// Provide or send a backup of this device. /// @@ -423,7 +423,9 @@ async fn transfer_from_provider( connected = true; async { Ok(()) } }; - let on_blob = |hash, reader, name| on_blob(context, &progress, ticket, hash, reader, name); + let jobs = Mutex::new(JoinSet::default()); + let on_blob = + |hash, reader, name| on_blob(context, &progress, &jobs, ticket, hash, reader, name); let res = iroh::get::run( ticket.hash, ticket.token, @@ -437,6 +439,12 @@ async fn transfer_from_provider( on_blob, ) .await; + + let mut jobs = jobs.lock().await; + while let Some(job) = jobs.join_next().await { + job.context("job failed").map_err(TransferError::Other)?; + } + drop(progress); res.map(|_| ()).map_err(|err| match connected { true => TransferError::Other(err), @@ -451,6 +459,7 @@ async fn transfer_from_provider( async fn on_blob( context: &Context, progress: &ProgressEmitter, + jobs: &Mutex>, ticket: &Ticket, _hash: Hash, mut reader: DataStream, @@ -473,11 +482,6 @@ async fn on_blob( let blobname = name.rsplit('/').next().context("malformatted blob name")?; context.get_blobdir().join(blobname) }; - let _guard = if name.starts_with("db/") { - Some(DeleteOnDrop(path.clone())) - } else { - None - }; let mut wrapped_reader = progress.wrap_async_read(&mut reader); let file = File::create(&path).await?; @@ -486,14 +490,21 @@ async fn on_blob( file.flush().await?; if name.starts_with("db/") { - context - .sql - .import(&path, ticket.token.to_string()) - .await - .context("cannot import database")?; - fs::remove_file(&path) - .await - .with_context(|| format!("database import file: {}", path.display()))?; + let context = context.clone(); + let token = ticket.token.to_string(); + jobs.lock().await.spawn(async move { + if let Err(err) = context.sql.import(&path, token).await { + error!(context, "cannot import database: {:#?}", err); + } + if let Err(err) = fs::remove_file(&path).await { + error!( + context, + "failed to delete database import file '{}': {:#?}", + path.display(), + err, + ); + } + }); } Ok(reader) } From 0a5542a6985122211a6021c077ba15f391cb2913 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 17 Mar 2023 10:45:43 +0000 Subject: [PATCH 096/103] Log transfer rate on success --- src/imex/transfer.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 664e8ca89..900ce57bc 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -446,10 +446,20 @@ async fn transfer_from_provider( } drop(progress); - res.map(|_| ()).map_err(|err| match connected { - true => TransferError::Other(err), - false => TransferError::ConnectionError(err), - }) + match res { + Ok(stats) => { + info!( + context, + "Backup transfer finished, transfer rate is {} Mbps.", + stats.mbits() + ); + Ok(()) + } + Err(err) => match connected { + true => Err(TransferError::Other(err)), + false => Err(TransferError::ConnectionError(err)), + }, + } } /// Get callback when a blob is received from the provider. From b26a351786e4791aa594e537f2adca0017ee88cf Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 17 Mar 2023 22:26:12 +0000 Subject: [PATCH 097/103] Switch quinn to the main branch It has Android fixes merged --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 951934b36..2203661e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3558,7 +3558,7 @@ dependencies = [ [[package]] name = "quinn-proto" version = "0.9.2" -source = "git+https://github.com/link2xt/quinn?branch=ecn-einval-fallback#b60a5474ae63fcc0cec0c7b104079a20b79c9ede" +source = "git+https://github.com/quinn-rs/quinn?branch=main#11b34a7b2652010cdbbd08b5dfa407832baff927" dependencies = [ "bytes", "rand 0.8.5", @@ -3576,7 +3576,7 @@ dependencies = [ [[package]] name = "quinn-udp" version = "0.3.2" -source = "git+https://github.com/link2xt/quinn?branch=ecn-einval-fallback#b60a5474ae63fcc0cec0c7b104079a20b79c9ede" +source = "git+https://github.com/quinn-rs/quinn?branch=main#11b34a7b2652010cdbbd08b5dfa407832baff927" dependencies = [ "libc", "quinn-proto", diff --git a/Cargo.toml b/Cargo.toml index d7f3d7249..4d76bc686 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,8 +26,8 @@ opt-level = "z" [patch.crates-io] default-net = { git = "https://github.com/dignifiedquire/default-net.git", branch="feat-android" } -quinn-udp = { git = "https://github.com/link2xt/quinn", branch="ecn-einval-fallback" } -quinn-proto = { git = "https://github.com/link2xt/quinn", branch="ecn-einval-fallback" } +quinn-udp = { git = "https://github.com/quinn-rs/quinn", branch="main" } +quinn-proto = { git = "https://github.com/quinn-rs/quinn", branch="main" } [dependencies] deltachat_derive = { path = "./deltachat_derive" } From fd358617f5d297ed1cd2ca79be0f33be55e55bd9 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 17 Mar 2023 23:36:54 +0100 Subject: [PATCH 098/103] feat: implement more detailed progress on sending --- Cargo.lock | 5 ++- src/imex/transfer.rs | 79 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2203661e6..a98be1cea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2460,7 +2460,7 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "iroh" version = "0.3.0" -source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#2149bc8c6819b044833f5af0b8fdac567ef93650" +source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#9038359f14496d5f4694c398c512cdcb5358771c" dependencies = [ "abao", "anyhow", @@ -2473,10 +2473,13 @@ dependencies = [ "default-net", "der", "derive_more", + "dirs-next", "ed25519-dalek", "futures", + "hex", "indicatif", "multibase", + "num_cpus", "portable-atomic 1.0.1", "postcard", "quic-rpc", diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 900ce57bc..7fd8f5bc7 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -188,6 +188,8 @@ impl BackupProvider { ) -> Result<()> { // _dbfile exists so we can clean up the file once it is no longer needed let mut events = provider.subscribe(); + let mut total_size = 0; + let mut current_size = 0; let res = loop { tokio::select! { biased; @@ -202,10 +204,26 @@ impl BackupProvider { context.emit_event(SendProgress::ClientConnected.into()); } Event::RequestReceived { .. } => { - context.emit_event(SendProgress::TransferStarted.into()); } - Event::TransferCompleted { .. } => { - context.emit_event(SendProgress::TransferFinished.into()); + Event::TransferCollectionStarted { total_blobs_size, .. } => { + total_size = total_blobs_size; + context.emit_event(SendProgress::TransferInProgress { + current_size, + total_size, + }.into()); + } + Event::TransferBlobCompleted { size, .. } => { + current_size += size; + context.emit_event(SendProgress::TransferInProgress { + current_size, + total_size, + }.into()); + } + Event::TransferCollectionCompleted { .. } => { + context.emit_event(SendProgress::TransferInProgress { + current_size: total_size, + total_size + }.into()); provider.shutdown(); } Event::TransferAborted { .. } => { @@ -301,22 +319,37 @@ impl Deref for TempPathGuard { /// /// Plus you get warnings if you don't use all variants. #[derive(Debug)] -#[repr(u16)] enum SendProgress { - Failed = 0, - Started = 100, - DatabaseExported = 300, - CollectionCreated = 400, - ProviderListening = 500, - ClientConnected = 600, - TransferStarted = 650, - TransferFinished = 950, - Completed = 1000, + Failed, + Started, + DatabaseExported, + CollectionCreated, + ProviderListening, + ClientConnected, + TransferInProgress { current_size: u64, total_size: u64 }, + Completed, } impl From for EventType { fn from(source: SendProgress) -> Self { - Self::ImexProgress((source as u16).into()) + use SendProgress::*; + let num: u16 = match source { + Failed => 0, + Started => 100, + DatabaseExported => 300, + CollectionCreated => 350, + ProviderListening => 400, + ClientConnected => 450, + TransferInProgress { + current_size, + total_size, + } => { + // the range is 450..=950 + 450 + ((current_size as f64 / total_size as f64) * 500.).floor() as u16 + } + Completed => 1000, + }; + Self::ImexProgress(num.into()) } } @@ -645,4 +678,22 @@ mod tests { .get_matching(|ev| matches!(ev, EventType::ImexProgress(1000))) .await; } + + #[test] + fn test_send_progress() { + let cases = [ + ((0, 100), 450), + ((10, 100), 500), + ((50, 100), 700), + ((100, 100), 950), + ]; + + for ((current_size, total_size), progress) in cases { + let out = EventType::from(SendProgress::TransferInProgress { + current_size, + total_size, + }); + assert_eq!(out, EventType::ImexProgress(progress)); + } + } } From a59e72e7d8a60e55047aebad53936557080b6479 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Fri, 17 Mar 2023 23:41:36 +0100 Subject: [PATCH 099/103] update iroh --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a98be1cea..3af389c66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2460,7 +2460,7 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "iroh" version = "0.3.0" -source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#9038359f14496d5f4694c398c512cdcb5358771c" +source = "git+https://github.com/n0-computer/iroh?branch=flub/ticket-multiple-addrs#aacd1d84e11b218e55e9622c4500973224904fe9" dependencies = [ "abao", "anyhow", From eb0f896d57346abacc2be0a948e35345c32c4256 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 19 Mar 2023 11:23:09 +0000 Subject: [PATCH 100/103] Use scheduler.is_running() --- src/imex.rs | 2 +- src/imex/transfer.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imex.rs b/src/imex.rs index d8753a56d..bd965fa3d 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -750,7 +750,7 @@ where /// This also verifies that IO is not running during the export. async fn export_database(context: &Context, dest: &Path, passphrase: String) -> Result<()> { ensure!( - context.scheduler.read().await.is_none(), + !context.scheduler.is_running().await, "cannot export backup, IO is running" ); let now = time().try_into().context("32-bit UNIX time overflow")?; diff --git a/src/imex/transfer.rs b/src/imex/transfer.rs index 7fd8f5bc7..17f5650cf 100644 --- a/src/imex/transfer.rs +++ b/src/imex/transfer.rs @@ -374,7 +374,7 @@ pub async fn get_backup(context: &Context, qr: Qr) -> Result<()> { "Cannot import backups to accounts in use." ); ensure!( - context.scheduler.read().await.is_none(), + !context.scheduler.is_running().await, "cannot import backup, IO is running" ); From 3a861d2f843629f9de0c18ec2176500bc07ecece Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 19 Mar 2023 14:37:01 +0100 Subject: [PATCH 101/103] some doxygen fixes --- deltachat-ffi/deltachat.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 1484ab33b..638ab74b7 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2661,7 +2661,7 @@ void dc_str_unref (char* str); * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate * state and progress. * - * @memberof dc_backup_sender_t + * @memberof dc_backup_provider_t * @param context The context. * @return Opaque object for sending the backup. * On errors, NULL is returned and dc_get_last_error() returns an error that @@ -2681,7 +2681,6 @@ dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context); * dc_get_backup(). * * @memberof dc_backup_provider_t - * @param context The context. * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). * @return The text that should be put in the QR code. @@ -2698,7 +2697,6 @@ char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider); * SVG image containing the QR code. * * @memberof dc_backup_provider_t - * @param context The context. * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). * @return The QR code rendered as SVG. @@ -2715,7 +2713,6 @@ char* dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider * dc_start_io(). * * @memberof dc_backup_provider_t - * @param context The context. * @param backup_provider The backup provider object as created by * dc_backup_provider_new(). If NULL is given nothing is done. */ @@ -2725,6 +2722,8 @@ void dc_backup_provider_wait (dc_backup_provider_t* backup_provider); * Frees a dc_backup_provider_t object. * * @memberof dc_backup_provider_t + * @param backup_provider The backup provider object as created by + * dc_backup_provider_new(). */ void dc_backup_provider_unref (dc_backup_provider_t* backup_provider); @@ -2745,6 +2744,7 @@ void dc_backup_provider_unref (dc_backup_provider_t* backup_provider); * for showing progress and informational only, success and failure is also * shown in the return code of this function. * + * @memberof dc_context_t * @param context The context. * @param qr The qr code text, dc_check_qr() must have returned DC_QR_BACKUP * on this text. From 800d2b14a521f30745bd2ff213f97cbbcfe36dda Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 19 Mar 2023 14:37:14 +0000 Subject: [PATCH 102/103] Add cargo-deny exceptions for old crates --- deny.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deny.toml b/deny.toml index 02ef8bce7..53438b1d2 100644 --- a/deny.toml +++ b/deny.toml @@ -34,6 +34,17 @@ skip = [ { name = "version_check", version = "<0.9" }, { name = "wasi", version = "<0.11" }, { name = "windows-sys", version = "<0.45" }, + { name = "windows_x86_64_msvc", version = "<0.42" }, + { name = "windows_x86_64_gnu", version = "<0.42" }, + { name = "windows_i686_msvc", version = "<0.42" }, + { name = "windows_i686_gnu", version = "<0.42" }, + { name = "windows_aarch64_msvc", version = "<0.42" }, + { name = "unicode-xid", version = "<0.2.4" }, + { name = "syn", version = "<1.0" }, + { name = "quote", version = "<1.0" }, + { name = "proc-macro2", version = "<1.0" }, + { name = "portable-atomic", version = "<1.0" }, + { name = "spin", version = "<0.9.6" }, ] From 8e6434068e0570b756d7e42f06333d4e2146c1eb Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 19 Mar 2023 14:40:46 +0000 Subject: [PATCH 103/103] Fix remaining cargo-deny warnings --- deny.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deny.toml b/deny.toml index 53438b1d2..c620e8db0 100644 --- a/deny.toml +++ b/deny.toml @@ -45,6 +45,9 @@ skip = [ { name = "proc-macro2", version = "<1.0" }, { name = "portable-atomic", version = "<1.0" }, { name = "spin", version = "<0.9.6" }, + { name = "convert_case", version = "0.4.0" }, + { name = "clap_lex", version = "0.2.4" }, + { name = "clap", version = "3.2.23" }, ] @@ -76,4 +79,7 @@ license-files = [ github = [ "async-email", "deltachat", + "n0-computer", + "quinn-rs", + "dignifiedquire", ]