From 24f4cbbb2758c7863c379bbf606b71d8f43a1483 Mon Sep 17 00:00:00 2001 From: Friedel Ziegelmayer Date: Fri, 10 Apr 2020 22:39:28 +0200 Subject: [PATCH 001/156] refactor: replace failure - failure is deprecated - thiserror for deriving Error impl - anyhow for highlevel error handling --- Cargo.lock | 85 ++++++--------- Cargo.toml | 4 +- deltachat-ffi/Cargo.toml | 3 +- deltachat-ffi/src/lib.rs | 8 +- deltachat-ffi/src/string.rs | 7 +- examples/repl/cmdline.rs | 5 +- examples/repl/main.rs | 11 +- src/blob.rs | 100 +++--------------- src/chat.rs | 2 +- src/chatlist.rs | 2 +- src/configure/auto_mozilla.rs | 32 +----- src/configure/auto_outlook.rs | 32 +----- src/configure/mod.rs | 23 ++++ src/configure/read_url.rs | 10 +- src/contact.rs | 9 +- src/dc_receive_imf.rs | 2 +- src/dc_tools.rs | 2 +- src/e2ee.rs | 6 +- src/error.rs | 190 ++-------------------------------- src/imap/idle.rs | 38 +++---- src/imap/mod.rs | 68 ++++-------- src/imap/select_folder.rs | 14 +-- src/job.rs | 4 +- src/job_thread.rs | 34 +++--- src/key.rs | 34 ++---- src/lib.rs | 2 - src/location.rs | 2 +- src/message.rs | 9 +- src/mimefactory.rs | 2 +- src/mimeparser.rs | 7 +- src/param.rs | 2 +- src/pgp.rs | 16 ++- src/qr.rs | 22 ++-- src/securejoin.rs | 19 ++-- src/smtp/mod.rs | 28 ++--- src/smtp/send.rs | 12 +-- src/sql.rs | 58 ++++------- src/stock.rs | 2 +- 38 files changed, 250 insertions(+), 656 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1c9513eb..012ee63e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,6 +50,11 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "anyhow" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arrayref" version = "0.3.6" @@ -101,7 +106,7 @@ dependencies = [ "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", "rental 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "stop-token 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -111,7 +116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "async-std 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -626,6 +631,7 @@ dependencies = [ name = "deltachat" version = "1.28.0" dependencies = [ + "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "async-native-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "async-smtp 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -641,11 +647,9 @@ dependencies = [ "email 0.0.21 (git+https://github.com/deltachat/rust-email)", "encoded-words 0.1.0 (git+https://github.com/async-email/encoded-words)", "escaper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.22.5 (registry+https://github.com/rust-lang/crates.io-index)", - "image-meta 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "image-meta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -677,6 +681,7 @@ dependencies = [ "strum 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "strum_macros 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "thread-local-object 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -692,12 +697,13 @@ dependencies = [ name = "deltachat_ffi" version = "1.28.0" dependencies = [ + "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "deltachat 1.28.0", - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -808,7 +814,7 @@ dependencies = [ "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1194,18 +1200,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "human-panic" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "os_type 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", - "uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1282,12 +1286,12 @@ dependencies = [ [[package]] name = "image-meta" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "skeptic 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1852,7 +1856,7 @@ dependencies = [ "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "twofish 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "x25519-dalek 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2686,14 +2690,6 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "termcolor" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "termcolor" version = "1.1.0" @@ -2704,15 +2700,15 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "thiserror-impl 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "thiserror-impl" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2786,7 +2782,7 @@ dependencies = [ [[package]] name = "toml" -version = "0.4.10" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2897,15 +2893,6 @@ name = "utf8parse" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "uuid" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "uuid" version = "0.8.1" @@ -3074,14 +3061,6 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "wincolor" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "winreg" version = "0.6.2" @@ -3143,6 +3122,7 @@ dependencies = [ "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" "checksum aho-corasick 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e63fd144e18ba274ae7095c0197a870a7b9468abc801dd62f190d80817d2ec" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff" "checksum arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" "checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" @@ -3268,14 +3248,14 @@ dependencies = [ "checksum http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b" "checksum http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" -"checksum human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "21638c5955a6daf3ecc42cae702335fc37a72a4abcc6959ce457b31a7d43bbdd" +"checksum human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "39f357a500abcbd7c5f967c1d45c8838585b36743823b9d43488f24850534e36" "checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" "checksum hyper 0.13.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e7b15203263d1faa615f9337d79c1d37959439dc46c2b4faab33286fadc2a1c5" "checksum hyper-tls 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" "checksum ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" "checksum image 0.22.5 (registry+https://github.com/rust-lang/crates.io-index)" = "08ed2ada878397b045454ac7cfb011d73132c59f31a955d230bd1f1c2e68eb4a" -"checksum image-meta 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b00861cbbb254a627d8acc0cec786b484297d896ab8f20fdc8e28536a3e918ef" +"checksum image-meta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a2da7b4225d0954c9b8ba1a0dcec85be29f496cba4d85f9390426f810e3ab0d" "checksum imap-proto 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "16a6def1d5ac8975d70b3fd101d57953fe3278ef2ee5d7816cba54b1d1dfc22f" "checksum indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" "checksum inflate 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff" @@ -3428,17 +3408,16 @@ dependencies = [ "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" -"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83" "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" -"checksum thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ee14bf8e6767ab4c687c9e8bc003879e042a96fd67a3ba5934eadb6536bef4db" -"checksum thiserror-impl 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a7b51e1fbc44b5a0840be594fbc0f960be09050f2617e61e6aa43bef97cd3ef4" +"checksum thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "f0570dc61221295909abdb95c739f2e74325e14293b2026b0a7e195091ec54ae" +"checksum thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "227362df41d566be41a28f64401e07a043157c21c14b9785a0d8e256f940a8fd" "checksum thread-local-object 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7da3caa820d0308c84c8654f6cafd81cc3195d45433311cbe22fcf44fc8be071" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tokio 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "0fa5e81d6bc4e67fe889d5783bd2a128ab2e0cfa487e0be16b6a8d177b101616" "checksum tokio-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828" "checksum tokio-util 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" -"checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +"checksum toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" "checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" @@ -3455,7 +3434,6 @@ dependencies = [ "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" -"checksum uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e1436e58182935dcd9ce0add9ea0b558e8a87befe01c1a301e6020aeb0876363" "checksum uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" "checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" @@ -3478,7 +3456,6 @@ dependencies = [ "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767" "checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" "checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/Cargo.toml b/Cargo.toml index 72cb02b89..7d85672b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,6 @@ percent-encoding = "2.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" chrono = "0.4.6" -failure = "0.1.5" -failure_derive = "0.1.5" indexmap = "1.3.0" lazy_static = "1.4.0" regex = "1.1.6" @@ -60,6 +58,8 @@ image = { version = "0.22.4", default-features=false, features = ["gif_codec", " pretty_env_logger = "0.3.1" rustyline = { version = "4.1.0", optional = true } +thiserror = "1.0.14" +anyhow = "1.0.28" [dev-dependencies] tempfile = "3.0" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 8901dffec..593dd9c26 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -19,8 +19,9 @@ deltachat = { path = "../", default-features = false } libc = "0.2" human-panic = "1.0.1" num-traits = "0.2.6" -failure = "0.1.6" serde_json = "1.0" +anyhow = "1.0.28" +thiserror = "1.0.14" [features] default = ["vendored", "nightly"] diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index f1019650b..74773f18e 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -120,17 +120,17 @@ impl ContextWrapper { /// Unlock the context and execute a closure with it. /// /// This is like [ContextWrapper::with_inner] but uses - /// [failure::Error] as error type. This allows you to write a + /// [anyhow::Error] as error type. This allows you to write a /// closure which could produce many errors, use the `?` operator /// to return them and handle them all as the return of this call. - fn try_inner(&self, ctxfn: F) -> Result + fn try_inner(&self, ctxfn: F) -> Result where - F: FnOnce(&Context) -> Result, + F: FnOnce(&Context) -> Result, { let guard = self.inner.read().unwrap(); match guard.as_ref() { Some(ref ctx) => ctxfn(ctx), - None => Err(failure::err_msg("context not open")), + None => Err(anyhow::format_err!("context not open")), } } diff --git a/deltachat-ffi/src/string.rs b/deltachat-ffi/src/string.rs index 256b110b0..db9baa7bb 100644 --- a/deltachat-ffi/src/string.rs +++ b/deltachat-ffi/src/string.rs @@ -1,4 +1,3 @@ -use failure::Fail; use std::ffi::{CStr, CString}; use std::ptr; @@ -31,13 +30,13 @@ unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char { } /// Error type for the [OsStrExt] trait -#[derive(Debug, Fail, PartialEq)] +#[derive(Debug, PartialEq, thiserror::Error)] pub(crate) enum CStringError { /// The string contains an interior null byte - #[fail(display = "String contains an interior null byte")] + #[error("String contains an interior null byte")] InteriorNullByte, /// The string is not valid Unicode - #[fail(display = "String is not valid unicode")] + #[error("String is not valid unicode")] NotUnicode, } diff --git a/examples/repl/cmdline.rs b/examples/repl/cmdline.rs index 926287e8f..d3842f42f 100644 --- a/examples/repl/cmdline.rs +++ b/examples/repl/cmdline.rs @@ -1,6 +1,7 @@ use std::path::Path; use std::str::FromStr; +use anyhow::{bail, ensure}; use deltachat::chat::{self, Chat, ChatId, ChatVisibility}; use deltachat::chatlist::*; use deltachat::constants::*; @@ -91,7 +92,7 @@ fn dc_reset_tables(context: &Context, bits: i32) -> i32 { 1 } -fn dc_poke_eml_file(context: &Context, filename: impl AsRef) -> Result<(), Error> { +fn dc_poke_eml_file(context: &Context, filename: impl AsRef) -> Result<(), anyhow::Error> { let data = dc_read_file(context, filename)?; if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) { @@ -297,7 +298,7 @@ fn chat_prefix(chat: &Chat) -> &'static str { chat.typ.into() } -pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> { +pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> { let chat_id = *context.cmdline_sel_chat_id.read().unwrap(); let mut sel_chat = if !chat_id.is_unset() { Chat::load_from_db(context, chat_id).ok() diff --git a/examples/repl/main.rs b/examples/repl/main.rs index e34deadd8..c4662cad3 100644 --- a/examples/repl/main.rs +++ b/examples/repl/main.rs @@ -7,8 +7,6 @@ #[macro_use] extern crate deltachat; #[macro_use] -extern crate failure; -#[macro_use] extern crate lazy_static; #[macro_use] extern crate rusqlite; @@ -20,6 +18,7 @@ use std::process::Command; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, RwLock}; +use anyhow::{bail, Error}; use deltachat::chat::ChatId; use deltachat::config; use deltachat::context::*; @@ -370,10 +369,10 @@ impl Highlighter for DcHelper { impl Helper for DcHelper {} -fn main_0(args: Vec) -> Result<(), failure::Error> { +fn main_0(args: Vec) -> Result<(), Error> { if args.len() < 2 { println!("Error: Bad arguments, expected [db-name]."); - return Err(format_err!("No db-name specified")); + bail!("No db-name specified"); } let context = Context::new( Box::new(receive_event), @@ -443,7 +442,7 @@ enum ExitResult { Exit, } -fn handle_cmd(line: &str, ctx: Arc>) -> Result { +fn handle_cmd(line: &str, ctx: Arc>) -> Result { let mut args = line.splitn(2, ' '); let arg0 = args.next().unwrap_or_default(); let arg1 = args.next().unwrap_or_default(); @@ -526,7 +525,7 @@ fn handle_cmd(line: &str, ctx: Arc>) -> Result Result<(), failure::Error> { +pub fn main() -> Result<(), Error> { let _ = pretty_env_logger::try_init(); let args: Vec = std::env::args().collect(); diff --git a/src/blob.rs b/src/blob.rs index 70d5ff7a0..f5bd2f47f 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -6,13 +6,13 @@ use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -use self::image::GenericImageView; +use image::GenericImageView; +use thiserror::Error; + use crate::constants::AVATAR_SIZE; use crate::context::Context; use crate::events::Event; -extern crate image; - /// Represents a file in the blob directory. /// /// The object has a name, which will always be valid UTF-8. Having a @@ -56,7 +56,6 @@ impl<'a> BlobObject<'a> { blobdir: blobdir.to_path_buf(), blobname: name.clone(), cause: err, - backtrace: failure::Backtrace::new(), })?; let blob = BlobObject { blobdir, @@ -84,7 +83,6 @@ impl<'a> BlobObject<'a> { blobdir: dir.to_path_buf(), blobname: name, cause: err, - backtrace: failure::Backtrace::new(), }); } else { name = format!("{}-{}{}", stem, rand::random::(), ext); @@ -97,7 +95,6 @@ impl<'a> BlobObject<'a> { blobdir: dir.to_path_buf(), blobname: name, cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"), - backtrace: failure::Backtrace::new(), }) } @@ -122,7 +119,6 @@ impl<'a> BlobObject<'a> { blobname: String::from(""), src: src.as_ref().to_path_buf(), cause: err, - backtrace: failure::Backtrace::new(), })?; let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy()); let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?; @@ -138,7 +134,6 @@ impl<'a> BlobObject<'a> { blobname: name_for_err, src: src.as_ref().to_path_buf(), cause: err, - backtrace: failure::Backtrace::new(), } })?; let blob = BlobObject { @@ -198,17 +193,14 @@ impl<'a> BlobObject<'a> { .map_err(|_| BlobError::WrongBlobdir { blobdir: context.get_blobdir().to_path_buf(), src: path.as_ref().to_path_buf(), - backtrace: failure::Backtrace::new(), })?; if !BlobObject::is_acceptible_blob_name(&rel_path) { return Err(BlobError::WrongName { blobname: path.as_ref().to_path_buf(), - backtrace: failure::Backtrace::new(), }); } let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName { blobname: path.as_ref().to_path_buf(), - backtrace: failure::Backtrace::new(), })?; BlobObject::from_name(context, name.to_string()) } @@ -236,7 +228,6 @@ impl<'a> BlobObject<'a> { if !BlobObject::is_acceptible_blob_name(&name) { return Err(BlobError::WrongName { blobname: PathBuf::from(name), - backtrace: failure::Backtrace::new(), }); } Ok(BlobObject { @@ -359,7 +350,6 @@ impl<'a> BlobObject<'a> { blobdir: context.get_blobdir().to_path_buf(), blobname: blob_abs.to_str().unwrap_or_default().to_string(), cause: err, - backtrace: failure::Backtrace::new(), })?; if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE { @@ -372,7 +362,6 @@ impl<'a> BlobObject<'a> { blobdir: context.get_blobdir().to_path_buf(), blobname: blob_abs.to_str().unwrap_or_default().to_string(), cause: err, - backtrace: failure::Backtrace::new(), })?; Ok(()) @@ -386,98 +375,41 @@ impl<'a> fmt::Display for BlobObject<'a> { } /// Errors for the [BlobObject]. -#[derive(Fail, Debug)] +#[derive(Debug, Error)] pub enum BlobError { + #[error("Failed to create blob {blobname} in {}", .blobdir.display())] CreateFailure { blobdir: PathBuf, blobname: String, - #[cause] + #[source] cause: std::io::Error, - backtrace: failure::Backtrace, }, + #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())] WriteFailure { blobdir: PathBuf, blobname: String, - #[cause] + #[source] cause: std::io::Error, - backtrace: failure::Backtrace, }, + #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())] CopyFailure { blobdir: PathBuf, blobname: String, src: PathBuf, - #[cause] + #[source] cause: std::io::Error, - backtrace: failure::Backtrace, }, + #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())] RecodeFailure { blobdir: PathBuf, blobname: String, - #[cause] + #[source] cause: image::ImageError, - backtrace: failure::Backtrace, }, - WrongBlobdir { - blobdir: PathBuf, - src: PathBuf, - backtrace: failure::Backtrace, - }, - WrongName { - blobname: PathBuf, - backtrace: failure::Backtrace, - }, -} - -// Implementing Display is done by hand because the failure -// #[fail(display = "...")] syntax does not allow using -// `blobdir.display()`. -impl fmt::Display for BlobError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - // Match on the data rather than kind, they are equivalent for - // identifying purposes but contain the actual data we need. - match &self { - BlobError::CreateFailure { - blobdir, blobname, .. - } => write!( - f, - "Failed to create blob {} in {}", - blobname, - blobdir.display() - ), - BlobError::WriteFailure { - blobdir, blobname, .. - } => write!( - f, - "Failed to write data to blob {} in {}", - blobname, - blobdir.display() - ), - BlobError::CopyFailure { - blobdir, - blobname, - src, - .. - } => write!( - f, - "Failed to copy data from {} to blob {} in {}", - src.display(), - blobname, - blobdir.display(), - ), - BlobError::RecodeFailure { - blobdir, blobname, .. - } => write!(f, "Failed to recode {} in {}", blobname, blobdir.display(),), - BlobError::WrongBlobdir { blobdir, src, .. } => write!( - f, - "File path {} is not in blobdir {}", - src.display(), - blobdir.display(), - ), - BlobError::WrongName { blobname, .. } => { - write!(f, "Blob has a bad name: {}", blobname.display(),) - } - } - } + #[error("File path {} is not in {}", .src.display(), .blobdir.display())] + WrongBlobdir { blobdir: PathBuf, src: PathBuf }, + #[error("Blob has a badname {}", .blobname.display())] + WrongName { blobname: PathBuf }, } #[cfg(test)] diff --git a/src/chat.rs b/src/chat.rs index e89a0561e..3ac2000e2 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -15,7 +15,7 @@ use crate::constants::*; use crate::contact::*; use crate::context::Context; use crate::dc_tools::*; -use crate::error::Error; +use crate::error::{bail, ensure, format_err, Error}; use crate::events::Event; use crate::job::*; use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId}; diff --git a/src/chatlist.rs b/src/chatlist.rs index 00380ed7f..cec5b520c 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -5,7 +5,7 @@ use crate::chat::*; use crate::constants::*; use crate::contact::*; use crate::context::*; -use crate::error::Result; +use crate::error::{ensure, Result}; use crate::lot::Lot; use crate::message::{Message, MessageState, MsgId}; use crate::stock::StockMessage; diff --git a/src/configure/auto_mozilla.rs b/src/configure/auto_mozilla.rs index a88fdedda..348a24fd9 100644 --- a/src/configure/auto_mozilla.rs +++ b/src/configure/auto_mozilla.rs @@ -9,33 +9,7 @@ use crate::context::Context; use crate::login_param::LoginParam; use super::read_url::read_url; - -#[derive(Debug, Fail)] -pub enum Error { - #[fail(display = "Invalid email address: {:?}", _0)] - InvalidEmailAddress(String), - - #[fail(display = "XML error at position {}", position)] - InvalidXml { - position: usize, - #[cause] - error: quick_xml::Error, - }, - - #[fail(display = "Bad or incomplete autoconfig")] - IncompleteAutoconfig(LoginParam), - - #[fail(display = "Failed to get URL {}", _0)] - ReadUrlError(#[cause] super::read_url::Error), -} - -pub type Result = std::result::Result; - -impl From for Error { - fn from(err: super::read_url::Error) -> Error { - Error::ReadUrlError(err) - } -} +use super::Error; #[derive(Debug)] struct MozAutoconfigure<'a> { @@ -65,7 +39,7 @@ enum MozConfigTag { Username, } -fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result { +fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result { let mut reader = quick_xml::Reader::from_str(xml_raw); reader.trim_text(true); @@ -125,7 +99,7 @@ pub fn moz_autoconfigure( context: &Context, url: &str, param_in: &LoginParam, -) -> Result { +) -> Result { let xml_raw = read_url(context, url)?; let res = parse_xml(¶m_in.addr, &xml_raw); diff --git a/src/configure/auto_outlook.rs b/src/configure/auto_outlook.rs index f7d1db31f..1e8cbba10 100644 --- a/src/configure/auto_outlook.rs +++ b/src/configure/auto_outlook.rs @@ -8,33 +8,7 @@ use crate::context::Context; use crate::login_param::LoginParam; use super::read_url::read_url; - -#[derive(Debug, Fail)] -pub enum Error { - #[fail(display = "XML error at position {}", position)] - InvalidXml { - position: usize, - #[cause] - error: quick_xml::Error, - }, - - #[fail(display = "Bad or incomplete autoconfig")] - IncompleteAutoconfig(LoginParam), - - #[fail(display = "Failed to get URL {}", _0)] - ReadUrlError(#[cause] super::read_url::Error), - - #[fail(display = "Number of redirection is exceeded")] - RedirectionError, -} - -pub type Result = std::result::Result; - -impl From for Error { - fn from(err: super::read_url::Error) -> Error { - Error::ReadUrlError(err) - } -} +use super::Error; struct OutlookAutodiscover { pub out: LoginParam, @@ -52,7 +26,7 @@ enum ParsingResult { RedirectUrl(String), } -fn parse_xml(xml_raw: &str) -> Result { +fn parse_xml(xml_raw: &str) -> Result { let mut outlk_ad = OutlookAutodiscover { out: LoginParam::new(), out_imap_set: false, @@ -143,7 +117,7 @@ pub fn outlk_autodiscover( context: &Context, url: &str, _param_in: &LoginParam, -) -> Result { +) -> Result { let mut url = url.to_string(); /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */ for _i in 0..10 { diff --git a/src/configure/mod.rs b/src/configure/mod.rs index 74ade9595..ab0e70aeb 100644 --- a/src/configure/mod.rs +++ b/src/configure/mod.rs @@ -12,6 +12,7 @@ use crate::config::Config; use crate::constants::*; use crate::context::Context; use crate::dc_tools::*; +use crate::error::format_err; use crate::job::{self, job_add, job_kill_action}; use crate::login_param::{CertificateChecks, LoginParam}; use crate::oauth2::*; @@ -651,6 +652,28 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option { } } +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Invalid email address: {0:?}")] + InvalidEmailAddress(String), + + #[error("XML error at position {position}")] + InvalidXml { + position: usize, + #[source] + error: quick_xml::Error, + }, + + #[error("Bad or incomplete autoconfig")] + IncompleteAutoconfig(LoginParam), + + #[error("Failed to get URL")] + ReadUrlError(#[from] self::read_url::Error), + + #[error("Number of redirection is exceeded")] + RedirectionError, +} + #[cfg(test)] mod tests { diff --git a/src/configure/read_url.rs b/src/configure/read_url.rs index 0688250f0..498f8e0c8 100644 --- a/src/configure/read_url.rs +++ b/src/configure/read_url.rs @@ -1,14 +1,12 @@ use crate::context::Context; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "URL request error")] - GetError(#[cause] reqwest::Error), + #[error("URL request error")] + GetError(#[from] reqwest::Error), } -pub type Result = std::result::Result; - -pub fn read_url(context: &Context, url: &str) -> Result { +pub fn read_url(context: &Context, url: &str) -> Result { info!(context, "Requesting URL {}", url); match reqwest::blocking::Client::new() diff --git a/src/contact.rs b/src/contact.rs index bc1f3b45d..65ff35ccf 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -13,7 +13,7 @@ use crate::constants::*; use crate::context::Context; use crate::dc_tools::*; use crate::e2ee; -use crate::error::{Error, Result}; +use crate::error::{bail, ensure, format_err, Result}; use crate::events::Event; use crate::key::*; use crate::login_param::LoginParam; @@ -1120,10 +1120,9 @@ fn cat_fingerprint( impl Context { /// determine whether the specified addr maps to the/a self addr pub fn is_self_addr(&self, addr: &str) -> Result { - let self_addr = match self.get_config(Config::ConfiguredAddr) { - Some(s) => s, - None => return Err(Error::NotConfigured), - }; + let self_addr = self + .get_config(Config::ConfiguredAddr) + .ok_or_else(|| format_err!("Not configured"))?; Ok(addr_cmp(self_addr, addr)) } diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index b50a66323..6620202db 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -9,7 +9,7 @@ use crate::constants::*; use crate::contact::*; use crate::context::Context; use crate::dc_tools::*; -use crate::error::Result; +use crate::error::{bail, ensure, Result}; use crate::events::Event; use crate::headerdef::HeaderDef; use crate::job::*; diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 0aa7bbeeb..19c7a6746 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -12,7 +12,7 @@ use chrono::{Local, TimeZone}; use rand::{thread_rng, Rng}; use crate::context::Context; -use crate::error::Error; +use crate::error::{bail, ensure, Error}; use crate::events::Event; pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool { diff --git a/src/e2ee.rs b/src/e2ee.rs index 0f40b3d1f..ec101fc91 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -200,15 +200,13 @@ fn load_or_generate_self_public_key( self_addr: impl AsRef, ) -> Result { if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) { - return SignedPublicKey::try_from(key) - .map_err(|_| Error::Message("Not a public key".into())); + return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key")); } let _guard = context.generating_key_mutex.lock().unwrap(); // Check again in case the key was generated while we were waiting for the lock. if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) { - return SignedPublicKey::try_from(key) - .map_err(|_| Error::Message("Not a public key".into())); + return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key")); } let start = std::time::Instant::now(); diff --git a/src/error.rs b/src/error.rs index 9381fd0f4..20e2782a9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,189 +1,13 @@ //! # Error handling -use lettre_email::mime; +pub use anyhow::{bail, ensure, format_err, Error, Result}; -#[derive(Debug, Fail)] -pub enum Error { - #[fail(display = "{:?}", _0)] - Failure(failure::Error), - - #[fail(display = "SQL error: {:?}", _0)] - SqlError(#[cause] crate::sql::Error), - - #[fail(display = "{:?}", _0)] - Io(std::io::Error), - - #[fail(display = "{:?}", _0)] - Message(String), - - #[fail(display = "{:?}", _0)] - MessageWithCause(String, #[cause] failure::Error, failure::Backtrace), - - #[fail(display = "{:?}", _0)] - Image(image_meta::ImageError), - - #[fail(display = "{:?}", _0)] - Utf8(std::str::Utf8Error), - - #[fail(display = "PGP: {:?}", _0)] - Pgp(pgp::errors::Error), - - #[fail(display = "Base64Decode: {:?}", _0)] - Base64Decode(base64::DecodeError), - - #[fail(display = "{:?}", _0)] - FromUtf8(std::string::FromUtf8Error), - - #[fail(display = "{}", _0)] - BlobError(#[cause] crate::blob::BlobError), - - #[fail(display = "Invalid Message ID.")] - InvalidMsgId, - - #[fail(display = "Watch folder not found {:?}", _0)] - WatchFolderNotFound(String), - - #[fail(display = "Invalid Email: {:?}", _0)] - MailParseError(#[cause] mailparse::MailParseError), - - #[fail(display = "Building invalid Email: {:?}", _0)] - LettreError(#[cause] lettre_email::error::Error), - - #[fail(display = "SMTP error: {:?}", _0)] - SmtpError(#[cause] async_smtp::error::Error), - - #[fail(display = "FromStr error: {:?}", _0)] - FromStr(#[cause] mime::FromStrError), - - #[fail(display = "Not Configured")] - NotConfigured, -} - -pub type Result = std::result::Result; - -impl From for Error { - fn from(err: crate::sql::Error) -> Error { - Error::SqlError(err) - } -} - -impl From for Error { - fn from(err: base64::DecodeError) -> Error { - Error::Base64Decode(err) - } -} - -impl From for Error { - fn from(err: failure::Error) -> Error { - Error::Failure(err) - } -} - -impl From for Error { - fn from(err: std::io::Error) -> Error { - Error::Io(err) - } -} - -impl From for Error { - fn from(err: std::str::Utf8Error) -> Error { - Error::Utf8(err) - } -} - -impl From for Error { - fn from(err: image_meta::ImageError) -> Error { - Error::Image(err) - } -} - -impl From for Error { - fn from(err: pgp::errors::Error) -> Error { - Error::Pgp(err) - } -} - -impl From for Error { - fn from(err: std::string::FromUtf8Error) -> Error { - Error::FromUtf8(err) - } -} - -impl From for Error { - fn from(err: crate::blob::BlobError) -> Error { - Error::BlobError(err) - } -} - -impl From for Error { - fn from(_err: crate::message::InvalidMsgId) -> Error { - Error::InvalidMsgId - } -} - -impl From for Error { - fn from(err: crate::key::SaveKeyError) -> Error { - Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new()) - } -} - -impl From for Error { - fn from(err: crate::pgp::PgpKeygenError) -> Error { - Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new()) - } -} - -impl From for Error { - fn from(err: mailparse::MailParseError) -> Error { - Error::MailParseError(err) - } -} - -impl From for Error { - fn from(err: lettre_email::error::Error) -> Error { - Error::LettreError(err) - } -} - -impl From for Error { - fn from(err: mime::FromStrError) -> Error { - Error::FromStr(err) - } -} - -#[macro_export] -macro_rules! bail { - ($e:expr) => { - return Err($crate::error::Error::Message($e.to_string())); - }; - ($fmt:expr, $($arg:tt)+) => { - return Err($crate::error::Error::Message(format!($fmt, $($arg)+))); - }; -} - -#[macro_export] -macro_rules! format_err { - ($e:expr) => { - $crate::error::Error::Message($e.to_string()); - }; - ($fmt:expr, $($arg:tt)+) => { - $crate::error::Error::Message(format!($fmt, $($arg)+)); - }; -} - -#[macro_export(local_inner_macros)] -macro_rules! ensure { - ($cond:expr, $e:expr) => { - if !($cond) { - bail!($e); - } - }; - ($cond:expr, $fmt:expr, $($arg:tt)+) => { - if !($cond) { - bail!($fmt, $($arg)+); - } - }; -} +// #[fail(display = "Invalid Message ID.")] +// InvalidMsgId, +// #[fail(display = "Watch folder not found {:?}", _0)] +// WatchFolderNotFound(String), +// #[fail(display = "Not Configured")] +// NotConfigured, #[macro_export] macro_rules! ensure_eq { diff --git a/src/imap/idle.rs b/src/imap/idle.rs index 86f0f292c..30e27b9ee 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -15,28 +15,22 @@ use super::session::Session; type Result = std::result::Result; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "IMAP IDLE protocol failed to init/complete")] - IdleProtocolFailed(#[cause] async_imap::error::Error), + #[error("IMAP IDLE protocol failed to init/complete")] + IdleProtocolFailed(#[from] async_imap::error::Error), - #[fail(display = "IMAP IDLE protocol timed out")] - IdleTimeout(#[cause] async_std::future::TimeoutError), + #[error("IMAP IDLE protocol timed out")] + IdleTimeout(#[from] async_std::future::TimeoutError), - #[fail(display = "IMAP server does not have IDLE capability")] + #[error("IMAP server does not have IDLE capability")] IdleAbilityMissing, - #[fail(display = "IMAP select folder error")] - SelectFolderError(#[cause] select_folder::Error), + #[error("IMAP select folder error")] + SelectFolderError(#[from] select_folder::Error), - #[fail(display = "Setup handle error")] - SetupHandleError(#[cause] super::Error), -} - -impl From for Error { - fn from(err: select_folder::Error) -> Error { - Error::SelectFolderError(err) - } + #[error("Setup handle error")] + SetupHandleError(#[from] super::Error), } #[derive(Debug)] @@ -71,9 +65,7 @@ impl Imap { return Err(Error::IdleAbilityMissing); } - self.setup_handle_if_needed(context) - .await - .map_err(Error::SetupHandleError)?; + self.setup_handle_if_needed(context).await?; self.select_folder(context, watch_folder.clone()).await?; @@ -84,9 +76,7 @@ impl Imap { // BEWARE: If you change the Secure branch you // typically also need to change the Insecure branch. IdleHandle::Secure(mut handle) => { - if let Err(err) = handle.init().await { - return Err(Error::IdleProtocolFailed(err)); - } + handle.init().await?; let (idle_wait, interrupt) = handle.wait_with_timeout(timeout); *self.interrupt.lock().await = Some(interrupt); @@ -141,9 +131,7 @@ impl Imap { } } IdleHandle::Insecure(mut handle) => { - if let Err(err) = handle.init().await { - return Err(Error::IdleProtocolFailed(err)); - } + handle.init().await?; let (idle_wait, interrupt) = handle.wait_with_timeout(timeout); *self.interrupt.lock().await = Some(interrupt); diff --git a/src/imap/mod.rs b/src/imap/mod.rs index f3ba8d90d..24937faf3 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -39,78 +39,48 @@ use session::Session; type Result = std::result::Result; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "IMAP Connect without configured params")] + #[error("IMAP Connect without configured params")] ConnectWithoutConfigure, - #[fail(display = "IMAP Connection Failed params: {}", _0)] + #[error("IMAP Connection Failed params: {0}")] ConnectionFailed(String), - #[fail(display = "IMAP No Connection established")] + #[error("IMAP No Connection established")] NoConnection, - #[fail(display = "IMAP Could not get OAUTH token")] + #[error("IMAP Could not get OAUTH token")] OauthError, - #[fail(display = "IMAP Could not login as {}", _0)] + #[error("IMAP Could not login as {0}")] LoginFailed(String), - #[fail(display = "IMAP Could not fetch")] - FetchFailed(#[cause] async_imap::error::Error), + #[error("IMAP Could not fetch")] + FetchFailed(#[from] async_imap::error::Error), - #[fail(display = "IMAP operation attempted while it is torn down")] + #[error("IMAP operation attempted while it is torn down")] InTeardown, - #[fail(display = "IMAP operation attempted while it is torn down")] - SqlError(#[cause] crate::sql::Error), + #[error("IMAP operation attempted while it is torn down")] + SqlError(#[from] crate::sql::Error), - #[fail(display = "IMAP got error from elsewhere")] - WrappedError(#[cause] crate::error::Error), + #[error("IMAP got error from elsewhere")] + WrappedError(#[from] crate::error::Error), - #[fail(display = "IMAP select folder error")] - SelectFolderError(#[cause] select_folder::Error), + #[error("IMAP select folder error")] + SelectFolderError(#[from] select_folder::Error), - #[fail(display = "Mail parse error")] - MailParseError(#[cause] mailparse::MailParseError), + #[error("Mail parse error")] + MailParseError(#[from] mailparse::MailParseError), - #[fail(display = "No mailbox selected, folder: {:?}", _0)] + #[error("No mailbox selected, folder: {0}")] NoMailbox(String), - #[fail(display = "IMAP other error: {:?}", _0)] + #[error("IMAP other error: {0}")] Other(String), } -impl From for Error { - fn from(err: crate::sql::Error) -> Error { - Error::SqlError(err) - } -} - -impl From for Error { - fn from(err: crate::error::Error) -> Error { - Error::WrappedError(err) - } -} - -impl From for crate::error::Error { - fn from(err: Error) -> crate::error::Error { - crate::error::Error::Message(err.to_string()) - } -} - -impl From for Error { - fn from(err: select_folder::Error) -> Error { - Error::SelectFolderError(err) - } -} - -impl From for Error { - fn from(err: mailparse::MailParseError) -> Error { - Error::MailParseError(err) - } -} - #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)] pub enum ImapActionResult { Failed, diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index 36c2dbf6b..c794611fd 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -4,21 +4,21 @@ use crate::context::Context; type Result = std::result::Result; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "IMAP Could not obtain imap-session object.")] + #[error("IMAP Could not obtain imap-session object.")] NoSession, - #[fail(display = "IMAP Connection Lost or no connection established")] + #[error("IMAP Connection Lost or no connection established")] ConnectionLost, - #[fail(display = "IMAP Folder name invalid: {:?}", _0)] + #[error("IMAP Folder name invalid: {0}")] BadFolderName(String), - #[fail(display = "IMAP close/expunge failed: {}", _0)] - CloseExpungeFailed(#[cause] async_imap::error::Error), + #[error("IMAP close/expunge failed")] + CloseExpungeFailed(#[from] async_imap::error::Error), - #[fail(display = "IMAP other error: {:?}", _0)] + #[error("IMAP other error: {0}")] Other(String), } diff --git a/src/job.rs b/src/job.rs index 7d6f08946..d53e84224 100644 --- a/src/job.rs +++ b/src/job.rs @@ -19,7 +19,7 @@ use crate::constants::*; use crate::contact::Contact; use crate::context::{Context, PerformJobsNeeded}; use crate::dc_tools::*; -use crate::error::{Error, Result}; +use crate::error::{bail, ensure, format_err, Error, Result}; use crate::events::Event; use crate::imap::*; use crate::imex::*; @@ -227,7 +227,7 @@ impl Job { // Local error, job is invalid, do not retry. smtp.disconnect(); warn!(context, "SMTP job is invalid: {}", err); - Status::Finished(Err(Error::SmtpError(err))) + Status::Finished(Err(err.into())) } Err(crate::smtp::send::Error::NoTransport) => { // Should never happen. diff --git a/src/job_thread.rs b/src/job_thread.rs index 7cf51353c..12512a3d6 100644 --- a/src/job_thread.rs +++ b/src/job_thread.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Condvar, Mutex}; use crate::context::Context; -use crate::error::{Error, Result}; +use crate::error::{format_err, Result}; use crate::imap::Imap; #[derive(Debug)] @@ -99,25 +99,21 @@ impl JobThread { async fn connect_and_fetch(&mut self, context: &Context) -> Result<()> { let prefix = format!("{}-fetch", self.name); - match self.imap.connect_configured(context) { - Ok(()) => { - if let Some(watch_folder) = self.get_watch_folder(context) { - let start = std::time::Instant::now(); - info!(context, "{} started...", prefix); - let res = self - .imap - .fetch(context, &watch_folder) - .await - .map_err(Into::into); - let elapsed = start.elapsed().as_millis(); - info!(context, "{} done in {:.3} ms.", prefix, elapsed); + self.imap.connect_configured(context)?; + if let Some(watch_folder) = self.get_watch_folder(context) { + let start = std::time::Instant::now(); + info!(context, "{} started...", prefix); + let res = self + .imap + .fetch(context, &watch_folder) + .await + .map_err(Into::into); + let elapsed = start.elapsed().as_millis(); + info!(context, "{} done in {:.3} ms.", prefix, elapsed); - res - } else { - Err(Error::WatchFolderNotFound("not-set".to_string())) - } - } - Err(err) => Err(crate::error::Error::Message(err.to_string())), + res + } else { + Err(format_err!("WatchFolder not found: not-set")) } } diff --git a/src/key.rs b/src/key.rs index 243f09e2c..fb658e92c 100644 --- a/src/key.rs +++ b/src/key.rs @@ -18,24 +18,12 @@ pub use crate::pgp::KeyPair; pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; /// Error type for deltachat key handling. -#[derive(Fail, Debug)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "Could not decode base64")] - Base64Decode(#[cause] base64::DecodeError, failure::Backtrace), - #[fail(display = "rPGP error: {}", _0)] - PgpError(#[cause] pgp::errors::Error, failure::Backtrace), -} - -impl From for Error { - fn from(err: base64::DecodeError) -> Error { - Error::Base64Decode(err, failure::Backtrace::new()) - } -} - -impl From for Error { - fn from(err: pgp::errors::Error) -> Error { - Error::PgpError(err, failure::Backtrace::new()) - } + #[error("Could not decode base64")] + Base64Decode(#[from] base64::DecodeError), + #[error("rPGP error: {0}")] + PgpError(#[from] pgp::errors::Error), } pub type Result = std::result::Result; @@ -316,21 +304,19 @@ pub enum KeyPairUse { } /// Error saving a keypair to the database. -#[derive(Fail, Debug)] -#[fail(display = "SaveKeyError: {}", message)] +#[derive(Debug, thiserror::Error)] +#[error("SaveKeyError: {message}")] pub struct SaveKeyError { message: String, - #[cause] - cause: failure::Error, - backtrace: failure::Backtrace, + #[source] + cause: anyhow::Error, } impl SaveKeyError { - fn new(message: impl Into, cause: impl Into) -> Self { + fn new(message: impl Into, cause: impl Into) -> Self { Self { message: message.into(), cause: cause.into(), - backtrace: failure::Backtrace::new(), } } } diff --git a/src/lib.rs b/src/lib.rs index 3bd459ad9..511a6a4c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,8 +2,6 @@ #![deny(clippy::correctness, missing_debug_implementations, clippy::all)] #![allow(clippy::match_bool)] -#[macro_use] -extern crate failure_derive; #[macro_use] extern crate num_derive; #[macro_use] diff --git a/src/location.rs b/src/location.rs index 4f8072278..5d86b3028 100644 --- a/src/location.rs +++ b/src/location.rs @@ -9,7 +9,7 @@ use crate::config::Config; use crate::constants::*; use crate::context::*; use crate::dc_tools::*; -use crate::error::Error; +use crate::error::{ensure, Error}; use crate::events::Event; use crate::job::{self, job_action_exists, job_add, Job}; use crate::message::{Message, MsgId}; diff --git a/src/message.rs b/src/message.rs index 1dbcf02d4..3d54aff8a 100644 --- a/src/message.rs +++ b/src/message.rs @@ -3,7 +3,6 @@ use std::path::{Path, PathBuf}; use deltachat_derive::{FromSql, ToSql}; -use failure::Fail; use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; @@ -12,7 +11,7 @@ use crate::constants::*; use crate::contact::*; use crate::context::*; use crate::dc_tools::*; -use crate::error::Error; +use crate::error::{ensure, Error}; use crate::events::Event; use crate::job::*; use crate::lot::{Lot, LotState, Meaning}; @@ -170,7 +169,7 @@ impl rusqlite::types::ToSql for MsgId { fn to_sql(&self) -> rusqlite::Result { if self.0 <= DC_MSG_ID_LAST_SPECIAL { return Err(rusqlite::Error::ToSqlConversionFailure(Box::new( - InvalidMsgId.compat(), + InvalidMsgId, ))); } let val = rusqlite::types::Value::Integer(self.0 as i64); @@ -198,8 +197,8 @@ impl rusqlite::types::FromSql for MsgId { /// This usually occurs when trying to use a message ID of /// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not /// possible. -#[derive(Debug, Fail)] -#[fail(display = "Invalid Message ID.")] +#[derive(Debug, thiserror::Error)] +#[error("Invalid Message ID.")] pub struct InvalidMsgId; #[derive( diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 4e957c58c..8a370cf57 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -9,7 +9,7 @@ use crate::contact::*; use crate::context::{get_version_str, Context}; use crate::dc_tools::*; use crate::e2ee::*; -use crate::error::Error; +use crate::error::{bail, ensure, format_err, Error}; use crate::location; use crate::message::{self, Message}; use crate::mimeparser::SystemMessage; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index b1c14bd22..cfd241d3b 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -1,11 +1,11 @@ use std::collections::{HashMap, HashSet}; +use anyhow::Context as _; use deltachat_derive::{FromSql, ToSql}; use lettre_email::mime::{self, Mime}; use mailparse::{DispositionType, MailAddr, MailHeaderMap}; use crate::aheader::Aheader; -use crate::bail; use crate::blob::BlobObject; use crate::constants::Viewtype; use crate::contact::*; @@ -13,7 +13,7 @@ use crate::context::Context; use crate::dc_tools::*; use crate::dehtml::dehtml; use crate::e2ee; -use crate::error::Result; +use crate::error::{bail, Result}; use crate::events::Event; use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::location; @@ -884,8 +884,7 @@ pub(crate) struct Report { } pub(crate) fn parse_message_id(value: &str) -> crate::error::Result { - let ids = mailparse::msgidparse(value) - .map_err(|err| format_err!("failed to parse message id {:?}", err))?; + let ids = mailparse::msgidparse(value).context("failed to parse message id")?; if let Some(id) = ids.first() { Ok(id.to_string()) diff --git a/src/param.rs b/src/param.rs index 03f53bd10..8fe9ce97c 100644 --- a/src/param.rs +++ b/src/param.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use crate::blob::{BlobError, BlobObject}; use crate::context::Context; -use crate::error; +use crate::error::{self, bail, ensure}; use crate::message::MsgId; use crate::mimeparser::SystemMessage; diff --git a/src/pgp.rs b/src/pgp.rs index 1806b6fcb..61fad558f 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -18,7 +18,7 @@ use rand::{thread_rng, CryptoRng, Rng}; use crate::constants::KeyGenType; use crate::dc_tools::EmailAddress; -use crate::error::Result; +use crate::error::{bail, ensure, format_err, Result}; use crate::key::*; use crate::keyring::*; @@ -117,21 +117,19 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap, cause: impl Into) -> Self { + fn new(message: impl Into, cause: impl Into) -> Self { Self { message: message.into(), cause: cause.into(), - backtrace: failure::Backtrace::new(), } } } @@ -189,7 +187,7 @@ pub(crate) fn create_keypair( .unwrap(), ) .build() - .map_err(|err| PgpKeygenError::new("invalid key params", failure::err_msg(err)))?; + .map_err(|err| PgpKeygenError::new("invalid key params", format_err!(err)))?; let key = key_params .generate() .map_err(|err| PgpKeygenError::new("invalid params", err))?; diff --git a/src/qr.rs b/src/qr.rs index 15e4cbc69..890c1df07 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -2,20 +2,20 @@ use lazy_static::lazy_static; use percent_encoding::percent_decode_str; +use reqwest::Url; +use serde::Deserialize; use crate::chat; use crate::config::*; use crate::constants::Blocked; use crate::contact::*; use crate::context::Context; -use crate::error::Error; +use crate::error::{bail, ensure, format_err, Error}; use crate::key::dc_format_fingerprint; use crate::key::dc_normalize_fingerprint; use crate::lot::{Lot, LotState}; use crate::param::*; use crate::peerstate::*; -use reqwest::Url; -use serde::Deserialize; const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase const DCACCOUNT_SCHEME: &str = "DCACCOUNT:"; @@ -221,27 +221,20 @@ pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> { let response = reqwest::blocking::Client::new().post(url_str).send(); if response.is_err() { - return Err(format_err!( - "Cannot create account, request to {} failed", - url_str - )); + bail!("Cannot create account, request to {} failed", url_str); } let response = response.unwrap(); if !response.status().is_success() { - return Err(format_err!( - "Request to {} unsuccessful: {:?}", - url_str, - response - )); + bail!("Request to {} unsuccessful: {:?}", url_str, response); } let parsed: reqwest::Result = response.json(); if parsed.is_err() { - return Err(format_err!( + bail!( "Failed to parse JSON response from {}: error: {:?}", url_str, parsed - )); + ); } println!("response: {:?}", &parsed); let parsed = parsed.unwrap(); @@ -289,7 +282,6 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot { Ok(addr) => addr, Err(err) => return err.into(), }; - let name = "".to_string(); Lot::from_address(context, name, addr) } diff --git a/src/securejoin.rs b/src/securejoin.rs index d6fce24a7..e50953e72 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -9,7 +9,7 @@ use crate::constants::*; use crate::contact::*; use crate::context::Context; use crate::e2ee::*; -use crate::error::Error; +use crate::error::{bail, Error}; use crate::events::Event; use crate::headerdef::HeaderDef; use crate::key::{dc_normalize_fingerprint, Key}; @@ -343,24 +343,21 @@ fn fingerprint_equals_sender( } false } -#[derive(Fail, Debug)] +#[derive(Debug, thiserror::Error)] pub(crate) enum HandshakeError { - #[fail(display = "Can not be called with special contact ID")] + #[error("Can not be called with special contact ID")] SpecialContactId, - #[fail(display = "Not a Secure-Join message")] + #[error("Not a Secure-Join message")] NotSecureJoinMsg, - #[fail( - display = "Failed to look up or create chat for contact #{}", - contact_id - )] + #[error("Failed to look up or create chat for contact #{contact_id}")] NoChat { contact_id: u32, - #[cause] + #[source] cause: Error, }, - #[fail(display = "Chat for group {} not found", group)] + #[error("Chat for group {group} not found")] ChatNotFound { group: String }, - #[fail(display = "No configured self address found")] + #[error("No configured self address found")] NoSelfAddr, } diff --git a/src/smtp/mod.rs b/src/smtp/mod.rs index 39b999171..da3673d58 100644 --- a/src/smtp/mod.rs +++ b/src/smtp/mod.rs @@ -16,35 +16,29 @@ use crate::oauth2::*; /// SMTP write and read timeout in seconds. const SMTP_TIMEOUT: u64 = 30; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "Bad parameters")] + #[error("Bad parameters")] BadParameters, - #[fail(display = "Invalid login address {}: {}", address, error)] + #[error("Invalid login address {address}: {error}")] InvalidLoginAddress { address: String, - #[cause] + #[source] error: error::Error, }, - #[fail(display = "SMTP failed to connect: {:?}", _0)] - ConnectionFailure(#[cause] smtp::error::Error), + #[error("SMTP: failed to connect: {0:?}")] + ConnectionFailure(#[source] smtp::error::Error), - #[fail(display = "SMTP: failed to setup connection {:?}", _0)] - ConnectionSetupFailure(#[cause] smtp::error::Error), + #[error("SMTP: failed to setup connection {0:?}")] + ConnectionSetupFailure(#[source] smtp::error::Error), - #[fail(display = "SMTP: oauth2 error {:?}", _0)] + #[error("SMTP: oauth2 error {address}")] Oauth2Error { address: String }, - #[fail(display = "TLS error")] - Tls(#[cause] async_native_tls::Error), -} - -impl From for Error { - fn from(err: async_native_tls::Error) -> Error { - Error::Tls(err) - } + #[error("TLS error")] + Tls(#[from] async_native_tls::Error), } pub type Result = std::result::Result; diff --git a/src/smtp/send.rs b/src/smtp/send.rs index 306ba917d..cade602f2 100644 --- a/src/smtp/send.rs +++ b/src/smtp/send.rs @@ -8,15 +8,15 @@ use crate::events::Event; pub type Result = std::result::Result; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "Envelope error: {}", _0)] - EnvelopeError(#[cause] async_smtp::error::Error), + #[error("Envelope error: {}", _0)] + EnvelopeError(#[from] async_smtp::error::Error), - #[fail(display = "Send error: {}", _0)] - SendError(#[cause] async_smtp::smtp::error::Error), + #[error("Send error: {}", _0)] + SendError(#[from] async_smtp::smtp::error::Error), - #[fail(display = "SMTP has no transport")] + #[error("SMTP has no transport")] NoTransport, } diff --git a/src/sql.rs b/src/sql.rs index 51ee65262..ac1867da5 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -14,50 +14,28 @@ use crate::dc_tools::*; use crate::param::*; use crate::peerstate::*; -#[derive(Debug, Fail)] +#[derive(Debug, thiserror::Error)] pub enum Error { - #[fail(display = "Sqlite Error: {:?}", _0)] - Sql(#[cause] rusqlite::Error), - #[fail(display = "Sqlite Connection Pool Error: {:?}", _0)] - ConnectionPool(#[cause] r2d2::Error), - #[fail(display = "Sqlite: Connection closed")] + #[error("Sqlite Error: {0:?}")] + Sql(#[from] rusqlite::Error), + #[error("Sqlite Connection Pool Error: {0:?}")] + ConnectionPool(#[from] r2d2::Error), + #[error("Sqlite: Connection closed")] SqlNoConnection, - #[fail(display = "Sqlite: Already open")] + #[error("Sqlite: Already open")] SqlAlreadyOpen, - #[fail(display = "Sqlite: Failed to open")] + #[error("Sqlite: Failed to open")] SqlFailedToOpen, - #[fail(display = "{:?}", _0)] - Io(#[cause] std::io::Error), - #[fail(display = "{:?}", _0)] - BlobError(#[cause] crate::blob::BlobError), + #[error("{0}")] + Io(#[from] std::io::Error), + #[error("{0:?}")] + BlobError(#[from] crate::blob::BlobError), + #[error("{0}")] + Other(#[from] crate::error::Error), } pub type Result = std::result::Result; -impl From for Error { - fn from(err: rusqlite::Error) -> Error { - Error::Sql(err) - } -} - -impl From for Error { - fn from(err: r2d2::Error) -> Error { - Error::ConnectionPool(err) - } -} - -impl From for Error { - fn from(err: std::io::Error) -> Error { - Error::Io(err) - } -} - -impl From for Error { - fn from(err: crate::blob::BlobError) -> Error { - Error::BlobError(err) - } -} - /// A wrapper around the underlying Sqlite3 object. #[derive(DebugStub)] pub struct Sql { @@ -96,7 +74,7 @@ impl Sql { pub fn open(&self, context: &Context, dbfile: &std::path::Path, readonly: bool) -> bool { match open(context, self, dbfile, readonly) { Ok(_) => true, - Err(crate::error::Error::SqlError(Error::SqlAlreadyOpen)) => false, + Err(Error::SqlAlreadyOpen) => false, Err(_) => { self.close(context); false @@ -388,14 +366,14 @@ fn open( sql: &Sql, dbfile: impl AsRef, readonly: bool, -) -> crate::error::Result<()> { +) -> Result<()> { if sql.is_open() { error!( context, "Cannot open, database \"{:?}\" already opened.", dbfile.as_ref(), ); - return Err(Error::SqlAlreadyOpen.into()); + return Err(Error::SqlAlreadyOpen); } let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX; @@ -545,7 +523,7 @@ fn open( dbfile.as_ref(), ); // cannot create the tables - maybe we cannot write? - return Err(Error::SqlFailedToOpen.into()); + return Err(Error::SqlFailedToOpen); } else { sql.set_raw_config_int(context, "dbversion", 0)?; } diff --git a/src/stock.rs b/src/stock.rs index 6d4775a85..1653b4855 100644 --- a/src/stock.rs +++ b/src/stock.rs @@ -10,7 +10,7 @@ use crate::chat; use crate::constants::{Viewtype, DC_CONTACT_ID_SELF}; use crate::contact::*; use crate::context::Context; -use crate::error::Error; +use crate::error::{bail, Error}; use crate::message::Message; use crate::param::Param; use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage}; From 711f3f69dad03aad87577d144575b80f85db90a5 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sat, 4 Apr 2020 12:13:50 +0200 Subject: [PATCH 002/156] Emit an event when SMTP fails (the same is already done for IMAP) --- src/smtp/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/smtp/mod.rs b/src/smtp/mod.rs index da3673d58..83f9106e3 100644 --- a/src/smtp/mod.rs +++ b/src/smtp/mod.rs @@ -169,7 +169,11 @@ impl Smtp { .timeout(Some(Duration::from_secs(SMTP_TIMEOUT))); let mut trans = client.into_transport(); - trans.connect().await.map_err(Error::ConnectionFailure)?; + + trans.connect().await.map_err(|err| { + emit_event!(context, Event::ErrorNetwork(err.to_string())); + Error::ConnectionFailure(err) + })?; self.transport = Some(trans); self.last_success = Some(Instant::now()); From f51fd1267f6ea0c83c0ba0b7cf8c7aa833966c0a Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 11 Apr 2020 21:49:53 +0300 Subject: [PATCH 003/156] chat: move parent_query() and related methods from Chat to ChatId --- src/chat.rs | 80 ++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 3ac2000e2..1229d947c 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -378,6 +378,44 @@ impl ChatId { Ok(self.get_param(context)?.exists(Param::Devicetalk)) } + fn parent_query(fields: &str) -> String { + // Check for server_uid guarantees that we don't + // select a draft or undelivered message. + format!( + "SELECT {} \ + FROM msgs WHERE chat_id=?1 AND server_uid!=0 \ + ORDER BY timestamp DESC, id DESC \ + LIMIT 1;", + fields + ) + } + + fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> { + let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)); + let params = params![self]; + let sql = &context.sql; + + let query = Self::parent_query("rfc724_mid, mime_in_reply_to, mime_references"); + + sql.query_row(&query, params, collect).ok() + } + + fn parent_is_encrypted(self, context: &Context) -> Result { + let sql = &context.sql; + let params = params![self]; + let query = Self::parent_query("param"); + + let packed: Option = sql.query_get_value_result(&query, params)?; + + if let Some(ref packed) = packed { + let param = packed.parse::()?; + Ok(param.exists(Param::GuaranteeE2ee)) + } else { + // No messages + Ok(false) + } + } + /// Bad evil escape hatch. /// /// Avoid using this, eventually types should be cleaned up enough @@ -587,44 +625,6 @@ impl Chat { "Err".to_string() } - fn parent_query(fields: &str) -> String { - // Check for server_uid guarantees that we don't - // select a draft or undelivered message. - format!( - "SELECT {} \ - FROM msgs WHERE chat_id=?1 AND server_uid!=0 \ - ORDER BY timestamp DESC, id DESC \ - LIMIT 1;", - fields - ) - } - - fn get_parent_mime_headers(&self, context: &Context) -> Option<(String, String, String)> { - let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)); - let params = params![self.id]; - let sql = &context.sql; - - let query = Self::parent_query("rfc724_mid, mime_in_reply_to, mime_references"); - - sql.query_row(&query, params, collect).ok() - } - - fn parent_is_encrypted(&self, context: &Context) -> Result { - let sql = &context.sql; - let params = params![self.id]; - let query = Self::parent_query("param"); - - let packed: Option = sql.query_get_value_result(&query, params)?; - - if let Some(ref packed) = packed { - let param = packed.parse::()?; - Ok(param.exists(Param::GuaranteeE2ee)) - } else { - // No messages - Ok(false) - } - } - pub fn get_profile_image(&self, context: &Context) -> Option { if let Some(image_rel) = self.param.get(Param::ProfileImage) { if !image_rel.is_empty() { @@ -831,7 +831,7 @@ impl Chat { } } - if can_encrypt && (all_mutual || self.parent_is_encrypted(context)?) { + if can_encrypt && (all_mutual || self.id.parent_is_encrypted(context)?) { msg.param.set_int(Param::GuaranteeE2ee, 1); } } @@ -846,7 +846,7 @@ impl Chat { // we do not set In-Reply-To/References in this case. if !self.is_self_talk() { if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) = - self.get_parent_mime_headers(context) + self.id.get_parent_mime_headers(context) { if !parent_rfc724_mid.is_empty() { new_in_reply_to = parent_rfc724_mid.clone(); From f65dbee74b40f137d25a47af342ff952d1d5677f Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 11 Apr 2020 22:06:27 +0300 Subject: [PATCH 004/156] Add basic test for ChatId.parent_is_encrypted() --- src/chat.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/chat.rs b/src/chat.rs index 1229d947c..e4cddfdb0 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -3119,4 +3119,16 @@ mod tests { false ); } + + #[test] + fn test_parent_is_encrypted() { + let t = dummy_context(); + let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap(); + assert!(!chat_id.parent_is_encrypted(&t.ctx).unwrap()); + + let mut msg = Message::new(Viewtype::Text); + msg.set_text(Some("hello".to_string())); + chat_id.set_draft(&t.ctx, Some(&mut msg)); + assert!(!chat_id.parent_is_encrypted(&t.ctx).unwrap()); + } } From 493a213d41b66884bdaa3e78741490e8511aa725 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 11 Apr 2020 22:24:25 +0300 Subject: [PATCH 005/156] sql: move QueryReturnedNoRows and NULL handling into query_row_optional() --- src/sql.rs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/sql.rs b/src/sql.rs index ac1867da5..e43360a6b 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -190,6 +190,30 @@ impl Sql { self.with_conn(|conn| conn.query_row(sql.as_ref(), params, f).map_err(Into::into)) } + /// Execute a query which is expected to return zero or one row. + pub fn query_row_optional( + &self, + sql: impl AsRef, + params: P, + f: F, + ) -> Result> + where + P: IntoIterator, + P::Item: rusqlite::ToSql, + F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + { + match self.query_row(sql, params, f) { + Ok(res) => Ok(Some(res)), + Err(Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => Ok(None), + Err(Error::Sql(rusqlite::Error::InvalidColumnType( + _, + _, + rusqlite::types::Type::Null, + ))) => Ok(None), + Err(err) => Err(err), + } + } + pub fn table_exists(&self, name: impl AsRef) -> bool { self.with_conn(|conn| table_exists(conn, name)) .unwrap_or_default() @@ -204,16 +228,7 @@ impl Sql { P::Item: rusqlite::ToSql, T: rusqlite::types::FromSql, { - match self.query_row(query, params, |row| row.get::<_, T>(0)) { - Ok(res) => Ok(Some(res)), - Err(Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => Ok(None), - Err(Error::Sql(rusqlite::Error::InvalidColumnType( - _, - _, - rusqlite::types::Type::Null, - ))) => Ok(None), - Err(err) => Err(err), - } + self.query_row_optional(query, params, |row| row.get::<_, T>(0)) } /// Not resultified version of `query_get_value_result`. Returns From e9967c32e69470f3c060ba77b6117ded91810d91 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 11 Apr 2020 22:07:43 +0300 Subject: [PATCH 006/156] chat: filter parent messages by state instead of UID Since introduction of server message deletion, message may not have server UID even when it is not a draft or pending message. To handle such messages correctly, we explicitly check message state. --- src/chat.rs | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index e4cddfdb0..1c24635a3 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -378,34 +378,45 @@ impl ChatId { Ok(self.get_param(context)?.exists(Param::Devicetalk)) } - fn parent_query(fields: &str) -> String { - // Check for server_uid guarantees that we don't - // select a draft or undelivered message. - format!( + fn parent_query(self, context: &Context, fields: &str, f: F) -> sql::Result> + where + F: FnOnce(&rusqlite::Row) -> rusqlite::Result, + { + let sql = &context.sql; + let query = format!( "SELECT {} \ - FROM msgs WHERE chat_id=?1 AND server_uid!=0 \ + FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) \ ORDER BY timestamp DESC, id DESC \ LIMIT 1;", fields + ); + sql.query_row_optional( + query, + params![ + self, + MessageState::OutPreparing, + MessageState::OutDraft, + MessageState::OutPending, + MessageState::OutFailed + ], + f, ) } fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> { let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)); - let params = params![self]; - let sql = &context.sql; - - let query = Self::parent_query("rfc724_mid, mime_in_reply_to, mime_references"); - - sql.query_row(&query, params, collect).ok() + self.parent_query( + context, + "rfc724_mid, mime_in_reply_to, mime_references", + collect, + ) + .ok() + .flatten() } fn parent_is_encrypted(self, context: &Context) -> Result { - let sql = &context.sql; - let params = params![self]; - let query = Self::parent_query("param"); - - let packed: Option = sql.query_get_value_result(&query, params)?; + let collect = |row: &rusqlite::Row| Ok(row.get(0)?); + let packed: Option = self.parent_query(context, "param", collect)?; if let Some(ref packed) = packed { let param = packed.parse::()?; From ec67b3975c215025320c53d6556ea9ac59794258 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 17:43:00 +0100 Subject: [PATCH 007/156] good bye global plugin manager ... we only do per-account object plugin_management for now --- python/src/deltachat/account.py | 41 +++++++++++++---------------- python/src/deltachat/eventlogger.py | 11 ++++---- python/src/deltachat/hookspec.py | 32 +++++++++++----------- 3 files changed, 40 insertions(+), 44 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index d699aff6d..bed2a6d50 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -3,6 +3,7 @@ from __future__ import print_function import atexit import threading +from contextlib import contextmanager import os import time from array import array @@ -16,7 +17,7 @@ from .chat import Chat from .message import Message from .contact import Contact from .eventlogger import EventLogger -from .hookspec import get_plugin_manager, hookimpl +from .hookspec import AccountHookSpecs, account_hookimpl class Account(object): @@ -41,15 +42,15 @@ class Account(object): self._evlogger = EventLogger(self, logid, debug) self._threads = IOThreads(self._dc_context, self._evlogger._log_event) - # register event call back and initialize plugin system + # register event call back and initialize per-account plugin system def _ll_event(ctx, evt_name, data1, data2): assert ctx == self._dc_context - self.pluggy.hook.process_low_level_event( + self.plugin_manager.hook.process_low_level_event( account=self, event_name=evt_name, data1=data1, data2=data2 ) - self.pluggy = get_plugin_manager() - self.pluggy.register(self._evlogger) + self.plugin_manager = AccountHookSpecs._make_plugin_manager() + self.plugin_manager.register(self._evlogger) deltachat.set_context_callback(self._dc_context, _ll_event) # open database @@ -382,7 +383,7 @@ class Account(object): return export_files[0] def _export(self, path, imex_cmd): - with ImexTracker(self) as imex_tracker: + with temp_plugin(self.plugin_manager, ImexTracker()) as imex_tracker: lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL) if not self._threads.is_started(): lib.dc_perform_imap_jobs(self._dc_context) @@ -404,7 +405,7 @@ class Account(object): self._import(path, imex_cmd=12) def _import(self, path, imex_cmd): - with ImexTracker(self) as imex_tracker: + with temp_plugin(self.plugin_manager, ImexTracker()) as imex_tracker: lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL) if not self._threads.is_started(): lib.dc_perform_imap_jobs(self._dc_context) @@ -511,7 +512,7 @@ class Account(object): deltachat.clear_context_callback(self._dc_context) del self._dc_context atexit.unregister(self.shutdown) - self.pluggy.unregister(self._evlogger) + self.plugin_manager.unregister(self._evlogger) def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0): """set a new location. It effects all chats where we currently @@ -528,23 +529,19 @@ class Account(object): raise ValueError("no chat is streaming locations") +@contextmanager +def temp_plugin(plugin_manager, plugin): + plugin_manager.register(plugin) + yield plugin + plugin_manager.unregister(plugin) + + class ImexTracker: - def __init__(self, account): + def __init__(self): self._imex_events = Queue() - self.account = account - def __enter__(self): - self.account.pluggy.register(self) - return self - - def __exit__(self, *args): - self.account.pluggy.unregister(self) - - @hookimpl - def process_low_level_event(self, account, event_name, data1, data2): - # there could be multiple accounts instantiated - if self.account is not account: - return + @account_hookimpl + def process_low_level_event(self, event_name, data1, data2): if event_name == "DC_EVENT_IMEX_PROGRESS": self._imex_events.put(data1) elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN": diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index e1dfffda6..12a2e2cf1 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -2,7 +2,7 @@ import threading import re import time from queue import Queue, Empty -from .hookspec import hookimpl +from .hookspec import account_hookimpl class EventLogger: @@ -18,11 +18,10 @@ class EventLogger: self._timeout = None self.init_time = time.time() - @hookimpl - def process_low_level_event(self, account, event_name, data1, data2): - if self.account == account: - self._log_event(event_name, data1, data2) - self._event_queue.put((event_name, data1, data2)) + @account_hookimpl + def process_low_level_event(self, event_name, data1, data2): + self._log_event(event_name, data1, data2) + self._event_queue.put((event_name, data1, data2)) def set_timeout(self, timeout): self._timeout = timeout diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index b79b5ffef..6a65d40c9 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -1,25 +1,25 @@ -""" Hooks for python bindings """ +""" Hooks for Python bindings to Delta Chat Core Rust CFFI""" import pluggy -name = "deltachat" +__all__ = ["account_hookspec", "account_hookimpl", "AccountHookSpecs"] -hookspec = pluggy.HookspecMarker(name) -hookimpl = pluggy.HookimplMarker(name) -_plugin_manager = None +_account_name = "deltachat-account" +account_hookspec = pluggy.HookspecMarker(_account_name) +account_hookimpl = pluggy.HookimplMarker(_account_name) -def get_plugin_manager(): - global _plugin_manager - if _plugin_manager is None: - _plugin_manager = pluggy.PluginManager(name) - _plugin_manager.add_hookspecs(DeltaChatHookSpecs) - return _plugin_manager +class AccountHookSpecs: + """ per-Account-instance hook specifications. + Account hook implementations need to be registered with an Account instance. + """ + @classmethod + def _make_plugin_manager(cls): + pm = pluggy.PluginManager(_account_name) + pm.add_hookspecs(cls) + return pm -class DeltaChatHookSpecs: - """ Plugin Hook specifications for Python bindings to Delta Chat CFFI. """ - - @hookspec - def process_low_level_event(self, account, event_name, data1, data2): + @account_hookspec + def process_low_level_event(self, event_name, data1, data2): """ process a CFFI low level events for a given account. """ From bbc8bed39c6af075ef7ab8219ef470e309e72f77 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 17:59:25 +0100 Subject: [PATCH 008/156] move temp_plugin to account --- python/src/deltachat/account.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index bed2a6d50..cec7a438c 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -383,7 +383,7 @@ class Account(object): return export_files[0] def _export(self, path, imex_cmd): - with temp_plugin(self.plugin_manager, ImexTracker()) as imex_tracker: + with self.temp_plugin(ImexTracker()) as imex_tracker: lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL) if not self._threads.is_started(): lib.dc_perform_imap_jobs(self._dc_context) @@ -405,7 +405,7 @@ class Account(object): self._import(path, imex_cmd=12) def _import(self, path, imex_cmd): - with temp_plugin(self.plugin_manager, ImexTracker()) as imex_tracker: + with self.temp_plugin(ImexTracker()) as imex_tracker: lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL) if not self._threads.is_started(): lib.dc_perform_imap_jobs(self._dc_context) @@ -528,12 +528,12 @@ class Account(object): if dc_res == 0: raise ValueError("no chat is streaming locations") - -@contextmanager -def temp_plugin(plugin_manager, plugin): - plugin_manager.register(plugin) - yield plugin - plugin_manager.unregister(plugin) + @contextmanager + def temp_plugin(self, plugin): + """ run a code block with the given plugin temporarily registered. """ + self.plugin_manager.register(plugin) + yield plugin + self.plugin_manager.unregister(plugin) class ImexTracker: From 95d45b386f21ea6a9209693b957fc286e7b4d40b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 18:28:42 +0100 Subject: [PATCH 009/156] separate out FFI eventracking to only be used in running tests --- python/src/deltachat/account.py | 14 +-- python/src/deltachat/eventlogger.py | 45 ------- python/tests/conftest.py | 23 ++-- python/tests/ffi_event.py | 59 +++++++++ python/tests/test_account.py | 182 ++++++++++++++-------------- python/tests/test_increation.py | 8 +- 6 files changed, 172 insertions(+), 159 deletions(-) create mode 100644 python/tests/ffi_event.py diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index cec7a438c..c46dfe066 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -42,15 +42,16 @@ class Account(object): self._evlogger = EventLogger(self, logid, debug) self._threads = IOThreads(self._dc_context, self._evlogger._log_event) - # register event call back and initialize per-account plugin system + # initialize per-account plugin system + self.plugin_manager = AccountHookSpecs._make_plugin_manager() + self.plugin_manager.register(self._evlogger) + + # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): assert ctx == self._dc_context self.plugin_manager.hook.process_low_level_event( account=self, event_name=evt_name, data1=data1, data2=data2 ) - - self.plugin_manager = AccountHookSpecs._make_plugin_manager() - self.plugin_manager.register(self._evlogger) deltachat.set_context_callback(self._dc_context, _ll_event) # open database @@ -481,11 +482,6 @@ class Account(object): # meta API for start/stop and event based processing # - def wait_next_incoming_message(self): - """ wait for and return next incoming message. """ - ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG") - return self.get_message_by_id(ev[2]) - def start_threads(self, mvbox=False, sentbox=False): """ start IMAP/SMTP threads (and configure account if it hasn't happened). diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 12a2e2cf1..2d0d3c761 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,7 +1,5 @@ import threading -import re import time -from queue import Queue, Empty from .hookspec import account_hookimpl @@ -10,58 +8,15 @@ class EventLogger: def __init__(self, account, logid=None, debug=True): self.account = account - self._event_queue = Queue() self._debug = debug if logid is None: logid = str(self.account._dc_context).strip(">").split()[-1] self.logid = logid - self._timeout = None self.init_time = time.time() @account_hookimpl def process_low_level_event(self, event_name, data1, data2): self._log_event(event_name, data1, data2) - self._event_queue.put((event_name, data1, data2)) - - def set_timeout(self, timeout): - self._timeout = timeout - - def consume_events(self, check_error=True): - while not self._event_queue.empty(): - self.get(check_error=check_error) - - def get(self, timeout=None, check_error=True): - timeout = timeout or self._timeout - ev = self._event_queue.get(timeout=timeout) - if check_error and ev[0] == "DC_EVENT_ERROR": - raise ValueError("{}({!r},{!r})".format(*ev)) - return ev - - def ensure_event_not_queued(self, event_name_regex): - __tracebackhide__ = True - rex = re.compile("(?:{}).*".format(event_name_regex)) - while 1: - try: - ev = self._event_queue.get(False) - except Empty: - break - else: - assert not rex.match(ev[0]), "event found {}".format(ev) - - def get_matching(self, event_name_regex, check_error=True, timeout=None): - self._log("-- waiting for event with regex: {} --".format(event_name_regex)) - rex = re.compile("(?:{}).*".format(event_name_regex)) - while 1: - ev = self.get(timeout=timeout, check_error=check_error) - if rex.match(ev[0]): - return ev - - def get_info_matching(self, regex): - rex = re.compile("(?:{}).*".format(regex)) - while 1: - ev = self.get_matching("DC_EVENT_INFO") - if rex.match(ev[2]): - return ev def _log_event(self, evt_name, data1, data2): # don't show events that are anyway empty impls now diff --git a/python/tests/conftest.py b/python/tests/conftest.py index b6868bc2c..5e0141354 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -9,6 +9,7 @@ from deltachat import Account from deltachat import const from deltachat.capi import lib from _pytest.monkeypatch import MonkeyPatch +from ffi_event import FFIEventTracker import tempfile @@ -163,6 +164,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): def make_account(self, path, logid): ac = Account(path, logid=logid) + ac._evtracker = FFIEventTracker(ac) + ac.plugin_manager.register(ac._evtracker) self._finalizers.append(ac.shutdown) return ac @@ -170,8 +173,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): self.offline_count += 1 tmpdb = tmpdir.join("offlinedb%d" % self.offline_count) ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count)) - ac._evlogger.init_time = self.init_time - ac._evlogger.set_timeout(2) + ac._evtracker.init_time = self.init_time + ac._evtracker.set_timeout(2) return ac def _preconfigure_key(self, account, addr): @@ -213,8 +216,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) if pre_generated_key: self._preconfigure_key(ac, configdict['addr']) - ac._evlogger.init_time = self.init_time - ac._evlogger.set_timeout(30) + ac._evtracker.init_time = self.init_time + ac._evtracker.set_timeout(30) return ac, dict(configdict) def get_online_configuring_account(self, mvbox=False, sentbox=False, @@ -248,8 +251,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) if pre_generated_key: self._preconfigure_key(ac, account.get_config("addr")) - ac._evlogger.init_time = self.init_time - ac._evlogger.set_timeout(30) + ac._evtracker.init_time = self.init_time + ac._evtracker.set_timeout(30) ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw")) ac.start_threads() return ac @@ -280,7 +283,7 @@ def wait_configuration_progress(account, min_target, max_target=1001): min_target = min(min_target, max_target) while 1: evt_name, data1, data2 = \ - account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS") + account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS") if data1 >= min_target and data1 <= max_target: print("** CONFIG PROGRESS {}".format(min_target), account) break @@ -289,7 +292,7 @@ def wait_configuration_progress(account, min_target, max_target=1001): def wait_securejoin_inviter_progress(account, target): while 1: evt_name, data1, data2 = \ - account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS") + account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS") if data2 >= target: print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account) break @@ -299,7 +302,7 @@ def wait_successful_IMAP_SMTP_connection(account): imap_ok = smtp_ok = False while not imap_ok or not smtp_ok: evt_name, data1, data2 = \ - account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED") + account._evtracker.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED") if evt_name == "DC_EVENT_IMAP_CONNECTED": imap_ok = True print("** IMAP OK", account) @@ -310,7 +313,7 @@ def wait_successful_IMAP_SMTP_connection(account): def wait_msgs_changed(account, chat_id, msg_id=None): - ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ev[1] == chat_id if msg_id is not None: assert ev[2] == msg_id diff --git a/python/tests/ffi_event.py b/python/tests/ffi_event.py new file mode 100644 index 000000000..4ffd11b1e --- /dev/null +++ b/python/tests/ffi_event.py @@ -0,0 +1,59 @@ +import re +from queue import Queue, Empty +from deltachat.hookspec import account_hookimpl + + +class FFIEventTracker: + def __init__(self, account, timeout=None): + self.account = account + self._timeout = timeout + self._event_queue = Queue() + + @account_hookimpl + def process_low_level_event(self, event_name, data1, data2): + self._event_queue.put((event_name, data1, data2)) + + def set_timeout(self, timeout): + self._timeout = timeout + + def consume_events(self, check_error=True): + while not self._event_queue.empty(): + self.get(check_error=check_error) + + def get(self, timeout=None, check_error=True): + timeout = timeout if timeout is not None else self._timeout + ev = self._event_queue.get(timeout=timeout) + if check_error and ev[0] == "DC_EVENT_ERROR": + raise ValueError("{}({!r},{!r})".format(*ev)) + return ev + + def ensure_event_not_queued(self, event_name_regex): + __tracebackhide__ = True + rex = re.compile("(?:{}).*".format(event_name_regex)) + while 1: + try: + ev = self._event_queue.get(False) + except Empty: + break + else: + assert not rex.match(ev[0]), "event found {}".format(ev) + + def get_matching(self, event_name_regex, check_error=True, timeout=None): + self.account._evlogger._log("-- waiting for event with regex: {} --".format(event_name_regex)) + rex = re.compile("(?:{}).*".format(event_name_regex)) + while 1: + ev = self.get(timeout=timeout, check_error=check_error) + if rex.match(ev[0]): + return ev + + def get_info_matching(self, regex): + rex = re.compile("(?:{}).*".format(regex)) + while 1: + ev = self.get_matching("DC_EVENT_INFO") + if rex.match(ev[2]): + return ev + + def wait_next_incoming_message(self): + """ wait for and return next incoming message. """ + ev = self.get_matching("DC_EVENT_INCOMING_MSG") + return self.account.get_message_by_id(ev[2]) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 31835b152..d88d2e437 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -193,13 +193,13 @@ class TestOfflineChat: def test_group_chat_creation_with_translation(self, ac1): ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s") - ac1._evlogger.consume_events() + ac1._evtracker.consume_events() with pytest.raises(ValueError): ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %2$s") - ac1._evlogger.get_matching("DC_EVENT_WARNING") + ac1._evtracker.get_matching("DC_EVENT_WARNING") with pytest.raises(ValueError): ac1.set_stock_translation(500, "xyz %1$s") - ac1._evlogger.get_matching("DC_EVENT_WARNING") + ac1._evtracker.get_matching("DC_EVENT_WARNING") contact1 = ac1.create_contact("some1@hello.com", name="some1") contact2 = ac1.create_contact("some2@hello.com", name="some2") chat = ac1.create_group_chat(name="title1") @@ -242,7 +242,7 @@ class TestOfflineChat: def test_delete_and_send_fails(self, ac1, chat1): chat1.delete() - ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") with pytest.raises(ValueError): chat1.send_text("msg1") @@ -305,9 +305,9 @@ class TestOfflineChat: chat1.send_image(path="notexists") fn = data.get_path("d.png") lp.sec("sending image") - chat1.account._evlogger.consume_events() + chat1.account._evtracker.consume_events() msg = chat1.send_image(fn) - chat1.account._evlogger.get_matching("DC_EVENT_NEW_BLOB_FILE") + chat1.account._evtracker.get_matching("DC_EVENT_NEW_BLOB_FILE") assert msg.is_image() assert msg assert msg.id > 0 @@ -463,7 +463,7 @@ class TestOnlineAccount: lp.sec("ac1: send unencrypted message to ac2") chat.send_text("message1") lp.sec("ac2: waiting for message from ac1") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg_in = ac2.get_message_by_id(ev[2]) assert msg_in.text == "message1" assert not msg_in.is_encrypted() @@ -471,7 +471,7 @@ class TestOnlineAccount: lp.sec("ac2: send encrypted message to ac1") msg_in.chat.send_text("message2") lp.sec("ac1: waiting for message from ac2") - ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg2_in = ac1.get_message_by_id(ev[2]) assert msg2_in.text == "message2" assert msg2_in.is_encrypted() @@ -479,7 +479,7 @@ class TestOnlineAccount: lp.sec("ac1: send encrypted message to ac2") msg2_in.chat.send_text("message3") lp.sec("ac2: waiting for message from ac1") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg3_in = ac1.get_message_by_id(ev[2]) assert msg3_in.text == "message3" assert msg3_in.is_encrypted() @@ -497,7 +497,7 @@ class TestOnlineAccount: assert len(export_files) == 2 for x in export_files: assert x.startswith(dir.strpath) - ac1._evlogger.consume_events() + ac1._evtracker.consume_events() ac1.import_self_keys(dir.strpath) def test_one_account_send_bcc_setting(self, acfactory, lp): @@ -523,13 +523,13 @@ class TestOnlineAccount: assert not msg_out.is_forwarded() # wait for send out (no BCC) - ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + ev = ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") assert ac1.get_config("bcc_self") == "0" # make sure we are not sending message to ourselves assert self_addr not in ev[2] assert other_addr in ev[2] - ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE") + ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE") lp.sec("ac1: setting bcc_self=1") ac1.set_config("bcc_self", "1") @@ -538,16 +538,16 @@ class TestOnlineAccount: msg_out = chat.send_text("message2") # wait for send out (BCC) - ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + ev = ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") assert ac1.get_config("bcc_self") == "1" # now make sure we are sending message to ourselves too assert self_addr in ev[2] assert other_addr in ev[2] - ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE") + ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE") # Second client receives only second message, but not the first - ev = ac1_clone._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ac1_clone.get_message_by_id(ev[2]).text == msg_out.text def test_send_file_twice_unicode_filename_mangling(self, tmpdir, acfactory, lp): @@ -567,7 +567,7 @@ class TestOnlineAccount: chat.send_msg(msg1) lp.sec("ac2: receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL return ac2.get_message_by_id(ev[2]) @@ -599,7 +599,7 @@ class TestOnlineAccount: chat.send_file(p, mime_type="text/html") lp.sec("ac2: receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL msg = ac2.get_message_by_id(ev[2]) @@ -622,7 +622,7 @@ class TestOnlineAccount: lp.sec("ac1: send message and wait for ac2 to receive it") chat = self.get_chat(ac1, ac2) chat.send_text("message1") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL lp.sec("test finished") @@ -633,9 +633,9 @@ class TestOnlineAccount: wait_configuration_progress(ac1, 1000) chat = self.get_chat(ac1, ac2) chat.send_text("message1") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL - ev = ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + ev = ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") def test_move_works_on_self_sent(self, acfactory): ac1 = acfactory.get_online_configuring_account(mvbox=True) @@ -647,9 +647,9 @@ class TestOnlineAccount: chat.send_text("message1") chat.send_text("message2") chat.send_text("message3") - ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") - ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") - ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") def test_forward_messages(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -659,7 +659,7 @@ class TestOnlineAccount: msg_out = chat.send_text("message2") lp.sec("ac2: wait for receive") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") assert ev[2] == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message2" @@ -692,7 +692,7 @@ class TestOnlineAccount: msg_out = chat.send_text("message2") lp.sec("receiving message") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg_in = ac2.get_message_by_id(ev[2]) assert msg_in.text == "message2" assert not msg_in.is_forwarded() @@ -703,7 +703,7 @@ class TestOnlineAccount: ac1.forward_messages([msg_out], group) # wait for other account to receive - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg_in = ac2.get_message_by_id(ev[2]) assert msg_in.text == "message2" assert msg_in.is_forwarded() @@ -713,11 +713,11 @@ class TestOnlineAccount: lp.sec("ac1: create self chat") chat = ac1.create_chat_by_contact(ac1.get_self_contact()) chat.send_text("hello") - ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") ac1.empty_server_folders(inbox=True, mvbox=True) - ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") + ev = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") assert ev[2] == "DeltaChat" - ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") + ev = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") assert ev[2] == "INBOX" def test_send_and_receive_message_markseen(self, acfactory, lp): @@ -731,14 +731,14 @@ class TestOnlineAccount: lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") - ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED") + ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") evt_name, data1, data2 = ev assert data1 == chat.id assert data2 == msg_out.id assert msg_out.is_out_delivered() lp.sec("wait for ac2 to receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ev[2] == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message1" @@ -761,13 +761,13 @@ class TestOnlineAccount: chat2b.mark_noticed() assert chat2b.count_fresh_messages() == 0 - ac2._evlogger.consume_events() + ac2._evtracker.consume_events() lp.sec("sending a second message from ac1 to ac2") msg_out2 = chat.send_text("message2") lp.sec("wait for ac2 to receive second message") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") assert ev[2] == msg_out2.id msg_in2 = ac2.get_message_by_id(msg_out2.id) @@ -775,7 +775,7 @@ class TestOnlineAccount: ac2.mark_seen_messages([msg_in, msg_in2]) lp.step("1") for i in range(2): - ev = ac1._evlogger.get_matching("DC_EVENT_MSG_READ") + ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ") assert ev[1] > const.DC_CHAT_ID_LAST_SPECIAL assert ev[2] > const.DC_MSG_ID_LAST_SPECIAL lp.step("2") @@ -783,10 +783,10 @@ class TestOnlineAccount: assert msg_out2.is_out_mdn_received() lp.sec("check that a second call to mark_seen does not create change or smtp job") - ac2._evlogger.consume_events() + ac2._evtracker.consume_events() ac2.mark_seen_messages([msg_in]) try: - ac2._evlogger.get_matching("DC_EVENT_MSG_READ", timeout=0.01) + ac2._evtracker.get_matching("DC_EVENT_MSG_READ", timeout=0.01) except queue.Empty: pass # mark_seen_messages() has generated events before it returns @@ -809,7 +809,7 @@ class TestOnlineAccount: ac1.set_config("mdns_enabled", "0") lp.sec("wait for ac2 to receive message") - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() assert len(msg.chat.get_messages()) == 1 @@ -818,7 +818,7 @@ class TestOnlineAccount: lp.sec("ac1: waiting for incoming activity") # MDN should be moved even though MDNs are already disabled - ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") assert len(chat.get_messages()) == 1 @@ -836,7 +836,7 @@ class TestOnlineAccount: assert not msg_out.is_encrypted() lp.sec("wait for ac2 to receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ev[2] == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message1" @@ -846,7 +846,7 @@ class TestOnlineAccount: chat2b.send_text("message-back") lp.sec("wait for ac1 to receive message") - ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") assert ev[1] == chat.id assert ev[2] > msg_out.id msg_back = ac1.get_message_by_id(ev[2]) @@ -864,7 +864,7 @@ class TestOnlineAccount: chat.add_contact(ac1.create_contact(ac2.get_config("addr"))) chat.add_contact(ac1.create_contact("notexisting@testrun.org")) msg = chat.send_text("test not encrypt") - ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + ev = ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") assert not msg.is_encrypted() def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp): @@ -885,11 +885,11 @@ class TestOnlineAccount: assert not msg_out.is_encrypted() lp.sec("wait for ac2 to receive multi-line non-unicode message") - msg_in = ac2.wait_next_incoming_message() + msg_in = ac2._evtracker.wait_next_incoming_message() assert msg_in.text == text1 lp.sec("wait for ac2 to receive multi-line unicode message") - msg_in = ac2.wait_next_incoming_message() + msg_in = ac2._evtracker.wait_next_incoming_message() assert msg_in.text == text2 assert ac1.get_config("addr") in msg_in.chat.get_name() @@ -904,7 +904,7 @@ class TestOnlineAccount: assert not msg_out.is_encrypted() lp.sec("wait for ac2 to receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message1" assert not msg_in.is_encrypted() @@ -914,7 +914,7 @@ class TestOnlineAccount: chat2b.send_text("message-back") lp.sec("wait for ac1 to receive message") - ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") assert ev[1] == chat.id msg_back = ac1.get_message_by_id(ev[2]) assert msg_back.text == "message-back" @@ -941,7 +941,7 @@ class TestOnlineAccount: assert chat.get_draft() is None lp.sec("wait for ac2 to receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg_in = ac2.get_message_by_id(ev[2]) assert msg_in.text == "message2 -- should be encrypted" assert msg_in.is_encrypted() @@ -955,11 +955,11 @@ class TestOnlineAccount: lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") - ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED") + ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") assert msg_out.get_mime_headers() is None lp.sec("wait for ac2 to receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") in_id = ev[2] mime = ac2.get_message_by_id(in_id).get_mime_headers() assert mime.get_all("From") @@ -972,14 +972,14 @@ class TestOnlineAccount: lp.sec("sending image message from ac1 to ac2") path = data.get_path("d.png") msg_out = chat.send_image(path) - ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED") + ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") evt_name, data1, data2 = ev assert data1 == chat.id assert data2 == msg_out.id assert msg_out.is_out_delivered() lp.sec("wait for ac2 to receive message") - ev = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ev[2] == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.is_image() @@ -1041,8 +1041,8 @@ class TestOnlineAccount: lp.sec("trigger ac setup message and return setupcode") assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"] setup_code = ac1.initiate_key_transfer() - ac2._evlogger.set_timeout(30) - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ac2._evtracker.set_timeout(30) + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") msg = ac2.get_message_by_id(ev[2]) assert msg.is_setup_message() assert msg.get_setupcodebegin() == setup_code[:2] @@ -1058,18 +1058,18 @@ class TestOnlineAccount: def test_ac_setup_message_twice(self, acfactory, lp): ac1 = acfactory.get_online_configuring_account() ac2 = acfactory.clone_online_account(ac1) - ac2._evlogger.set_timeout(30) + ac2._evtracker.set_timeout(30) wait_configuration_progress(ac2, 1000) wait_configuration_progress(ac1, 1000) lp.sec("trigger ac setup message but ignore") assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"] ac1.initiate_key_transfer() - ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") lp.sec("trigger second ac setup message, wait for receive ") setup_code2 = ac1.initiate_key_transfer() - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") msg = ac2.get_message_by_id(ev[2]) assert msg.is_setup_message() assert msg.get_setupcodebegin() == setup_code2[:2] @@ -1096,10 +1096,10 @@ class TestOnlineAccount: ch = ac2.qr_join_chat(qr) assert ch.id >= 10 # check that at least some of the handshake messages are deleted - ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") - ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") + ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") wait_securejoin_inviter_progress(ac1, 1000) - ac1._evlogger.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED") + ac1._evtracker.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED") def test_qr_verified_group_and_chatting(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -1111,10 +1111,10 @@ class TestOnlineAccount: chat2 = ac2.qr_join_chat(qr) assert chat2.id >= 10 wait_securejoin_inviter_progress(ac1, 1000) - ac1._evlogger.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED") + ac1._evtracker.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED") lp.sec("ac2: read member added message") - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() assert msg.is_encrypted() assert "added" in msg.text.lower() @@ -1123,14 +1123,14 @@ class TestOnlineAccount: assert msg_out.is_encrypted() lp.sec("ac2: read message and check it's verified chat") - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() assert msg.text == "hello" assert msg.chat.is_verified() assert msg.is_encrypted() lp.sec("ac2: send message and let ac1 read it") chat2.send_text("world") - msg = ac1.wait_next_incoming_message() + msg = ac1._evtracker.wait_next_incoming_message() assert msg.text == "world" assert msg.is_encrypted() @@ -1149,7 +1149,7 @@ class TestOnlineAccount: assert not msg.is_encrypted() lp.sec("ac2: wait for receiving message and avatar from ac1") - msg1 = ac2.wait_next_incoming_message() + msg1 = ac2._evtracker.wait_next_incoming_message() assert not msg1.chat.is_deaddrop() received_path = msg1.get_sender_contact().get_profile_image() assert open(received_path, "rb").read() == open(p, "rb").read() @@ -1163,13 +1163,13 @@ class TestOnlineAccount: assert m.is_encrypted() lp.sec("ac1: wait for receiving message and avatar from ac2") - msg2 = ac1.wait_next_incoming_message() + msg2 = ac1._evtracker.wait_next_incoming_message() received_path = msg2.get_sender_contact().get_profile_image() assert received_path is not None, "did not get avatar through encrypted message" assert open(received_path, "rb").read() == open(p, "rb").read() - ac2._evlogger.consume_events() - ac1._evlogger.consume_events() + ac2._evtracker.consume_events() + ac1._evtracker.consume_events() # XXX not sure if the following is correct / possible. you may remove it lp.sec("ac1: delete profile image from chat, and send message to ac2") @@ -1178,7 +1178,7 @@ class TestOnlineAccount: assert m.is_encrypted() lp.sec("ac2: wait for message along with avatar deletion of ac1") - msg3 = ac2.wait_next_incoming_message() + msg3 = ac2._evtracker.wait_next_incoming_message() assert msg3.get_sender_contact().get_profile_image() is None def test_set_get_group_image(self, acfactory, data, lp): @@ -1190,7 +1190,7 @@ class TestOnlineAccount: lp.sec("ac1: set profile image on unpromoted chat") chat.set_profile_image(p) - ac1._evlogger.get_matching("DC_EVENT_CHAT_MODIFIED") + ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED") assert not chat.is_promoted() lp.sec("ac1: send text to promote chat (XXX without contact added)") @@ -1203,7 +1203,7 @@ class TestOnlineAccount: lp.sec("ac2: add ac1 to a chat so the message does not land in DEADDROP") c1 = ac2.create_contact(email=ac1.get_config("addr")) ac2.create_chat_by_contact(c1) - ev = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") lp.sec("ac1: add ac2 to promoted group chat") c2 = ac1.create_contact(email=ac2.get_config("addr")) @@ -1214,7 +1214,7 @@ class TestOnlineAccount: assert chat.is_promoted() lp.sec("ac2: wait for receiving message from ac1") - ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") msg_in = ac2.get_message_by_id(ev[2]) assert not msg_in.chat.is_deaddrop() @@ -1224,11 +1224,11 @@ class TestOnlineAccount: assert p2 is not None assert open(p2, "rb").read() == open(p, "rb").read() - ac2._evlogger.consume_events() - ac1._evlogger.consume_events() + ac2._evtracker.consume_events() + ac1._evtracker.consume_events() lp.sec("ac2: delete profile image from chat") chat2.remove_profile_image() - ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG") + ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") assert ev[1] == chat.id chat1b = ac1.create_chat_by_message(ev[2]) assert chat1b.get_profile_image() is None @@ -1246,25 +1246,25 @@ class TestOnlineAccount: with pytest.raises(ValueError): ac1.set_location(latitude=0.0, longitude=10.0) - ac1._evlogger.consume_events() - ac2._evlogger.consume_events() + ac1._evtracker.consume_events() + ac2._evtracker.consume_events() lp.sec("ac1: enable location sending in chat") chat1.enable_sending_locations(seconds=100) assert chat1.is_sending_locations() - ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5) - ac1._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED") + ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED") chat1.send_text("hello") - ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") + ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") lp.sec("ac2: wait for incoming location message") - ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") # "enabled-location streaming" + ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") # "enabled-location streaming" # currently core emits location changed before event_incoming message - ac2._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED") - ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG") # text message with location + ac2._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED") + ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") # text message with location locations = chat2.get_locations() assert len(locations) == 1 @@ -1306,7 +1306,7 @@ class TestGroupStressTests: # send a message to get the contact key via autocrypt header chat1.send_text("hi") - msg = ac1.wait_next_incoming_message() + msg = ac1._evtracker.wait_next_incoming_message() assert msg.text == "hi" # Save fifth account for later @@ -1332,14 +1332,14 @@ class TestGroupStressTests: lp.sec("ac2: checking that the chat arrived correctly") ac2 = accounts[0] - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() assert msg.text == "hello" print("chat is", msg.chat) assert len(msg.chat.get_contacts()) == 4 lp.sec("ac3: checking that 'ac4' is a known contact") ac3 = accounts[1] - msg3 = ac3.wait_next_incoming_message() + msg3 = ac3._evtracker.wait_next_incoming_message() assert msg3.text == "hello" ac3_contacts = ac3.get_contacts() assert len(ac3_contacts) == 3 @@ -1351,7 +1351,7 @@ class TestGroupStressTests: msg.chat.remove_contact(to_remove) lp.sec("ac1: receiving system message about contact removal") - sysmsg = ac1.wait_next_incoming_message() + sysmsg = ac1._evtracker.wait_next_incoming_message() assert to_remove.addr in sysmsg.text assert len(sysmsg.chat.get_contacts()) == 3 @@ -1360,7 +1360,7 @@ class TestGroupStressTests: lp.sec("ac1: sending another message to the chat") chat.send_text("hello2") - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() assert msg.text == "hello2" assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp @@ -1370,12 +1370,12 @@ class TestGroupStressTests: assert chat.get_summary()["gossiped_timestamp"] >= gossiped_timestamp lp.sec("ac2: receiving system message about contact addition") - sysmsg = ac2.wait_next_incoming_message() + sysmsg = ac2._evtracker.wait_next_incoming_message() assert contact5.addr in sysmsg.text assert len(sysmsg.chat.get_contacts()) == 4 lp.sec("ac5: waiting for message about addition to the chat") - sysmsg = ac5.wait_next_incoming_message() + sysmsg = ac5._evtracker.wait_next_incoming_message() msg = sysmsg.chat.send_text("hello!") # Message should be encrypted because keys of other members are gossiped assert msg.is_encrypted() @@ -1461,7 +1461,7 @@ class TestOnlineConfigureFails: ac1.configure(addr=configdict["addr"], mail_pw="123") ac1.start_threads() wait_configuration_progress(ac1, 500) - ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK") + ev1 = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "cannot login" in ev1[2].lower() wait_configuration_progress(ac1, 0, 0) @@ -1470,7 +1470,7 @@ class TestOnlineConfigureFails: ac1.configure(addr="x" + configdict["addr"], mail_pw=configdict["mail_pw"]) ac1.start_threads() wait_configuration_progress(ac1, 500) - ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK") + ev1 = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "cannot login" in ev1[2].lower() wait_configuration_progress(ac1, 0, 0) @@ -1479,6 +1479,6 @@ class TestOnlineConfigureFails: ac1.configure(addr=configdict["addr"] + "x", mail_pw=configdict["mail_pw"]) ac1.start_threads() wait_configuration_progress(ac1, 500) - ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK") + ev1 = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "could not connect" in ev1[2].lower() wait_configuration_progress(ac1, 0, 0) diff --git a/python/tests/test_increation.py b/python/tests/test_increation.py index 0b6735dda..a1b4384ac 100644 --- a/python/tests/test_increation.py +++ b/python/tests/test_increation.py @@ -91,21 +91,21 @@ class TestOnlineInCreation: assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered() lp.sec("wait for the messages to be delivered to SMTP") - ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED") + ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") assert ev[1] == chat.id assert ev[2] == prepared_original.id - ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED") + ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") assert ev[1] == chat2.id assert ev[2] == forwarded_id lp.sec("wait1 for original or forwarded messages to arrive") - ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL received_original = ac2.get_message_by_id(ev1[2]) assert cmp(received_original.filename, orig, shallow=False) lp.sec("wait2 for original or forwarded messages to arrive") - ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL assert ev2[1] != ev1[1] received_copy = ac2.get_message_by_id(ev2[2]) From 57311d731ec3c1c241e748aa8ed47c9afda55e27 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 19:57:24 +0100 Subject: [PATCH 010/156] simplify logging --- python/src/deltachat/account.py | 17 ++++++++++++++--- python/src/deltachat/eventlogger.py | 18 +++--------------- python/tests/ffi_event.py | 2 +- python/tests/test_lowlevel.py | 12 +++++------- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index c46dfe066..f11a4f99d 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -25,7 +25,10 @@ class Account(object): by the underlying deltachat core library. All public Account methods are meant to be memory-safe and return memory-safe objects. """ - def __init__(self, db_path, logid=None, os_name=None, debug=True): + # to prevent garbled logging + _loglock = threading.RLock() + + def __init__(self, db_path, logid=None, os_name=None): """ initialize account object. :param db_path: a path to the account database. The database @@ -33,13 +36,12 @@ class Account(object): :param logid: an optional logging prefix that should be used with the default internal logging. :param os_name: this will be put to the X-Mailer header in outgoing messages - :param debug: turn on debug logging for events. """ self._dc_context = ffi.gc( lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)), _destroy_dc_context, ) - self._evlogger = EventLogger(self, logid, debug) + self._evlogger = EventLogger(self, logid) self._threads = IOThreads(self._dc_context, self._evlogger._log_event) # initialize per-account plugin system @@ -65,6 +67,15 @@ class Account(object): # def __del__(self): # self.shutdown() + def log_line(self, msg): + t = threading.currentThread() + tname = getattr(t, "name", t) + if tname == "MainThread": + tname = "MAIN" + with self._loglock: + print("{:2.2f} [{}-{}] {}".format(time.time() - self._evlogger.init_time, + tname, self._evlogger.logid, msg)) + def _check_config_key(self, name): if name not in self._configkeys: raise KeyError("{!r} not a valid config key, existing keys: {!r}".format( diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 2d0d3c761..006459feb 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -4,11 +4,8 @@ from .hookspec import account_hookimpl class EventLogger: - _loglock = threading.RLock() - - def __init__(self, account, logid=None, debug=True): + def __init__(self, account, logid=None): self.account = account - self._debug = debug if logid is None: logid = str(self.account._dc_context).strip(">").split()[-1] self.logid = logid @@ -22,14 +19,5 @@ class EventLogger: # don't show events that are anyway empty impls now if evt_name == "DC_EVENT_GET_STRING": return - if self._debug: - evpart = "{}({!r},{!r})".format(evt_name, data1, data2) - self._log(evpart) - - def _log(self, msg): - t = threading.currentThread() - tname = getattr(t, "name", t) - if tname == "MainThread": - tname = "MAIN" - with self._loglock: - print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg)) + evpart = "{}({!r},{!r})".format(evt_name, data1, data2) + self.account.log_line(evpart) diff --git a/python/tests/ffi_event.py b/python/tests/ffi_event.py index 4ffd11b1e..c7d109552 100644 --- a/python/tests/ffi_event.py +++ b/python/tests/ffi_event.py @@ -39,7 +39,7 @@ class FFIEventTracker: assert not rex.match(ev[0]), "event found {}".format(ev) def get_matching(self, event_name_regex, check_error=True, timeout=None): - self.account._evlogger._log("-- waiting for event with regex: {} --".format(event_name_regex)) + self.account.log_line("-- waiting for event with regex: {} --".format(event_name_regex)) rex = re.compile("(?:{}).*".format(event_name_regex)) while 1: ev = self.get(timeout=timeout, check_error=check_error) diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index bf3b1f085..733cbe825 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -16,14 +16,12 @@ def test_callback_None2int(): clear_context_callback(ctx) -def test_dc_close_events(tmpdir): - from deltachat.account import Account - p = tmpdir.join("hello.db") - ac1 = Account(p.strpath) +def test_dc_close_events(tmpdir, acfactory): + ac1 = acfactory.get_unconfigured_account() ac1.shutdown() def find(info_string): - evlog = ac1._evlogger + evlog = ac1._evtracker while 1: ev = evlog.get_matching("DC_EVENT_INFO", check_error=False) data2 = ev[2] @@ -80,10 +78,10 @@ def test_markseen_invalid_message_ids(acfactory): contact1 = ac1.create_contact(email="some1@example.com", name="some1") chat = ac1.create_chat_by_contact(contact1) chat.send_text("one messae") - ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED") + ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") msg_ids = [9] lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids)) - ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR") + ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR") def test_get_special_message_id_returns_empty_message(acfactory): From cf6391d51b1a105a6743f59279c199f37752005f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 21:25:39 +0100 Subject: [PATCH 011/156] move event tracking to new tracker.py file some api cleanups --- python/src/deltachat/account.py | 53 ++++++++------------- python/src/deltachat/eventlogger.py | 1 - python/src/deltachat/hookspec.py | 4 ++ python/src/deltachat/tracker.py | 74 +++++++++++++++++++++++++++++ python/tests/conftest.py | 16 +++---- 5 files changed, 107 insertions(+), 41 deletions(-) create mode 100644 python/src/deltachat/tracker.py diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index f11a4f99d..045dd4de3 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -7,8 +7,6 @@ from contextlib import contextmanager import os import time from array import array -from queue import Queue - import deltachat from . import const from .capi import ffi, lib @@ -17,6 +15,7 @@ from .chat import Chat from .message import Message from .contact import Contact from .eventlogger import EventLogger +from .tracker import ImexTracker from .hookspec import AccountHookSpecs, account_hookimpl @@ -45,13 +44,14 @@ class Account(object): self._threads = IOThreads(self._dc_context, self._evlogger._log_event) # initialize per-account plugin system - self.plugin_manager = AccountHookSpecs._make_plugin_manager() - self.plugin_manager.register(self._evlogger) + self._pm = AccountHookSpecs._make_plugin_manager() + self.add_account_plugin(self) + self.add_account_plugin(self._evlogger) # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): assert ctx == self._dc_context - self.plugin_manager.hook.process_low_level_event( + self._pm.hook.process_low_level_event( account=self, event_name=evt_name, data1=data1, data2=data2 ) deltachat.set_context_callback(self._dc_context, _ll_event) @@ -64,6 +64,19 @@ class Account(object): self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) + @account_hookimpl + def process_low_level_event(self, event_name, data1, data2): + if event_name == "DC_EVENT_CONFIGURE_PROGRESS": + if data1 == 0 or data1 == 1000: + success = data1 == 1000 + self._pm.hook.configure_completed(success=success) + + def add_account_plugin(self, plugin): + """ add an account plugin whose hookimpls are called. """ + self._pm.register(plugin) + self._pm.check_pending() + return plugin + # def __del__(self): # self.shutdown() @@ -519,7 +532,7 @@ class Account(object): deltachat.clear_context_callback(self._dc_context) del self._dc_context atexit.unregister(self.shutdown) - self.plugin_manager.unregister(self._evlogger) + self._pm.unregister(self._evlogger) def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0): """set a new location. It effects all chats where we currently @@ -538,33 +551,9 @@ class Account(object): @contextmanager def temp_plugin(self, plugin): """ run a code block with the given plugin temporarily registered. """ - self.plugin_manager.register(plugin) + self._pm.register(plugin) yield plugin - self.plugin_manager.unregister(plugin) - - -class ImexTracker: - def __init__(self): - self._imex_events = Queue() - - @account_hookimpl - def process_low_level_event(self, event_name, data1, data2): - if event_name == "DC_EVENT_IMEX_PROGRESS": - self._imex_events.put(data1) - elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN": - self._imex_events.put(data1) - - def wait_finish(self, progress_timeout=60): - """ Return list of written files, raise ValueError if ExportFailed. """ - files_written = [] - while True: - ev = self._imex_events.get(timeout=progress_timeout) - if isinstance(ev, str): - files_written.append(ev) - elif ev == 0: - raise ValueError("export failed, exp-files: {}".format(files_written)) - elif ev == 1000: - return files_written + self._pm.unregister(plugin) class IOThreads: diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 006459feb..95831088e 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,4 +1,3 @@ -import threading import time from .hookspec import account_hookimpl diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 6a65d40c9..c960b4a82 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -23,3 +23,7 @@ class AccountHookSpecs: @account_hookspec def process_low_level_event(self, event_name, data1, data2): """ process a CFFI low level events for a given account. """ + + @account_hookspec + def configure_completed(self, success): + """ Called when a configure process completed. """ diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py new file mode 100644 index 000000000..5646a759a --- /dev/null +++ b/python/src/deltachat/tracker.py @@ -0,0 +1,74 @@ + +from queue import Queue +from threading import Event + +from .hookspec import account_hookimpl + + +class ImexFailed(RuntimeError): + """ Exception for signalling that import/export operations failed.""" + + +class ImexTracker: + def __init__(self): + self._imex_events = Queue() + + @account_hookimpl + def process_low_level_event(self, event_name, data1, data2): + if event_name == "DC_EVENT_IMEX_PROGRESS": + self._imex_events.put(data1) + elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN": + self._imex_events.put(data1) + + def wait_finish(self, progress_timeout=60): + """ Return list of written files, raise ValueError if ExportFailed. """ + files_written = [] + while True: + ev = self._imex_events.get(timeout=progress_timeout) + if isinstance(ev, str): + files_written.append(ev) + elif ev == 0: + raise ImexFailed("export failed, exp-files: {}".format(files_written)) + elif ev == 1000: + return files_written + + +class ConfigureFailed(RuntimeError): + """ Exception for signalling that configuration failed.""" + + +class ConfigureTracker: + def __init__(self): + self._configure_events = Queue() + self._smtp_finished = Event() + self._imap_finished = Event() + self._ffi_events = [] + + @account_hookimpl + def process_low_level_event(self, event_name, data1, data2): + self._ffi_events.append((event_name, data1, data2)) + if event_name == "DC_EVENT_SMTP_CONNECTED": + self._smtp_finished.set() + elif event_name == "DC_EVENT_IMAP_CONNECTED": + self._imap_finished.set() + + @account_hookimpl + def configure_completed(self, success): + self._configure_events.put(success) + + def wait_smtp_connected(self): + """ wait until smtp is configured. """ + self._smtp_finished.wait() + + def wait_imap_connected(self): + """ wait until smtp is configured. """ + self._imap_finished.wait() + + def wait_finish(self): + """ wait until configure is completed. + + Raise Exception if Configure failed + """ + if not self._configure_events.get(): + content = "\n".join("{}: {} {}".format(*args) for args in self._ffi_events) + raise ConfigureFailed(content) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 5e0141354..1e6423f3f 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -6,6 +6,7 @@ import pytest import requests import time from deltachat import Account +from deltachat.tracker import ConfigureTracker from deltachat import const from deltachat.capi import lib from _pytest.monkeypatch import MonkeyPatch @@ -164,8 +165,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): def make_account(self, path, logid): ac = Account(path, logid=logid) - ac._evtracker = FFIEventTracker(ac) - ac.plugin_manager.register(ac._evtracker) + ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac)) + ac._configtracker = ac.add_account_plugin(ConfigureTracker()) self._finalizers.append(ac.shutdown) return ac @@ -232,17 +233,16 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): def get_one_online_account(self, pre_generated_key=True): ac1 = self.get_online_configuring_account( pre_generated_key=pre_generated_key) - wait_successful_IMAP_SMTP_connection(ac1) - wait_configuration_progress(ac1, 1000) + ac1._configtracker.wait_imap_connected() + ac1._configtracker.wait_smtp_connected() + ac1._configtracker.wait_finish() return ac1 def get_two_online_accounts(self): ac1 = self.get_online_configuring_account() ac2 = self.get_online_configuring_account() - wait_successful_IMAP_SMTP_connection(ac1) - wait_configuration_progress(ac1, 1000) - wait_successful_IMAP_SMTP_connection(ac2) - wait_configuration_progress(ac2, 1000) + ac1._configtracker.wait_finish() + ac2._configtracker.wait_finish() return ac1, ac2 def clone_online_account(self, account, pre_generated_key=True): From d3c6f530e27ea4673ee50af78f0ae4c55841c855 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 22:09:18 +0100 Subject: [PATCH 012/156] introduce global plugin manager --- python/src/deltachat/__init__.py | 6 +++--- python/src/deltachat/account.py | 23 ++++++++++++++--------- python/src/deltachat/eventlogger.py | 7 ++++++- python/src/deltachat/hookspec.py | 26 ++++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 4a8f21bf9..780f9e462 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -1,6 +1,6 @@ -from deltachat import capi, const -from deltachat.capi import ffi -from deltachat.account import Account # noqa +from . import capi, const +from .capi import ffi +from .account import Account # noqa from pkg_resources import get_distribution, DistributionNotFound try: diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 045dd4de3..a4e53ed10 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -14,9 +14,12 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot from .chat import Chat from .message import Message from .contact import Contact -from .eventlogger import EventLogger +from . import eventlogger from .tracker import ImexTracker -from .hookspec import AccountHookSpecs, account_hookimpl +from . import hookspec + + +hookspec.Global._get_plugin_manager().register(eventlogger) class Account(object): @@ -36,17 +39,19 @@ class Account(object): the default internal logging. :param os_name: this will be put to the X-Mailer header in outgoing messages """ + # initialize per-account plugin system + self._pm = hookspec.PerAccount._make_plugin_manager() + self.add_account_plugin(self) + self._dc_context = ffi.gc( lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)), _destroy_dc_context, ) - self._evlogger = EventLogger(self, logid) - self._threads = IOThreads(self._dc_context, self._evlogger._log_event) - # initialize per-account plugin system - self._pm = AccountHookSpecs._make_plugin_manager() - self.add_account_plugin(self) - self.add_account_plugin(self._evlogger) + hook = hookspec.Global._get_plugin_manager().hook + hook.at_account_init(account=self, db_path=db_path, logid=logid) + + self._threads = IOThreads(self._dc_context) # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): @@ -64,7 +69,7 @@ class Account(object): self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) - @account_hookimpl + @hookspec.account_hookimpl def process_low_level_event(self, event_name, data1, data2): if event_name == "DC_EVENT_CONFIGURE_PROGRESS": if data1 == 0 or data1 == 1000: diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 95831088e..899777e8c 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,5 +1,10 @@ import time -from .hookspec import account_hookimpl +from .hookspec import account_hookimpl, global_hookimpl + + +@global_hookimpl +def at_account_init(account, logid): + account._evlogger = account.add_account_plugin(EventLogger(account, logid=logid)) class EventLogger: diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index c960b4a82..58dc850ef 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -2,14 +2,17 @@ import pluggy -__all__ = ["account_hookspec", "account_hookimpl", "AccountHookSpecs"] _account_name = "deltachat-account" account_hookspec = pluggy.HookspecMarker(_account_name) account_hookimpl = pluggy.HookimplMarker(_account_name) +_global_name = "deltachat-global" +global_hookspec = pluggy.HookspecMarker(_global_name) +global_hookimpl = pluggy.HookimplMarker(_global_name) -class AccountHookSpecs: + +class PerAccount: """ per-Account-instance hook specifications. Account hook implementations need to be registered with an Account instance. @@ -27,3 +30,22 @@ class AccountHookSpecs: @account_hookspec def configure_completed(self, success): """ Called when a configure process completed. """ + + + +class Global: + """ global hook specifications using a per-process singleton plugin manager instance. + + """ + _plugin_manager = None + + @classmethod + def _get_plugin_manager(cls): + if cls._plugin_manager is None: + cls._plugin_manager = pm = pluggy.PluginManager(_global_name) + pm.add_hookspecs(cls) + return cls._plugin_manager + + @global_hookspec + def at_account_init(self, account, logid): + """ called when `Account::__init__()` function starts executing. """ From ce00c627d4aa081d95cd7510d3619151d2627d72 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 22 Feb 2020 23:29:21 +0100 Subject: [PATCH 013/156] don't run Eventlogging by default -- the tests instantiate it, though. --- python/src/deltachat/__init__.py | 7 ++- python/src/deltachat/account.py | 92 ++++++++++++----------------- python/src/deltachat/eventlogger.py | 33 ++++++++--- python/src/deltachat/hookspec.py | 6 +- python/tests/conftest.py | 4 +- 5 files changed, 78 insertions(+), 64 deletions(-) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 780f9e462..a94779305 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -1,4 +1,4 @@ -from . import capi, const +from . import capi, const, hookspec from .capi import ffi from .account import Account # noqa @@ -74,3 +74,8 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): if name.startswith("DC_EVENT_"): _DC_EVENTNAME_MAP[val] = name return _DC_EVENTNAME_MAP[integer] + + +def register_global_plugin(plugin): + gm = hookspect.Global._get_plugin_manager() + gm.register(plugin) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index a4e53ed10..5dc155ae9 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -14,29 +14,20 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot from .chat import Chat from .message import Message from .contact import Contact -from . import eventlogger from .tracker import ImexTracker from . import hookspec -hookspec.Global._get_plugin_manager().register(eventlogger) - - class Account(object): """ Each account is tied to a sqlite database file which is fully managed by the underlying deltachat core library. All public Account methods are meant to be memory-safe and return memory-safe objects. """ - # to prevent garbled logging - _loglock = threading.RLock() - - def __init__(self, db_path, logid=None, os_name=None): + def __init__(self, db_path, os_name=None): """ initialize account object. :param db_path: a path to the account database. The database will be created if it doesn't exist. - :param logid: an optional logging prefix that should be used with - the default internal logging. :param os_name: this will be put to the X-Mailer header in outgoing messages """ # initialize per-account plugin system @@ -49,9 +40,9 @@ class Account(object): ) hook = hookspec.Global._get_plugin_manager().hook - hook.at_account_init(account=self, db_path=db_path, logid=logid) + hook.at_account_init(account=self, db_path=db_path) - self._threads = IOThreads(self._dc_context) + self._threads = IOThreads(self) # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): @@ -86,13 +77,7 @@ class Account(object): # self.shutdown() def log_line(self, msg): - t = threading.currentThread() - tname = getattr(t, "name", t) - if tname == "MainThread": - tname = "MAIN" - with self._loglock: - print("{:2.2f} [{}-{}] {}".format(time.time() - self._evlogger.init_time, - tname, self._evlogger.logid, msg)) + self._pm.hook.log_line(message=msg) def _check_config_key(self, name): if name not in self._configkeys: @@ -537,7 +522,6 @@ class Account(object): deltachat.clear_context_callback(self._dc_context) del self._dc_context atexit.unregister(self.shutdown) - self._pm.unregister(self._evlogger) def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0): """set a new location. It effects all chats where we currently @@ -562,11 +546,11 @@ class Account(object): class IOThreads: - def __init__(self, dc_context, log_event=lambda *args: None): - self._dc_context = dc_context + def __init__(self, account): + self.account = account + self._dc_context = account._dc_context self._thread_quitflag = False self._name2thread = {} - self._log_event = log_event def is_started(self): return len(self._name2thread) > 0 @@ -587,6 +571,12 @@ class IOThreads: t.setDaemon(1) t.start() + @contextmanager + def log_execution(self, message): + self.account.log_line(message + " START") + yield + self.account.log_line(message + " FINISHED") + def stop(self, wait=False): self._thread_quitflag = True @@ -603,42 +593,38 @@ class IOThreads: thread.join() def imap_thread_run(self): - self._log_event("py-bindings-info", 0, "INBOX THREAD START") - while not self._thread_quitflag: - lib.dc_perform_imap_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_imap_fetch(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_imap_idle(self._dc_context) - self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED") + with self.log_execution("INBOX THREAD START"): + while not self._thread_quitflag: + lib.dc_perform_imap_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_imap_fetch(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_imap_idle(self._dc_context) def mvbox_thread_run(self): - self._log_event("py-bindings-info", 0, "MVBOX THREAD START") - while not self._thread_quitflag: - lib.dc_perform_mvbox_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_mvbox_fetch(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_mvbox_idle(self._dc_context) - self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED") + with self.log_execution("MVBOX THREAD"): + while not self._thread_quitflag: + lib.dc_perform_mvbox_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_mvbox_fetch(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_mvbox_idle(self._dc_context) def sentbox_thread_run(self): - self._log_event("py-bindings-info", 0, "SENTBOX THREAD START") - while not self._thread_quitflag: - lib.dc_perform_sentbox_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_sentbox_fetch(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_sentbox_idle(self._dc_context) - self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED") + with self.log_execution("SENTBOX THREAD"): + while not self._thread_quitflag: + lib.dc_perform_sentbox_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_sentbox_fetch(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_sentbox_idle(self._dc_context) def smtp_thread_run(self): - self._log_event("py-bindings-info", 0, "SMTP THREAD START") - while not self._thread_quitflag: - lib.dc_perform_smtp_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_smtp_idle(self._dc_context) - self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED") + with self.log_execution("SMTP THREAD"): + while not self._thread_quitflag: + lib.dc_perform_smtp_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_smtp_idle(self._dc_context) def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 899777e8c..1172d1b5f 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,17 +1,21 @@ +import threading import time from .hookspec import account_hookimpl, global_hookimpl -@global_hookimpl -def at_account_init(account, logid): - account._evlogger = account.add_account_plugin(EventLogger(account, logid=logid)) +class FFIEventLogger: + """ If you register an instance of this logger with an Account + you'll get all ffi-events printed. + """ + # to prevent garbled logging + _loglock = threading.RLock() - -class EventLogger: - def __init__(self, account, logid=None): + def __init__(self, account, logid): + """ + :param logid: an optional logging prefix that should be used with + the default internal logging. + """ self.account = account - if logid is None: - logid = str(self.account._dc_context).strip(">").split()[-1] self.logid = logid self.init_time = time.time() @@ -25,3 +29,16 @@ class EventLogger: return evpart = "{}({!r},{!r})".format(evt_name, data1, data2) self.account.log_line(evpart) + + @account_hookimpl + def log_line(self, message): + t = threading.currentThread() + tname = getattr(t, "name", t) + if tname == "MainThread": + tname = "MAIN" + with self._loglock: + print("{:2.2f} [{}-{}] {}".format( + time.time() - self.init_time, + tname, + self.logid, + message)) diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 58dc850ef..16cc7ed10 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -27,6 +27,10 @@ class PerAccount: def process_low_level_event(self, event_name, data1, data2): """ process a CFFI low level events for a given account. """ + @account_hookspec + def log_line(self, message): + """ log a message related to the account. """ + @account_hookspec def configure_completed(self, success): """ Called when a configure process completed. """ @@ -47,5 +51,5 @@ class Global: return cls._plugin_manager @global_hookspec - def at_account_init(self, account, logid): + def at_account_init(self, account): """ called when `Account::__init__()` function starts executing. """ diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 1e6423f3f..9be0d32b8 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -9,6 +9,7 @@ from deltachat import Account from deltachat.tracker import ConfigureTracker from deltachat import const from deltachat.capi import lib +from deltachat.eventlogger import FFIEventLogger from _pytest.monkeypatch import MonkeyPatch from ffi_event import FFIEventTracker import tempfile @@ -164,9 +165,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): fin() def make_account(self, path, logid): - ac = Account(path, logid=logid) + ac = Account(path) ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac)) ac._configtracker = ac.add_account_plugin(ConfigureTracker()) + ac.add_account_plugin(FFIEventLogger(ac, logid=logid)) self._finalizers.append(ac.shutdown) return ac From f55d4fa73ab766904e686186a0d6c5728227b980 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 00:04:13 +0100 Subject: [PATCH 014/156] rename process_low_level_event to process_ffi_event --- python/src/deltachat/account.py | 4 ++-- python/src/deltachat/eventlogger.py | 2 +- python/src/deltachat/hookspec.py | 2 +- python/src/deltachat/tracker.py | 4 ++-- python/tests/ffi_event.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 5dc155ae9..dcff14191 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -47,7 +47,7 @@ class Account(object): # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): assert ctx == self._dc_context - self._pm.hook.process_low_level_event( + self._pm.hook.process_ffi_event( account=self, event_name=evt_name, data1=data1, data2=data2 ) deltachat.set_context_callback(self._dc_context, _ll_event) @@ -61,7 +61,7 @@ class Account(object): atexit.register(self.shutdown) @hookspec.account_hookimpl - def process_low_level_event(self, event_name, data1, data2): + def process_ffi_event(self, event_name, data1, data2): if event_name == "DC_EVENT_CONFIGURE_PROGRESS": if data1 == 0 or data1 == 1000: success = data1 == 1000 diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 1172d1b5f..d10fc7c9d 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -20,7 +20,7 @@ class FFIEventLogger: self.init_time = time.time() @account_hookimpl - def process_low_level_event(self, event_name, data1, data2): + def process_ffi_event(self, event_name, data1, data2): self._log_event(event_name, data1, data2) def _log_event(self, evt_name, data1, data2): diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 16cc7ed10..10ae6cb7a 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -24,7 +24,7 @@ class PerAccount: return pm @account_hookspec - def process_low_level_event(self, event_name, data1, data2): + def process_ffi_event(self, event_name, data1, data2): """ process a CFFI low level events for a given account. """ @account_hookspec diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 5646a759a..872d72f39 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -14,7 +14,7 @@ class ImexTracker: self._imex_events = Queue() @account_hookimpl - def process_low_level_event(self, event_name, data1, data2): + def process_ffi_event(self, event_name, data1, data2): if event_name == "DC_EVENT_IMEX_PROGRESS": self._imex_events.put(data1) elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN": @@ -45,7 +45,7 @@ class ConfigureTracker: self._ffi_events = [] @account_hookimpl - def process_low_level_event(self, event_name, data1, data2): + def process_ffi_event(self, event_name, data1, data2): self._ffi_events.append((event_name, data1, data2)) if event_name == "DC_EVENT_SMTP_CONNECTED": self._smtp_finished.set() diff --git a/python/tests/ffi_event.py b/python/tests/ffi_event.py index c7d109552..dee580e58 100644 --- a/python/tests/ffi_event.py +++ b/python/tests/ffi_event.py @@ -10,7 +10,7 @@ class FFIEventTracker: self._event_queue = Queue() @account_hookimpl - def process_low_level_event(self, event_name, data1, data2): + def process_ffi_event(self, event_name, data1, data2): self._event_queue.put((event_name, data1, data2)) def set_timeout(self, timeout): From 6baef49f9d9d196b89b37dbd30ee83135e1e09ac Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 00:08:34 +0100 Subject: [PATCH 015/156] add after_shutdown hook --- python/src/deltachat/account.py | 1 + python/src/deltachat/hookspec.py | 4 ++++ python/src/deltachat/tracker.py | 2 ++ python/tests/test_lowlevel.py | 12 ++++++++++++ 4 files changed, 19 insertions(+) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index dcff14191..2a02b22a8 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -522,6 +522,7 @@ class Account(object): deltachat.clear_context_callback(self._dc_context) del self._dc_context atexit.unregister(self.shutdown) + self._pm.hook.after_shutdown() def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0): """set a new location. It effects all chats where we currently diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 10ae6cb7a..092256c11 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -31,6 +31,10 @@ class PerAccount: def log_line(self, message): """ log a message related to the account. """ + @account_hookspec + def after_shutdown(self): + """ Called when the account has been shutdown. """ + @account_hookspec def configure_completed(self, success): """ Called when a configure process completed. """ diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 872d72f39..6e92186a5 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -38,6 +38,8 @@ class ConfigureFailed(RuntimeError): class ConfigureTracker: + ConfigureFailed = ConfigureFailed + def __init__(self): self._configure_events = Queue() self._smtp_finished = Event() diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index 733cbe825..74833f205 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -1,5 +1,6 @@ from __future__ import print_function from deltachat import capi, cutil, const, set_context_callback, clear_context_callback +from deltachat.hookspec import account_hookimpl from deltachat.capi import ffi from deltachat.capi import lib @@ -18,7 +19,18 @@ def test_callback_None2int(): def test_dc_close_events(tmpdir, acfactory): ac1 = acfactory.get_unconfigured_account() + + # register after_shutdown function + l = [] + class ShutdownPlugin: + @account_hookimpl + def after_shutdown(self): + assert not hasattr(ac1, "_dc_context") + l.append(1) + ac1.add_account_plugin(ShutdownPlugin()) + assert hasattr(ac1, "_dc_context") ac1.shutdown() + assert l == [1] def find(info_string): evlog = ac1._evtracker From 84012e760ebe0204cf331ce380e0559dff98b823 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 00:38:40 +0100 Subject: [PATCH 016/156] refine low level event handling slight refactor on printing --- python/src/deltachat/__init__.py | 2 +- python/src/deltachat/account.py | 18 +++++- python/src/deltachat/eventlogger.py | 24 +++---- python/src/deltachat/hookspec.py | 15 +++-- python/src/deltachat/tracker.py | 20 +++--- python/tests/conftest.py | 30 ++------- python/tests/ffi_event.py | 16 ++--- python/tests/test_account.py | 99 ++++++++++++++--------------- python/tests/test_lowlevel.py | 11 ++-- 9 files changed, 117 insertions(+), 118 deletions(-) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index a94779305..e6449181d 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -77,5 +77,5 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): def register_global_plugin(plugin): - gm = hookspect.Global._get_plugin_manager() + gm = hookspec.Global._get_plugin_manager() gm.register(plugin) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 2a02b22a8..2be16a2ed 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -47,8 +47,9 @@ class Account(object): # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): assert ctx == self._dc_context + ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2) self._pm.hook.process_ffi_event( - account=self, event_name=evt_name, data1=data1, data2=data2 + account=self, ffi_event=ffi_event ) deltachat.set_context_callback(self._dc_context, _ll_event) @@ -61,8 +62,9 @@ class Account(object): atexit.register(self.shutdown) @hookspec.account_hookimpl - def process_ffi_event(self, event_name, data1, data2): - if event_name == "DC_EVENT_CONFIGURE_PROGRESS": + def process_ffi_event(self, ffi_event): + if ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS": + data1 = ffi_event.data1 if data1 == 0 or data1 == 1000: success = data1 == 1000 self._pm.hook.configure_completed(success=success) @@ -639,6 +641,16 @@ def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): pass +class FFIEvent: + def __init__(self, name, data1, data2): + self.name = name + self.data1 = data1 + self.data2 = data2 + + def __str__(self): + return "{name} data1={data1} data2={data2}".format(**self.__dict__) + + class ScannedQRCode: def __init__(self, dc_lot): self._dc_lot = dc_lot diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index d10fc7c9d..f813ab1ec 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,6 +1,6 @@ import threading import time -from .hookspec import account_hookimpl, global_hookimpl +from .hookspec import account_hookimpl class FFIEventLogger: @@ -20,15 +20,14 @@ class FFIEventLogger: self.init_time = time.time() @account_hookimpl - def process_ffi_event(self, event_name, data1, data2): - self._log_event(event_name, data1, data2) + def process_ffi_event(self, ffi_event): + self._log_event(ffi_event) - def _log_event(self, evt_name, data1, data2): + def _log_event(self, ffi_event): # don't show events that are anyway empty impls now - if evt_name == "DC_EVENT_GET_STRING": + if ffi_event.name == "DC_EVENT_GET_STRING": return - evpart = "{}({!r},{!r})".format(evt_name, data1, data2) - self.account.log_line(evpart) + self.account.log_line(str(ffi_event)) @account_hookimpl def log_line(self, message): @@ -36,9 +35,10 @@ class FFIEventLogger: tname = getattr(t, "name", t) if tname == "MainThread": tname = "MAIN" + elapsed = time.time() - self.init_time + locname = tname + if self.logid: + locname += "-" + self.logid + s = "{:2.2f} [{}] {}".format(elapsed, locname, message) with self._loglock: - print("{:2.2f} [{}-{}] {}".format( - time.time() - self.init_time, - tname, - self.logid, - message)) + print(s) diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 092256c11..dc9bd028a 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -24,21 +24,24 @@ class PerAccount: return pm @account_hookspec - def process_ffi_event(self, event_name, data1, data2): - """ process a CFFI low level events for a given account. """ + def process_ffi_event(self, ffi_event): + """ process a CFFI low level events for a given account. + + ffi_event has "name", "data1", "data2" attributes according + to https://c.delta.chat/group__DC__EVENT.html + """ @account_hookspec def log_line(self, message): """ log a message related to the account. """ - @account_hookspec - def after_shutdown(self): - """ Called when the account has been shutdown. """ - @account_hookspec def configure_completed(self, success): """ Called when a configure process completed. """ + @account_hookspec + def after_shutdown(self): + """ Called after the account has been shutdown. """ class Global: diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 6e92186a5..815005e97 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -14,11 +14,11 @@ class ImexTracker: self._imex_events = Queue() @account_hookimpl - def process_ffi_event(self, event_name, data1, data2): - if event_name == "DC_EVENT_IMEX_PROGRESS": - self._imex_events.put(data1) - elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN": - self._imex_events.put(data1) + def process_ffi_event(self, ffi_event): + if ffi_event.name == "DC_EVENT_IMEX_PROGRESS": + self._imex_events.put(ffi_event.data1) + elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN": + self._imex_events.put(ffi_event.data1) def wait_finish(self, progress_timeout=60): """ Return list of written files, raise ValueError if ExportFailed. """ @@ -47,11 +47,11 @@ class ConfigureTracker: self._ffi_events = [] @account_hookimpl - def process_ffi_event(self, event_name, data1, data2): - self._ffi_events.append((event_name, data1, data2)) - if event_name == "DC_EVENT_SMTP_CONNECTED": + def process_ffi_event(self, ffi_event): + self._ffi_events.append(ffi_event) + if ffi_event.name == "DC_EVENT_SMTP_CONNECTED": self._smtp_finished.set() - elif event_name == "DC_EVENT_IMAP_CONNECTED": + elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED": self._imap_finished.set() @account_hookimpl @@ -72,5 +72,5 @@ class ConfigureTracker: Raise Exception if Configure failed """ if not self._configure_events.get(): - content = "\n".join("{}: {} {}".format(*args) for args in self._ffi_events) + content = "\n".join(map(str, self._ffi_events)) raise ConfigureFailed(content) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 9be0d32b8..753114eee 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -284,39 +284,23 @@ def lp(): def wait_configuration_progress(account, min_target, max_target=1001): min_target = min(min_target, max_target) while 1: - evt_name, data1, data2 = \ - account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS") - if data1 >= min_target and data1 <= max_target: + event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS") + if event.data1 >= min_target and event.data1 <= max_target: print("** CONFIG PROGRESS {}".format(min_target), account) break def wait_securejoin_inviter_progress(account, target): while 1: - evt_name, data1, data2 = \ - account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS") - if data2 >= target: + event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS") + if event.data2 >= target: print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account) break -def wait_successful_IMAP_SMTP_connection(account): - imap_ok = smtp_ok = False - while not imap_ok or not smtp_ok: - evt_name, data1, data2 = \ - account._evtracker.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED") - if evt_name == "DC_EVENT_IMAP_CONNECTED": - imap_ok = True - print("** IMAP OK", account) - if evt_name == "DC_EVENT_SMTP_CONNECTED": - smtp_ok = True - print("** SMTP OK", account) - print("** IMAP and SMTP logins successful", account) - - def wait_msgs_changed(account, chat_id, msg_id=None): ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev[1] == chat_id + assert ev.data1 == chat_id if msg_id is not None: - assert ev[2] == msg_id - return ev[2] + assert ev.data2 == msg_id + return ev.data2 diff --git a/python/tests/ffi_event.py b/python/tests/ffi_event.py index dee580e58..8be2df8f3 100644 --- a/python/tests/ffi_event.py +++ b/python/tests/ffi_event.py @@ -10,8 +10,8 @@ class FFIEventTracker: self._event_queue = Queue() @account_hookimpl - def process_ffi_event(self, event_name, data1, data2): - self._event_queue.put((event_name, data1, data2)) + def process_ffi_event(self, ffi_event): + self._event_queue.put(ffi_event) def set_timeout(self, timeout): self._timeout = timeout @@ -23,8 +23,8 @@ class FFIEventTracker: def get(self, timeout=None, check_error=True): timeout = timeout if timeout is not None else self._timeout ev = self._event_queue.get(timeout=timeout) - if check_error and ev[0] == "DC_EVENT_ERROR": - raise ValueError("{}({!r},{!r})".format(*ev)) + if check_error and ev.name == "DC_EVENT_ERROR": + raise ValueError(str(ev)) return ev def ensure_event_not_queued(self, event_name_regex): @@ -36,24 +36,24 @@ class FFIEventTracker: except Empty: break else: - assert not rex.match(ev[0]), "event found {}".format(ev) + assert not rex.match(ev.name), "event found {}".format(ev) def get_matching(self, event_name_regex, check_error=True, timeout=None): self.account.log_line("-- waiting for event with regex: {} --".format(event_name_regex)) rex = re.compile("(?:{}).*".format(event_name_regex)) while 1: ev = self.get(timeout=timeout, check_error=check_error) - if rex.match(ev[0]): + if rex.match(ev.name): return ev def get_info_matching(self, regex): rex = re.compile("(?:{}).*".format(regex)) while 1: ev = self.get_matching("DC_EVENT_INFO") - if rex.match(ev[2]): + if rex.match(ev.data2): return ev def wait_next_incoming_message(self): """ wait for and return next incoming message. """ ev = self.get_matching("DC_EVENT_INCOMING_MSG") - return self.account.get_message_by_id(ev[2]) + return self.account.get_message_by_id(ev.data2) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index d88d2e437..b91b9078a 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -464,7 +464,7 @@ class TestOnlineAccount: chat.send_text("message1") lp.sec("ac2: waiting for message from ac1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg_in = ac2.get_message_by_id(ev[2]) + msg_in = ac2.get_message_by_id(ev.data2) assert msg_in.text == "message1" assert not msg_in.is_encrypted() @@ -472,7 +472,7 @@ class TestOnlineAccount: msg_in.chat.send_text("message2") lp.sec("ac1: waiting for message from ac2") ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg2_in = ac1.get_message_by_id(ev[2]) + msg2_in = ac1.get_message_by_id(ev.data2) assert msg2_in.text == "message2" assert msg2_in.is_encrypted() @@ -480,7 +480,7 @@ class TestOnlineAccount: msg2_in.chat.send_text("message3") lp.sec("ac2: waiting for message from ac1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg3_in = ac1.get_message_by_id(ev[2]) + msg3_in = ac1.get_message_by_id(ev.data2) assert msg3_in.text == "message3" assert msg3_in.is_encrypted() @@ -527,8 +527,8 @@ class TestOnlineAccount: assert ac1.get_config("bcc_self") == "0" # make sure we are not sending message to ourselves - assert self_addr not in ev[2] - assert other_addr in ev[2] + assert self_addr not in ev.data2 + assert other_addr in ev.data2 ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE") lp.sec("ac1: setting bcc_self=1") @@ -542,13 +542,13 @@ class TestOnlineAccount: assert ac1.get_config("bcc_self") == "1" # now make sure we are sending message to ourselves too - assert self_addr in ev[2] - assert other_addr in ev[2] + assert self_addr in ev.data2 + assert other_addr in ev.data2 ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE") # Second client receives only second message, but not the first ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ac1_clone.get_message_by_id(ev[2]).text == msg_out.text + assert ac1_clone.get_message_by_id(ev.data2).text == msg_out.text def test_send_file_twice_unicode_filename_mangling(self, tmpdir, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -568,8 +568,8 @@ class TestOnlineAccount: lp.sec("ac2: receive message") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL - return ac2.get_message_by_id(ev[2]) + assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL + return ac2.get_message_by_id(ev.data2) msg = send_and_receive_message() assert msg.text == "withfile" @@ -600,8 +600,8 @@ class TestOnlineAccount: lp.sec("ac2: receive message") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL - msg = ac2.get_message_by_id(ev[2]) + assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL + msg = ac2.get_message_by_id(ev.data2) assert open(msg.filename).read() == content assert msg.filename.endswith(basename) @@ -623,7 +623,7 @@ class TestOnlineAccount: chat = self.get_chat(ac1, ac2) chat.send_text("message1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL + assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL lp.sec("test finished") def test_move_works(self, acfactory): @@ -634,8 +634,8 @@ class TestOnlineAccount: chat = self.get_chat(ac1, ac2) chat.send_text("message1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL - ev = ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") + assert ev.data2 > const.DC_CHAT_ID_LAST_SPECIAL + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") def test_move_works_on_self_sent(self, acfactory): ac1 = acfactory.get_online_configuring_account(mvbox=True) @@ -660,7 +660,7 @@ class TestOnlineAccount: lp.sec("ac2: wait for receive") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - assert ev[2] == msg_out.id + assert ev.data2 == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message2" @@ -693,7 +693,7 @@ class TestOnlineAccount: lp.sec("receiving message") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg_in = ac2.get_message_by_id(ev[2]) + msg_in = ac2.get_message_by_id(ev.data2) assert msg_in.text == "message2" assert not msg_in.is_forwarded() @@ -704,7 +704,7 @@ class TestOnlineAccount: # wait for other account to receive ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg_in = ac2.get_message_by_id(ev[2]) + msg_in = ac2.get_message_by_id(ev.data2) assert msg_in.text == "message2" assert msg_in.is_forwarded() @@ -716,9 +716,9 @@ class TestOnlineAccount: ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") ac1.empty_server_folders(inbox=True, mvbox=True) ev = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") - assert ev[2] == "DeltaChat" + assert ev.data2 == "DeltaChat" ev = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") - assert ev[2] == "INBOX" + assert ev.data2 == "INBOX" def test_send_and_receive_message_markseen(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -732,14 +732,13 @@ class TestOnlineAccount: lp.sec("sending text message from ac1 to ac2") msg_out = chat.send_text("message1") ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") - evt_name, data1, data2 = ev - assert data1 == chat.id - assert data2 == msg_out.id + assert ev.data1 == chat.id + assert ev.data2 == msg_out.id assert msg_out.is_out_delivered() lp.sec("wait for ac2 to receive message") ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev[2] == msg_out.id + assert ev.data2 == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message1" assert not msg_in.is_forwarded() @@ -768,7 +767,7 @@ class TestOnlineAccount: lp.sec("wait for ac2 to receive second message") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - assert ev[2] == msg_out2.id + assert ev.data2 == msg_out2.id msg_in2 = ac2.get_message_by_id(msg_out2.id) lp.sec("mark messages as seen on ac2, wait for changes on ac1") @@ -776,8 +775,8 @@ class TestOnlineAccount: lp.step("1") for i in range(2): ev = ac1._evtracker.get_matching("DC_EVENT_MSG_READ") - assert ev[1] > const.DC_CHAT_ID_LAST_SPECIAL - assert ev[2] > const.DC_MSG_ID_LAST_SPECIAL + assert ev.data1 > const.DC_CHAT_ID_LAST_SPECIAL + assert ev.data2 > const.DC_MSG_ID_LAST_SPECIAL lp.step("2") assert msg_out.is_out_mdn_received() assert msg_out2.is_out_mdn_received() @@ -837,7 +836,7 @@ class TestOnlineAccount: lp.sec("wait for ac2 to receive message") ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev[2] == msg_out.id + assert ev.data2 == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.text == "message1" @@ -847,9 +846,9 @@ class TestOnlineAccount: lp.sec("wait for ac1 to receive message") ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - assert ev[1] == chat.id - assert ev[2] > msg_out.id - msg_back = ac1.get_message_by_id(ev[2]) + assert ev.data1 == chat.id + assert ev.data2 > msg_out.id + msg_back = ac1.get_message_by_id(ev.data2) assert msg_back.text == "message-back" assert msg_back.is_encrypted() @@ -915,8 +914,8 @@ class TestOnlineAccount: lp.sec("wait for ac1 to receive message") ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - assert ev[1] == chat.id - msg_back = ac1.get_message_by_id(ev[2]) + assert ev.data1 == chat.id + msg_back = ac1.get_message_by_id(ev.data2) assert msg_back.text == "message-back" assert msg_back.is_encrypted() @@ -942,7 +941,7 @@ class TestOnlineAccount: lp.sec("wait for ac2 to receive message") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg_in = ac2.get_message_by_id(ev[2]) + msg_in = ac2.get_message_by_id(ev.data2) assert msg_in.text == "message2 -- should be encrypted" assert msg_in.is_encrypted() @@ -960,7 +959,7 @@ class TestOnlineAccount: lp.sec("wait for ac2 to receive message") ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - in_id = ev[2] + in_id = ev.data2 mime = ac2.get_message_by_id(in_id).get_mime_headers() assert mime.get_all("From") assert mime.get_all("Received") @@ -973,14 +972,13 @@ class TestOnlineAccount: path = data.get_path("d.png") msg_out = chat.send_image(path) ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") - evt_name, data1, data2 = ev - assert data1 == chat.id - assert data2 == msg_out.id + assert ev.data1 == chat.id + assert ev.data2 == msg_out.id assert msg_out.is_out_delivered() lp.sec("wait for ac2 to receive message") ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev[2] == msg_out.id + assert ev.data2 == msg_out.id msg_in = ac2.get_message_by_id(msg_out.id) assert msg_in.is_image() assert os.path.exists(msg_in.filename) @@ -1043,7 +1041,7 @@ class TestOnlineAccount: setup_code = ac1.initiate_key_transfer() ac2._evtracker.set_timeout(30) ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - msg = ac2.get_message_by_id(ev[2]) + msg = ac2.get_message_by_id(ev.data2) assert msg.is_setup_message() assert msg.get_setupcodebegin() == setup_code[:2] lp.sec("try a bad setup code") @@ -1070,7 +1068,7 @@ class TestOnlineAccount: lp.sec("trigger second ac setup message, wait for receive ") setup_code2 = ac1.initiate_key_transfer() ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") - msg = ac2.get_message_by_id(ev[2]) + msg = ac2.get_message_by_id(ev.data2) assert msg.is_setup_message() assert msg.get_setupcodebegin() == setup_code2[:2] @@ -1082,6 +1080,7 @@ class TestOnlineAccount: ac1, ac2 = acfactory.get_two_online_accounts() lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin") qr = ac1.get_setup_contact_qr() + lp.sec("ac2: start QR-code based setup contact protocol") ch = ac2.qr_setup_contact(qr) assert ch.id >= 10 @@ -1215,7 +1214,7 @@ class TestOnlineAccount: lp.sec("ac2: wait for receiving message from ac1") ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - msg_in = ac2.get_message_by_id(ev[2]) + msg_in = ac2.get_message_by_id(ev.data2) assert not msg_in.chat.is_deaddrop() lp.sec("ac2: create chat and read profile image") @@ -1229,8 +1228,8 @@ class TestOnlineAccount: lp.sec("ac2: delete profile image from chat") chat2.remove_profile_image() ev = ac1._evtracker.get_matching("DC_EVENT_INCOMING_MSG") - assert ev[1] == chat.id - chat1b = ac1.create_chat_by_message(ev[2]) + assert ev.data1 == chat.id + chat1b = ac1.create_chat_by_message(ev.data2) assert chat1b.get_profile_image() is None assert chat.get_profile_image() is None @@ -1461,8 +1460,8 @@ class TestOnlineConfigureFails: ac1.configure(addr=configdict["addr"], mail_pw="123") ac1.start_threads() wait_configuration_progress(ac1, 500) - ev1 = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") - assert "cannot login" in ev1[2].lower() + ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") + assert "cannot login" in ev.data2.lower() wait_configuration_progress(ac1, 0, 0) def test_invalid_user(self, acfactory): @@ -1470,8 +1469,8 @@ class TestOnlineConfigureFails: ac1.configure(addr="x" + configdict["addr"], mail_pw=configdict["mail_pw"]) ac1.start_threads() wait_configuration_progress(ac1, 500) - ev1 = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") - assert "cannot login" in ev1[2].lower() + ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") + assert "cannot login" in ev.data2.lower() wait_configuration_progress(ac1, 0, 0) def test_invalid_domain(self, acfactory): @@ -1479,6 +1478,6 @@ class TestOnlineConfigureFails: ac1.configure(addr=configdict["addr"] + "x", mail_pw=configdict["mail_pw"]) ac1.start_threads() wait_configuration_progress(ac1, 500) - ev1 = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") - assert "could not connect" in ev1[2].lower() + ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") + assert "could not connect" in ev.data2.lower() wait_configuration_progress(ac1, 0, 0) diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index 74833f205..f1c7238ba 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -21,26 +21,27 @@ def test_dc_close_events(tmpdir, acfactory): ac1 = acfactory.get_unconfigured_account() # register after_shutdown function - l = [] + shutdowns = [] + class ShutdownPlugin: @account_hookimpl def after_shutdown(self): assert not hasattr(ac1, "_dc_context") - l.append(1) + shutdowns.append(1) ac1.add_account_plugin(ShutdownPlugin()) assert hasattr(ac1, "_dc_context") ac1.shutdown() - assert l == [1] + assert shutdowns == [1] def find(info_string): evlog = ac1._evtracker while 1: ev = evlog.get_matching("DC_EVENT_INFO", check_error=False) - data2 = ev[2] + data2 = ev.data2 if info_string in data2: return else: - print("skipping event", *ev) + print("skipping event", ev) find("disconnecting inbox-thread") find("disconnecting sentbox-thread") From 2d74514dd0c88c489a1d95c0965812acddb2120e Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 02:21:51 +0100 Subject: [PATCH 017/156] add some incoming/outgoing message hooks --- python/src/deltachat/account.py | 13 ++++++++++++- python/src/deltachat/hookspec.py | 8 ++++++++ python/tests/test_account.py | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 2be16a2ed..6089a5362 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -63,11 +63,22 @@ class Account(object): @hookspec.account_hookimpl def process_ffi_event(self, ffi_event): - if ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS": + name = ffi_event.name + if name == "DC_EVENT_CONFIGURE_PROGRESS": data1 = ffi_event.data1 if data1 == 0 or data1 == 1000: success = data1 == 1000 self._pm.hook.configure_completed(success=success) + elif name == "DC_EVENT_INCOMING_MSG": + msg = self.get_message_by_id(ffi_event.data2) + self._pm.hook.process_incoming_message(message=msg) + elif name == "DC_EVENT_MSGS_CHANGED": + if ffi_event.data2 != 0: + msg = self.get_message_by_id(ffi_event.data2) + self._pm.hook.process_incoming_message(message=msg) + elif name == "DC_EVENT_MSG_DELIVERED": + msg = self.get_message_by_id(ffi_event.data2) + self._pm.hook.process_message_delivered(message=msg) def add_account_plugin(self, plugin): """ add an account plugin whose hookimpls are called. """ diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index dc9bd028a..a697028f2 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -39,6 +39,14 @@ class PerAccount: def configure_completed(self, success): """ Called when a configure process completed. """ + @account_hookspec + def process_incoming_message(self, message): + """ Called on any incoming message (to deaddrop or chat). """ + + @account_hookspec + def process_message_delivered(self, message): + """ Called when an outgoing message has been delivered to SMTP. """ + @account_hookspec def after_shutdown(self): """ Called after the account has been shutdown. """ diff --git a/python/tests/test_account.py b/python/tests/test_account.py index b91b9078a..c279e0bd5 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -5,6 +5,7 @@ import queue import time from deltachat import const, Account from deltachat.message import Message +from deltachat.hookspec import account_hookimpl from datetime import datetime, timedelta from conftest import (wait_configuration_progress, wait_securejoin_inviter_progress) @@ -968,6 +969,23 @@ class TestOnlineAccount: ac1, ac2 = acfactory.get_two_online_accounts() chat = self.get_chat(ac1, ac2) + message_queue = queue.Queue() + + class InPlugin: + @account_hookimpl + def process_incoming_message(self, message): + message_queue.put(message) + + delivered = queue.Queue() + + class OutPlugin: + @account_hookimpl + def process_message_delivered(self, message): + delivered.put(message) + + ac1.add_account_plugin(OutPlugin()) + ac2.add_account_plugin(InPlugin()) + lp.sec("sending image message from ac1 to ac2") path = data.get_path("d.png") msg_out = chat.send_image(path) @@ -975,6 +993,8 @@ class TestOnlineAccount: assert ev.data1 == chat.id assert ev.data2 == msg_out.id assert msg_out.is_out_delivered() + m = delivered.get() + assert m == msg_out lp.sec("wait for ac2 to receive message") ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") @@ -983,6 +1003,8 @@ class TestOnlineAccount: assert msg_in.is_image() assert os.path.exists(msg_in.filename) assert os.stat(msg_in.filename).st_size == os.stat(path).st_size + m = message_queue.get() + assert m == msg_in def test_import_export_online_all(self, acfactory, tmpdir, lp): ac1 = acfactory.get_online_configuring_account() From c851f9d5a3ba019fa185ba74a32d0b33c67e488d Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 15:49:25 +0100 Subject: [PATCH 018/156] simplify internal thread handling --- python/src/deltachat/account.py | 23 ++++++++++++----------- python/tests/conftest.py | 12 ++++++++++-- python/tests/test_increation.py | 18 +++++++++--------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 6089a5362..702897281 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -509,7 +509,7 @@ class Account(object): # meta API for start/stop and event based processing # - def start_threads(self, mvbox=False, sentbox=False): + def start_threads(self): """ start IMAP/SMTP threads (and configure account if it hasn't happened). :raises: ValueError if 'addr' or 'mail_pw' are not configured. @@ -517,7 +517,7 @@ class Account(object): """ if not self.is_configured(): self.configure() - self._threads.start(mvbox=mvbox, sentbox=sentbox) + self._threads.start() def stop_threads(self, wait=True): """ stop IMAP/SMTP threads. """ @@ -569,16 +569,15 @@ class IOThreads: def is_started(self): return len(self._name2thread) > 0 - def start(self, imap=True, smtp=True, mvbox=False, sentbox=False): + def start(self): assert not self.is_started() - if imap: - self._start_one_thread("inbox", self.imap_thread_run) - if mvbox: + self._start_one_thread("inbox", self.imap_thread_run) + self._start_one_thread("smtp", self.smtp_thread_run) + + if int(self.account.get_config("mvbox_watch")): self._start_one_thread("mvbox", self.mvbox_thread_run) - if sentbox: + if int(self.account.get_config("sentbox_watch")): self._start_one_thread("sentbox", self.sentbox_thread_run) - if smtp: - self._start_one_thread("smtp", self.smtp_thread_run) def _start_one_thread(self, name, func): self._name2thread[name] = t = threading.Thread(target=func, name=name) @@ -600,8 +599,10 @@ class IOThreads: lib.dc_interrupt_imap_idle(self._dc_context) lib.dc_interrupt_smtp_idle(self._dc_context) - lib.dc_interrupt_mvbox_idle(self._dc_context) - lib.dc_interrupt_sentbox_idle(self._dc_context) + if "mvbox" in self._name2thread: + lib.dc_interrupt_mvbox_idle(self._dc_context) + if "sentbox" in self._name2thread: + lib.dc_interrupt_sentbox_idle(self._dc_context) if wait: for name, thread in self._name2thread.items(): thread.join() diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 753114eee..50b35beb1 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -228,8 +228,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): ac, configdict = self.get_online_config( pre_generated_key=pre_generated_key) configdict.update(config) + configdict["mvbox_watch"] = str(int(mvbox)) + configdict["mvbox_move"] = "1" ac.configure(**configdict) - ac.start_threads(mvbox=mvbox, sentbox=sentbox) + ac.start_threads() return ac def get_one_online_account(self, pre_generated_key=True): @@ -255,7 +257,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): self._preconfigure_key(ac, account.get_config("addr")) ac._evtracker.init_time = self.init_time ac._evtracker.set_timeout(30) - ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw")) + ac.configure( + addr=account.get_config("addr"), + mail_pw=account.get_config("mail_pw"), + mvbox_watch=account.get_config("mvbox_watch"), + mvbox_move=account.get_config("mvbox_move"), + sentbox_watch=account.get_config("sentbox_watch"), + ) ac.start_threads() return ac diff --git a/python/tests/test_increation.py b/python/tests/test_increation.py index a1b4384ac..d2f16285f 100644 --- a/python/tests/test_increation.py +++ b/python/tests/test_increation.py @@ -92,21 +92,21 @@ class TestOnlineInCreation: lp.sec("wait for the messages to be delivered to SMTP") ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") - assert ev[1] == chat.id - assert ev[2] == prepared_original.id + assert ev.data1 == chat.id + assert ev.data2 == prepared_original.id ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED") - assert ev[1] == chat2.id - assert ev[2] == forwarded_id + assert ev.data1 == chat2.id + assert ev.data2 == forwarded_id lp.sec("wait1 for original or forwarded messages to arrive") ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL - received_original = ac2.get_message_by_id(ev1[2]) + assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL + received_original = ac2.get_message_by_id(ev1.data2) assert cmp(received_original.filename, orig, shallow=False) lp.sec("wait2 for original or forwarded messages to arrive") ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL - assert ev2[1] != ev1[1] - received_copy = ac2.get_message_by_id(ev2[2]) + assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL + assert ev2.data1 != ev1.data1 + received_copy = ac2.get_message_by_id(ev2.data2) assert cmp(received_copy.filename, orig, shallow=False) From 5c8f558f602481da1fa2134f1040ce75e2f5033b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 17:00:13 +0100 Subject: [PATCH 019/156] - simplify to offer start() and shutdown() as primary account methods, strike start_threads/stop_threads. - introduce update_config(kwargs) method. - group APIs a bit better --- python/src/deltachat/account.py | 115 ++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 50 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 702897281..a9e9654da 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -18,11 +18,17 @@ from .tracker import ImexTracker from . import hookspec +class MissingCredentials(ValueError): + """ Account is missing `addr` and `mail_pw` config values. """ + + class Account(object): """ Each account is tied to a sqlite database file which is fully managed by the underlying deltachat core library. All public Account methods are meant to be memory-safe and return memory-safe objects. """ + MissingCredentials = MissingCredentials + def __init__(self, db_path, os_name=None): """ initialize account object. @@ -80,12 +86,6 @@ class Account(object): msg = self.get_message_by_id(ffi_event.data2) self._pm.hook.process_message_delivered(message=msg) - def add_account_plugin(self, plugin): - """ add an account plugin whose hookimpls are called. """ - self._pm.register(plugin) - self._pm.check_pending() - return plugin - # def __del__(self): # self.shutdown() @@ -163,16 +163,14 @@ class Account(object): if res == 0: raise Exception("Failed to set key") - def configure(self, **kwargs): - """ set config values and configure this account. - + def update_config(self, kwargs): + """ update config values. :param kwargs: name=value config settings for this account. values need to be unicode. :returns: None """ - for name, value in kwargs.items(): - self.set_config(name, value) - lib.dc_configure(self._dc_context) + for key, value in kwargs.items(): + self.set_config(key, str(value)) def is_configured(self): """ determine if the account is configured already; an initial connection @@ -180,7 +178,7 @@ class Account(object): :returns: True if account is configured. """ - return lib.dc_is_configured(self._dc_context) + return bool(lib.dc_is_configured(self._dc_context)) def set_avatar(self, img_path): """Set self avatar. @@ -397,13 +395,18 @@ class Account(object): lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids)) def export_self_keys(self, path): - """ export public and private keys to the specified directory. """ + """ export public and private keys to the specified directory. + + Note that the account does not have to be started. + """ return self._export(path, imex_cmd=1) def export_all(self, path): """return new file containing a backup of all database state (chats, contacts, keys, media, ...). The file is created in the the `path` directory. + + Note that the account does not have to be started. """ export_files = self._export(path, 11) if len(export_files) != 1: @@ -421,6 +424,8 @@ class Account(object): """ Import private keys found in the `path` directory. The last imported key is made the default keys unless its name contains the string legacy. Public keys are not imported. + + Note that the account does not have to be started. """ self._import(path, imex_cmd=2) @@ -502,41 +507,6 @@ class Account(object): raise ValueError("could not join group") return Chat(self, chat_id) - def stop_ongoing(self): - lib.dc_stop_ongoing_process(self._dc_context) - - # - # meta API for start/stop and event based processing - # - - def start_threads(self): - """ start IMAP/SMTP threads (and configure account if it hasn't happened). - - :raises: ValueError if 'addr' or 'mail_pw' are not configured. - :returns: None - """ - if not self.is_configured(): - self.configure() - self._threads.start() - - def stop_threads(self, wait=True): - """ stop IMAP/SMTP threads. """ - if self._threads.is_started(): - self.stop_ongoing() - self._threads.stop(wait=wait) - - def shutdown(self, wait=True): - """ stop threads and close and remove underlying dc_context and callbacks. """ - if hasattr(self, "_dc_context") and hasattr(self, "_threads"): - # print("SHUTDOWN", self) - self.stop_threads(wait=False) - lib.dc_close(self._dc_context) - self.stop_threads(wait=wait) # to wait for threads - deltachat.clear_context_callback(self._dc_context) - del self._dc_context - atexit.unregister(self.shutdown) - self._pm.hook.after_shutdown() - def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0): """set a new location. It effects all chats where we currently have enabled location streaming. @@ -551,13 +521,58 @@ class Account(object): if dc_res == 0: raise ValueError("no chat is streaming locations") + # + # meta API for start/stop and event based processing + # + + def add_account_plugin(self, plugin): + """ add an account plugin whose hookimpls are called. """ + self._pm.register(plugin) + self._pm.check_pending() + return plugin + @contextmanager def temp_plugin(self, plugin): - """ run a code block with the given plugin temporarily registered. """ + """ run a with-block with the given plugin temporarily registered. """ self._pm.register(plugin) yield plugin self._pm.unregister(plugin) + def stop_ongoing(self): + """ Stop ongoing securejoin, configuration or other core jobs. """ + lib.dc_stop_ongoing_process(self._dc_context) + + def start(self): + """ start this account (activate imap/smtp threads etc.) + and return immediately. + + If this account is not configured, an internal configuration + job will be scheduled if config values are sufficiently specified. + + :raises MissingCredentials: if `addr` and `mail_pw` values are not set. + + :returns: None + """ + if not self.is_configured(): + if not self.get_config("addr") or not self.get_config("mail_pwd"): + raise MissingCredentials("addr or mail_pwd not set in config") + lib.dc_configure(self._dc_context) + self._threads.start() + + def shutdown(self, wait=True): + """ shutdown account, stop threads and close and remove + underlying dc_context and callbacks. """ + if hasattr(self, "_dc_context") and hasattr(self, "_threads"): + if self._threads.is_started(): + self.stop_ongoing() + self._threads.stop(wait=False) + lib.dc_close(self._dc_context) + self._threads.stop(wait=wait) # to wait for threads + deltachat.clear_context_callback(self._dc_context) + del self._dc_context + atexit.unregister(self.shutdown) + self._pm.hook.after_shutdown() + class IOThreads: def __init__(self, account): From 0d4b6f56270ab380388399a45f670a4b8c12b90c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 17:22:27 +0100 Subject: [PATCH 020/156] move io thread handling into own module --- python/src/deltachat/account.py | 99 ++++--------------------------- python/src/deltachat/iothreads.py | 90 ++++++++++++++++++++++++++++ python/tests/test_account.py | 4 +- 3 files changed, 103 insertions(+), 90 deletions(-) create mode 100644 python/src/deltachat/iothreads.py diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index a9e9654da..8976b3cd4 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -2,10 +2,9 @@ from __future__ import print_function import atexit -import threading from contextlib import contextmanager +from threading import Event import os -import time from array import array import deltachat from . import const @@ -15,7 +14,7 @@ from .chat import Chat from .message import Message from .contact import Contact from .tracker import ImexTracker -from . import hookspec +from . import hookspec, iothreads class MissingCredentials(ValueError): @@ -48,7 +47,8 @@ class Account(object): hook = hookspec.Global._get_plugin_manager().hook hook.at_account_init(account=self, db_path=db_path) - self._threads = IOThreads(self) + self._shutdown_event = Event() + self._threads = iothreads.IOThreads(self) # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): @@ -559,6 +559,14 @@ class Account(object): lib.dc_configure(self._dc_context) self._threads.start() + @hookspec.account_hookimpl + def after_shutdown(self): + self._shutdown_event.set() + + def wait_shutdown(self): + """ wait until shutdown of this account has completed. """ + self._shutdown_event.wait() + def shutdown(self, wait=True): """ shutdown account, stop threads and close and remove underlying dc_context and callbacks. """ @@ -574,89 +582,6 @@ class Account(object): self._pm.hook.after_shutdown() -class IOThreads: - def __init__(self, account): - self.account = account - self._dc_context = account._dc_context - self._thread_quitflag = False - self._name2thread = {} - - def is_started(self): - return len(self._name2thread) > 0 - - def start(self): - assert not self.is_started() - self._start_one_thread("inbox", self.imap_thread_run) - self._start_one_thread("smtp", self.smtp_thread_run) - - if int(self.account.get_config("mvbox_watch")): - self._start_one_thread("mvbox", self.mvbox_thread_run) - if int(self.account.get_config("sentbox_watch")): - self._start_one_thread("sentbox", self.sentbox_thread_run) - - def _start_one_thread(self, name, func): - self._name2thread[name] = t = threading.Thread(target=func, name=name) - t.setDaemon(1) - t.start() - - @contextmanager - def log_execution(self, message): - self.account.log_line(message + " START") - yield - self.account.log_line(message + " FINISHED") - - def stop(self, wait=False): - self._thread_quitflag = True - - # Workaround for a race condition. Make sure that thread is - # not in between checking for quitflag and entering idle. - time.sleep(0.5) - - lib.dc_interrupt_imap_idle(self._dc_context) - lib.dc_interrupt_smtp_idle(self._dc_context) - if "mvbox" in self._name2thread: - lib.dc_interrupt_mvbox_idle(self._dc_context) - if "sentbox" in self._name2thread: - lib.dc_interrupt_sentbox_idle(self._dc_context) - if wait: - for name, thread in self._name2thread.items(): - thread.join() - - def imap_thread_run(self): - with self.log_execution("INBOX THREAD START"): - while not self._thread_quitflag: - lib.dc_perform_imap_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_imap_fetch(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_imap_idle(self._dc_context) - - def mvbox_thread_run(self): - with self.log_execution("MVBOX THREAD"): - while not self._thread_quitflag: - lib.dc_perform_mvbox_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_mvbox_fetch(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_mvbox_idle(self._dc_context) - - def sentbox_thread_run(self): - with self.log_execution("SENTBOX THREAD"): - while not self._thread_quitflag: - lib.dc_perform_sentbox_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_sentbox_fetch(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_sentbox_idle(self._dc_context) - - def smtp_thread_run(self): - with self.log_execution("SMTP THREAD"): - while not self._thread_quitflag: - lib.dc_perform_smtp_jobs(self._dc_context) - if not self._thread_quitflag: - lib.dc_perform_smtp_idle(self._dc_context) - - def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): # destructor for dc_context dc_context_unref(dc_context) diff --git a/python/src/deltachat/iothreads.py b/python/src/deltachat/iothreads.py new file mode 100644 index 000000000..798483f74 --- /dev/null +++ b/python/src/deltachat/iothreads.py @@ -0,0 +1,90 @@ + +import threading +import time + +from contextlib import contextmanager + +from .capi import lib + + +class IOThreads: + def __init__(self, account): + self.account = account + self._dc_context = account._dc_context + self._thread_quitflag = False + self._name2thread = {} + + def is_started(self): + return len(self._name2thread) > 0 + + def start(self): + assert not self.is_started() + self._start_one_thread("inbox", self.imap_thread_run) + self._start_one_thread("smtp", self.smtp_thread_run) + + if int(self.account.get_config("mvbox_watch")): + self._start_one_thread("mvbox", self.mvbox_thread_run) + if int(self.account.get_config("sentbox_watch")): + self._start_one_thread("sentbox", self.sentbox_thread_run) + + def _start_one_thread(self, name, func): + self._name2thread[name] = t = threading.Thread(target=func, name=name) + t.setDaemon(1) + t.start() + + @contextmanager + def log_execution(self, message): + self.account.log_line(message + " START") + yield + self.account.log_line(message + " FINISHED") + + def stop(self, wait=False): + self._thread_quitflag = True + + # Workaround for a race condition. Make sure that thread is + # not in between checking for quitflag and entering idle. + time.sleep(0.5) + + lib.dc_interrupt_imap_idle(self._dc_context) + lib.dc_interrupt_smtp_idle(self._dc_context) + if "mvbox" in self._name2thread: + lib.dc_interrupt_mvbox_idle(self._dc_context) + if "sentbox" in self._name2thread: + lib.dc_interrupt_sentbox_idle(self._dc_context) + if wait: + for name, thread in self._name2thread.items(): + thread.join() + + def imap_thread_run(self): + with self.log_execution("INBOX THREAD START"): + while not self._thread_quitflag: + lib.dc_perform_imap_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_imap_fetch(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_imap_idle(self._dc_context) + + def mvbox_thread_run(self): + with self.log_execution("MVBOX THREAD"): + while not self._thread_quitflag: + lib.dc_perform_mvbox_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_mvbox_fetch(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_mvbox_idle(self._dc_context) + + def sentbox_thread_run(self): + with self.log_execution("SENTBOX THREAD"): + while not self._thread_quitflag: + lib.dc_perform_sentbox_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_sentbox_fetch(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_sentbox_idle(self._dc_context) + + def smtp_thread_run(self): + with self.log_execution("SMTP THREAD"): + while not self._thread_quitflag: + lib.dc_perform_smtp_jobs(self._dc_context) + if not self._thread_quitflag: + lib.dc_perform_smtp_idle(self._dc_context) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index c279e0bd5..a714c76b6 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -355,13 +355,11 @@ class TestOfflineChat: contact = msg.get_sender_contact() assert contact == ac1.get_self_contact() - def test_basic_configure_ok_addr_setting_forbidden(self, ac1): + def test_set_config_after_configure_is_forbidden(self, ac1): assert ac1.get_config("mail_pw") assert ac1.is_configured() with pytest.raises(ValueError): ac1.set_config("addr", "123@example.org") - with pytest.raises(ValueError): - ac1.configure(addr="123@example.org") def test_import_export_one_contact(self, acfactory, tmpdir): backupdir = tmpdir.mkdir("backup") From 79f5e736b002e089fa5b56524a2afc98820cbc87 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 23 Feb 2020 17:43:29 +0100 Subject: [PATCH 021/156] make eventlogger module a global plugin --- python/src/deltachat/__init__.py | 5 +++ python/src/deltachat/account.py | 57 +++++++++++------------------ python/src/deltachat/eventlogger.py | 30 ++++++++++++++- python/src/deltachat/hookspec.py | 11 +++--- python/tests/conftest.py | 10 ++--- python/tests/test_lowlevel.py | 15 ++++---- 6 files changed, 74 insertions(+), 54 deletions(-) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index e6449181d..2e52cbc46 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -1,6 +1,7 @@ from . import capi, const, hookspec from .capi import ffi from .account import Account # noqa +from . import eventlogger from pkg_resources import get_distribution, DistributionNotFound try: @@ -79,3 +80,7 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): def register_global_plugin(plugin): gm = hookspec.Global._get_plugin_manager() gm.register(plugin) + gm.check_pending() + + +register_global_plugin(eventlogger) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 8976b3cd4..f138b4450 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -45,20 +45,10 @@ class Account(object): ) hook = hookspec.Global._get_plugin_manager().hook - hook.at_account_init(account=self, db_path=db_path) + hook.account_init(account=self, db_path=db_path) - self._shutdown_event = Event() self._threads = iothreads.IOThreads(self) - # send all FFI events for this account to a plugin hook - def _ll_event(ctx, evt_name, data1, data2): - assert ctx == self._dc_context - ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2) - self._pm.hook.process_ffi_event( - account=self, ffi_event=ffi_event - ) - deltachat.set_context_callback(self._dc_context, _ll_event) - # open database if hasattr(db_path, "encode"): db_path = db_path.encode("utf8") @@ -66,6 +56,8 @@ class Account(object): raise ValueError("Could not dc_open: {}".format(db_path)) self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) + self._shutdown_event = Event() + @hookspec.account_hookimpl def process_ffi_event(self, ffi_event): @@ -549,20 +541,19 @@ class Account(object): If this account is not configured, an internal configuration job will be scheduled if config values are sufficiently specified. + You may call :method:`wait_shutdown` or `shutdown` after the + account is in started mode. + :raises MissingCredentials: if `addr` and `mail_pw` values are not set. :returns: None """ if not self.is_configured(): - if not self.get_config("addr") or not self.get_config("mail_pwd"): + if not self.get_config("addr") or not self.get_config("mail_pw"): raise MissingCredentials("addr or mail_pwd not set in config") lib.dc_configure(self._dc_context) self._threads.start() - @hookspec.account_hookimpl - def after_shutdown(self): - self._shutdown_event.set() - def wait_shutdown(self): """ wait until shutdown of this account has completed. """ self._shutdown_event.wait() @@ -570,16 +561,20 @@ class Account(object): def shutdown(self, wait=True): """ shutdown account, stop threads and close and remove underlying dc_context and callbacks. """ - if hasattr(self, "_dc_context") and hasattr(self, "_threads"): - if self._threads.is_started(): - self.stop_ongoing() - self._threads.stop(wait=False) - lib.dc_close(self._dc_context) - self._threads.stop(wait=wait) # to wait for threads - deltachat.clear_context_callback(self._dc_context) - del self._dc_context - atexit.unregister(self.shutdown) - self._pm.hook.after_shutdown() + dc_context = self._dc_context + if dc_context is None: + return + + if self._threads.is_started(): + self.stop_ongoing() + self._threads.stop(wait=False) + lib.dc_close(dc_context) + self._threads.stop(wait=wait) # to wait for threads + self._dc_context = None + atexit.unregister(self.shutdown) + hook = hookspec.Global._get_plugin_manager().hook + hook.account_after_shutdown(account=self, dc_context=dc_context) + self._shutdown_event.set() def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): @@ -593,16 +588,6 @@ def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): pass -class FFIEvent: - def __init__(self, name, data1, data2): - self.name = name - self.data1 = data1 - self.data2 = data2 - - def __str__(self): - return "{name} data1={data1} data2={data2}".format(**self.__dict__) - - class ScannedQRCode: def __init__(self, dc_lot): self._dc_lot = dc_lot diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index f813ab1ec..363f0359a 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,6 +1,34 @@ +import deltachat import threading import time -from .hookspec import account_hookimpl +from .hookspec import account_hookimpl, global_hookimpl + + +@global_hookimpl +def account_init(account): + # send all FFI events for this account to a plugin hook + def _ll_event(ctx, evt_name, data1, data2): + assert ctx == account._dc_context + ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2) + account._pm.hook.process_ffi_event( + account=account, ffi_event=ffi_event + ) + deltachat.set_context_callback(account._dc_context, _ll_event) + + +@global_hookimpl +def account_after_shutdown(dc_context): + deltachat.clear_context_callback(dc_context) + + +class FFIEvent: + def __init__(self, name, data1, data2): + self.name = name + self.data1 = data1 + self.data2 = data2 + + def __str__(self): + return "{name} data1={data1} data2={data2}".format(**self.__dict__) class FFIEventLogger: diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index a697028f2..159b4a3f8 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -47,10 +47,6 @@ class PerAccount: def process_message_delivered(self, message): """ Called when an outgoing message has been delivered to SMTP. """ - @account_hookspec - def after_shutdown(self): - """ Called after the account has been shutdown. """ - class Global: """ global hook specifications using a per-process singleton plugin manager instance. @@ -66,5 +62,10 @@ class Global: return cls._plugin_manager @global_hookspec - def at_account_init(self, account): + def account_init(self, account): """ called when `Account::__init__()` function starts executing. """ + + @global_hookspec + def account_after_shutdown(self, account, dc_context): + """ Called after the account has been shutdown. """ + diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 50b35beb1..c470b2236 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -230,8 +230,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): configdict.update(config) configdict["mvbox_watch"] = str(int(mvbox)) configdict["mvbox_move"] = "1" - ac.configure(**configdict) - ac.start_threads() + ac.update_config(configdict) + ac.start() return ac def get_one_online_account(self, pre_generated_key=True): @@ -257,14 +257,14 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): self._preconfigure_key(ac, account.get_config("addr")) ac._evtracker.init_time = self.init_time ac._evtracker.set_timeout(30) - ac.configure( + ac.update_config(dict( addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"), mvbox_watch=account.get_config("mvbox_watch"), mvbox_move=account.get_config("mvbox_move"), sentbox_watch=account.get_config("sentbox_watch"), - ) - ac.start_threads() + )) + ac.start() return ac am = AccountMaker() diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index f1c7238ba..1a6259a2e 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -1,6 +1,7 @@ from __future__ import print_function from deltachat import capi, cutil, const, set_context_callback, clear_context_callback -from deltachat.hookspec import account_hookimpl +from deltachat import register_global_plugin +from deltachat.hookspec import account_hookimpl, global_hookimpl from deltachat.capi import ffi from deltachat.capi import lib @@ -24,14 +25,14 @@ def test_dc_close_events(tmpdir, acfactory): shutdowns = [] class ShutdownPlugin: - @account_hookimpl - def after_shutdown(self): - assert not hasattr(ac1, "_dc_context") - shutdowns.append(1) - ac1.add_account_plugin(ShutdownPlugin()) + @global_hookimpl + def account_after_shutdown(self, account): + assert account._dc_context is None + shutdowns.append(account) + register_global_plugin(ShutdownPlugin()) assert hasattr(ac1, "_dc_context") ac1.shutdown() - assert shutdowns == [1] + assert shutdowns == [ac1] def find(info_string): evlog = ac1._evtracker From fb33c3137819d373f7f640fc73d153b439d2dce6 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 26 Feb 2020 14:15:28 +0100 Subject: [PATCH 022/156] fix a couple of issues wrt to configuring move/mvbox behaviour in tests --- python/src/deltachat/account.py | 1 - python/src/deltachat/hookspec.py | 1 - python/tests/conftest.py | 13 +++++++------ python/tests/test_account.py | 32 ++++++++++++++++---------------- python/tests/test_lowlevel.py | 2 +- 5 files changed, 24 insertions(+), 25 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index f138b4450..946a515b7 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -58,7 +58,6 @@ class Account(object): atexit.register(self.shutdown) self._shutdown_event = Event() - @hookspec.account_hookimpl def process_ffi_event(self, ffi_event): name = ffi_event.name diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 159b4a3f8..da99a1660 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -68,4 +68,3 @@ class Global: @global_hookspec def account_after_shutdown(self, account, dc_context): """ Called after the account has been shutdown. """ - diff --git a/python/tests/conftest.py b/python/tests/conftest.py index c470b2236..ea545012d 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -223,27 +223,28 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): ac._evtracker.set_timeout(30) return ac, dict(configdict) - def get_online_configuring_account(self, mvbox=False, sentbox=False, + def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False, pre_generated_key=True, config={}): ac, configdict = self.get_online_config( pre_generated_key=pre_generated_key) configdict.update(config) configdict["mvbox_watch"] = str(int(mvbox)) - configdict["mvbox_move"] = "1" + configdict["mvbox_move"] = str(int(move)) + configdict["sentbox_watch"] = str(int(sentbox)) ac.update_config(configdict) ac.start() return ac - def get_one_online_account(self, pre_generated_key=True): + def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False): ac1 = self.get_online_configuring_account( - pre_generated_key=pre_generated_key) + pre_generated_key=pre_generated_key, mvbox=mvbox, move=move) ac1._configtracker.wait_imap_connected() ac1._configtracker.wait_smtp_connected() ac1._configtracker.wait_finish() return ac1 - def get_two_online_accounts(self): - ac1 = self.get_online_configuring_account() + def get_two_online_accounts(self, move=False): + ac1 = self.get_online_configuring_account(move=True) ac2 = self.get_online_configuring_account() ac1._configtracker.wait_finish() ac2._configtracker.wait_finish() diff --git a/python/tests/test_account.py b/python/tests/test_account.py index a714c76b6..1b16238ce 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -497,7 +497,7 @@ class TestOnlineAccount: for x in export_files: assert x.startswith(dir.strpath) ac1._evtracker.consume_events() - ac1.import_self_keys(dir.strpath) + ac2.import_self_keys(dir.strpath) def test_one_account_send_bcc_setting(self, acfactory, lp): ac1 = acfactory.get_online_configuring_account() @@ -607,7 +607,7 @@ class TestOnlineAccount: def test_mvbox_sentbox_threads(self, acfactory, lp): lp.sec("ac1: start with mvbox thread") - ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True) + ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True) lp.sec("ac2: start without mvbox/sentbox threads") ac2 = acfactory.get_online_configuring_account() @@ -627,7 +627,7 @@ class TestOnlineAccount: def test_move_works(self, acfactory): ac1 = acfactory.get_online_configuring_account() - ac2 = acfactory.get_online_configuring_account(mvbox=True) + ac2 = acfactory.get_online_configuring_account(mvbox=True, move=True) wait_configuration_progress(ac2, 1000) wait_configuration_progress(ac1, 1000) chat = self.get_chat(ac1, ac2) @@ -637,7 +637,7 @@ class TestOnlineAccount: ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED") def test_move_works_on_self_sent(self, acfactory): - ac1 = acfactory.get_online_configuring_account(mvbox=True) + ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True) ac1.set_config("bcc_self", "1") ac2 = acfactory.get_online_configuring_account() wait_configuration_progress(ac2, 1000) @@ -708,16 +708,16 @@ class TestOnlineAccount: assert msg_in.is_forwarded() def test_send_self_message_and_empty_folder(self, acfactory, lp): - ac1 = acfactory.get_one_online_account() + ac1 = acfactory.get_one_online_account(mvbox=True, move=True) lp.sec("ac1: create self chat") chat = ac1.create_chat_by_contact(ac1.get_self_contact()) chat.send_text("hello") ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT") ac1.empty_server_folders(inbox=True, mvbox=True) - ev = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") - assert ev.data2 == "DeltaChat" - ev = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") - assert ev.data2 == "INBOX" + ev1 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") + ev2 = ac1._evtracker.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED") + boxes = sorted([ev1.data2, ev2.data2]) + assert boxes == ["DeltaChat", "INBOX"] def test_send_and_receive_message_markseen(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -789,7 +789,7 @@ class TestOnlineAccount: pass # mark_seen_messages() has generated events before it returns def test_mdn_asymetric(self, acfactory, lp): - ac1, ac2 = acfactory.get_two_online_accounts() + ac1, ac2 = acfactory.get_two_online_accounts(move=True) lp.sec("ac1: create chat with ac2") chat = self.get_chat(ac1, ac2, both_created=True) @@ -1477,8 +1477,8 @@ class TestGroupStressTests: class TestOnlineConfigureFails: def test_invalid_password(self, acfactory): ac1, configdict = acfactory.get_online_config() - ac1.configure(addr=configdict["addr"], mail_pw="123") - ac1.start_threads() + ac1.update_config(dict(addr=configdict["addr"], mail_pw="123")) + ac1.start() wait_configuration_progress(ac1, 500) ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "cannot login" in ev.data2.lower() @@ -1486,8 +1486,8 @@ class TestOnlineConfigureFails: def test_invalid_user(self, acfactory): ac1, configdict = acfactory.get_online_config() - ac1.configure(addr="x" + configdict["addr"], mail_pw=configdict["mail_pw"]) - ac1.start_threads() + ac1.update_config(dict(addr="x" + configdict["addr"], mail_pw=configdict["mail_pw"])) + ac1.start() wait_configuration_progress(ac1, 500) ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "cannot login" in ev.data2.lower() @@ -1495,8 +1495,8 @@ class TestOnlineConfigureFails: def test_invalid_domain(self, acfactory): ac1, configdict = acfactory.get_online_config() - ac1.configure(addr=configdict["addr"] + "x", mail_pw=configdict["mail_pw"]) - ac1.start_threads() + ac1.update_config((dict(addr=configdict["addr"] + "x", mail_pw=configdict["mail_pw"]))) + ac1.start() wait_configuration_progress(ac1, 500) ev = ac1._evtracker.get_matching("DC_EVENT_ERROR_NETWORK") assert "could not connect" in ev.data2.lower() diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index 1a6259a2e..37dfd5625 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -1,7 +1,7 @@ from __future__ import print_function from deltachat import capi, cutil, const, set_context_callback, clear_context_callback from deltachat import register_global_plugin -from deltachat.hookspec import account_hookimpl, global_hookimpl +from deltachat.hookspec import global_hookimpl from deltachat.capi import ffi from deltachat.capi import lib From 6213917089f954fe013f274a806939151cf012eb Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 26 Feb 2020 15:10:57 +0100 Subject: [PATCH 023/156] start some docs --- python/doc/examples.rst | 7 ++++--- python/doc/index.rst | 5 +++-- python/doc/plugins.rst | 27 +++++++++++++++++++++++++++ python/src/deltachat/__init__.py | 3 +++ python/src/deltachat/account.py | 4 +++- python/src/deltachat/hookspec.py | 10 +++++++--- python/tests/conftest.py | 13 +++++++++++++ python/tests/test_account.py | 11 +++++++++++ 8 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 python/doc/plugins.rst diff --git a/python/doc/examples.rst b/python/doc/examples.rst index 94e15e754..9a489f783 100644 --- a/python/doc/examples.rst +++ b/python/doc/examples.rst @@ -14,10 +14,11 @@ For example you can type ``python`` and then:: # instantiate and configure deltachat account import deltachat ac = deltachat.Account("/tmp/db") + ac.set_config("addr", "test2@hq5.merlinux.eu") + ac.set_config("mail_pwd", "some password") - # start configuration activity and smtp/imap threads - ac.start_threads() - ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********") + # start the IO threads and perform configuration + ac.start() # create a contact and send a message contact = ac.create_contact("someother@email.address") diff --git a/python/doc/index.rst b/python/doc/index.rst index 1c27b9317..fb93a56d4 100644 --- a/python/doc/index.rst +++ b/python/doc/index.rst @@ -4,8 +4,9 @@ deltachat python bindings The ``deltachat`` Python package provides two layers of bindings for the core Rust-library of the https://delta.chat messaging ecosystem: -- :doc:`api` is a high level interface to deltachat-core which aims - to be memory safe and thoroughly tested through continous tox/pytest runs. +- :doc:`api` is a high level interface to deltachat-core. + +- :doc:`plugins` is a brief introduction into implementing plugin hooks. - :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core `_. diff --git a/python/doc/plugins.rst b/python/doc/plugins.rst new file mode 100644 index 000000000..281669991 --- /dev/null +++ b/python/doc/plugins.rst @@ -0,0 +1,27 @@ + +Implementing Plugin Hooks +========================== + +The Delta Chat Python bindings use `pluggy `_ +for managing global and per-account plugin registration, and performing +hook calls. + + +.. autoclass:: deltachat.register_global_plugin + +.. autoclass:: deltachat.account.Account.add_account_plugin + + +Per-Account Hook specifications +------------------------------- + +.. autoclass:: deltachat.hookspec.PerAccount + :members: + + +Global Hook specifications +-------------------------- + +.. autoclass:: deltachat.hookspec.Global + :members: + diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 2e52cbc46..43c100700 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -78,6 +78,9 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): def register_global_plugin(plugin): + """ Register a global plugin which implements one or more + of the :class:`deltachat.hookspec.Global` specs. + """ gm = hookspec.Global._get_plugin_manager() gm.register(plugin) gm.check_pending() diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 946a515b7..d9a14d077 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -517,7 +517,9 @@ class Account(object): # def add_account_plugin(self, plugin): - """ add an account plugin whose hookimpls are called. """ + """ add an account plugin which implements one or more of + the :class:`deltachat.hookspec.PerAccount` specs. + """ self._pm.register(plugin) self._pm.check_pending() return plugin diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index da99a1660..15bf7eb94 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -15,7 +15,7 @@ global_hookimpl = pluggy.HookimplMarker(_global_name) class PerAccount: """ per-Account-instance hook specifications. - Account hook implementations need to be registered with an Account instance. + If you write a plugin you need to implement one of the following hooks. """ @classmethod def _make_plugin_manager(cls): @@ -27,8 +27,8 @@ class PerAccount: def process_ffi_event(self, ffi_event): """ process a CFFI low level events for a given account. - ffi_event has "name", "data1", "data2" attributes according - to https://c.delta.chat/group__DC__EVENT.html + ffi_event has "name", "data1", "data2" values as specified + with `DC_EVENT_* `_. """ @account_hookspec @@ -47,6 +47,10 @@ class PerAccount: def process_message_delivered(self, message): """ Called when an outgoing message has been delivered to SMTP. """ + @account_hookspec + def member_added(self, chat, contact): + """ Called for each contact added to a chat. """ + class Global: """ global hook specifications using a per-process singleton plugin manager instance. diff --git a/python/tests/conftest.py b/python/tests/conftest.py index ea545012d..5e0129cab 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -9,6 +9,7 @@ from deltachat import Account from deltachat.tracker import ConfigureTracker from deltachat import const from deltachat.capi import lib +from deltachat.hookspec import PerAccount from deltachat.eventlogger import FFIEventLogger from _pytest.monkeypatch import MonkeyPatch from ffi_event import FFIEventTracker @@ -290,6 +291,18 @@ def lp(): return Printer() +@pytest.fixture +def make_plugin_recorder(): + def make_plugin_recorder(account): + class HookImpl: + def __init__(self): + self.calls_member_added = [] + + @account_hookimpl + def member_added(self, chat, member): + self.calls_member_added.append(dict(chat=chat, member=member)) + + def wait_configuration_progress(account, min_target, max_target=1001): min_target = min(min_target, max_target) while 1: diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 1b16238ce..c1a665cc4 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -167,6 +167,17 @@ class TestOfflineChat: else: pytest.fail("could not find chat") + def test_add_member_event(self, ac1): + contact1 = ac1.create_contact("some1@hello.com", name="some1") + contact2 = ac1.create_contact("some2@hello.com", name="some2") + chat = ac1.create_group_chat(name="title1") + + with make_plugin_recorder(ac1) as rec: + chat.add_contact(contact2) + kwargs = rec.get_first("member_added") + assert kwargs["chat"] == chat + assert kwargs["member"] == contact2 + def test_group_chat_creation(self, ac1): contact1 = ac1.create_contact("some1@hello.com", name="some1") contact2 = ac1.create_contact("some2@hello.com", name="some2") From 57141e478c375e0913d11d27be02c9b032921925 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 26 Feb 2020 15:15:45 +0100 Subject: [PATCH 024/156] also add a changelog for plugin things --- python/CHANGELOG | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/CHANGELOG b/python/CHANGELOG index 7e6038376..9b8ec1de8 100644 --- a/python/CHANGELOG +++ b/python/CHANGELOG @@ -1,3 +1,11 @@ +0.900.0 (DRAFT) +--------------- + +- refactored internals to use plugin-approach + +- introduced PerAccount and Global hooks that plugins can implement + + 0.800.0 ------- From 84f17b7539e3637decadeb50f0517a645cce2dd0 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 26 Feb 2020 16:51:08 +0100 Subject: [PATCH 025/156] emit "DC_EVENT_MEMBER_ADDED" and python plugin event "member_added" for securejoin or non-securejoin additions of a contact to a chat. also fixup some docs --- deltachat-ffi/deltachat.h | 10 +++++++++- deltachat-ffi/src/lib.rs | 4 ++++ python/CHANGELOG | 2 ++ python/doc/plugins.rst | 3 +++ python/src/deltachat/__init__.py | 5 +++++ python/src/deltachat/account.py | 11 +++++++++++ python/src/deltachat/const.py | 1 + python/tests/conftest.py | 18 +++++++++++++++--- python/tests/test_account.py | 15 ++++++++------- src/chat.rs | 4 ++++ src/events.rs | 6 ++++++ src/securejoin.rs | 2 +- 12 files changed, 69 insertions(+), 12 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 632600d6d..7e2d7c5b9 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4505,7 +4505,7 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); /** - * This event is sent out to the inviter when a joiner successfully joined a group. + * (DEPRECATED) * * @param data1 (int) chat_id * @param data2 (int) contact_id @@ -4513,6 +4513,14 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); */ #define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062 +/** + * This event is sent for each member that gets added to a (verified or unverified) chat. + * + * @param data1 (int) chat_id + * @param data2 (int) contact_id + * @return 0 + */ +#define DC_EVENT_MEMBER_ADDED 2063 /** * @} diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 74773f18e..10cdfaccf 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -199,6 +199,10 @@ impl ContextWrapper { Event::SecurejoinMemberAdded { chat_id, contact_id, + } + | Event::MemberAdded { + chat_id, + contact_id, } => { ffi_cb( self, diff --git a/python/CHANGELOG b/python/CHANGELOG index 9b8ec1de8..82b6402cb 100644 --- a/python/CHANGELOG +++ b/python/CHANGELOG @@ -5,6 +5,8 @@ - introduced PerAccount and Global hooks that plugins can implement +- introduced `member_added()` plugin event. + 0.800.0 ------- diff --git a/python/doc/plugins.rst b/python/doc/plugins.rst index 281669991..6b37d19ea 100644 --- a/python/doc/plugins.rst +++ b/python/doc/plugins.rst @@ -7,6 +7,9 @@ for managing global and per-account plugin registration, and performing hook calls. +Registering a plugin +-------------------- + .. autoclass:: deltachat.register_global_plugin .. autoclass:: deltachat.account.Account.add_account_plugin diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 43c100700..4630c7ea6 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -86,4 +86,9 @@ def register_global_plugin(plugin): gm.check_pending() +def unregister_global_plugin(plugin): + gm = hookspec.Global._get_plugin_manager() + gm.unregister(plugin) + + register_global_plugin(eventlogger) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index d9a14d077..70f2eac8e 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -76,6 +76,10 @@ class Account(object): elif name == "DC_EVENT_MSG_DELIVERED": msg = self.get_message_by_id(ffi_event.data2) self._pm.hook.process_message_delivered(message=msg) + elif name == "DC_EVENT_MEMBER_ADDED": + chat = self.get_chat_by_id(ffi_event.data1) + contact = self.get_contact_by_id(ffi_event.data2) + self._pm.hook.member_added(chat=chat, contact=contact) # def __del__(self): # self.shutdown() @@ -342,6 +346,13 @@ class Account(object): """ return Message.from_db(self, msg_id) + def get_contact_by_id(self, contact_id): + """ return Contact instance or None. + :param contact_id: integer id of this contact. + :returns: None or :class:`deltachat.contact.Contact` instance. + """ + return Contact(self._dc_context, contact_id) + def get_chat_by_id(self, chat_id): """ return Chat instance. :param chat_id: integer id of this chat. diff --git a/python/src/deltachat/const.py b/python/src/deltachat/const.py index 7135b4905..09229ff75 100644 --- a/python/src/deltachat/const.py +++ b/python/src/deltachat/const.py @@ -99,6 +99,7 @@ DC_EVENT_IMEX_FILE_WRITTEN = 2052 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061 DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062 +DC_EVENT_MEMBER_ADDED = 2063 DC_EVENT_FILE_COPIED = 2055 DC_EVENT_IS_OFFLINE = 2081 DC_EVENT_GET_STRING = 2091 diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 5e0129cab..eb83d76b0 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -4,12 +4,13 @@ import sys import py import pytest import requests +from contextlib import contextmanager import time from deltachat import Account from deltachat.tracker import ConfigureTracker from deltachat import const from deltachat.capi import lib -from deltachat.hookspec import PerAccount +from deltachat.hookspec import account_hookimpl from deltachat.eventlogger import FFIEventLogger from _pytest.monkeypatch import MonkeyPatch from ffi_event import FFIEventTracker @@ -293,14 +294,25 @@ def lp(): @pytest.fixture def make_plugin_recorder(): + @contextmanager def make_plugin_recorder(account): class HookImpl: def __init__(self): self.calls_member_added = [] @account_hookimpl - def member_added(self, chat, member): - self.calls_member_added.append(dict(chat=chat, member=member)) + def member_added(self, chat, contact): + self.calls_member_added.append(dict(chat=chat, contact=contact)) + + def get_first(self, name): + val = getattr(self, "calls_" + name, None) + if val is not None: + return val.pop(0) + + with account.temp_plugin(HookImpl()) as plugin: + yield plugin + + return make_plugin_recorder def wait_configuration_progress(account, min_target, max_target=1001): diff --git a/python/tests/test_account.py b/python/tests/test_account.py index c1a665cc4..c625b94c9 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -167,16 +167,17 @@ class TestOfflineChat: else: pytest.fail("could not find chat") - def test_add_member_event(self, ac1): - contact1 = ac1.create_contact("some1@hello.com", name="some1") - contact2 = ac1.create_contact("some2@hello.com", name="some2") + def test_add_member_event(self, ac1, make_plugin_recorder): chat = ac1.create_group_chat(name="title1") + # promote the chat + chat.send_text("hello") + contact1 = ac1.create_contact("some1@hello.com", name="some1") with make_plugin_recorder(ac1) as rec: - chat.add_contact(contact2) + chat.add_contact(contact1) kwargs = rec.get_first("member_added") assert kwargs["chat"] == chat - assert kwargs["member"] == contact2 + assert kwargs["contact"] == contact1 def test_group_chat_creation(self, ac1): contact1 = ac1.create_contact("some1@hello.com", name="some1") @@ -1129,7 +1130,7 @@ class TestOnlineAccount: ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") wait_securejoin_inviter_progress(ac1, 1000) - ac1._evtracker.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED") + ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED") def test_qr_verified_group_and_chatting(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -1141,7 +1142,7 @@ class TestOnlineAccount: chat2 = ac2.qr_join_chat(qr) assert chat2.id >= 10 wait_securejoin_inviter_progress(ac1, 1000) - ac1._evtracker.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED") + ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED") lp.sec("ac2: read member added message") msg = ac2._evtracker.wait_next_incoming_message() diff --git a/src/chat.rs b/src/chat.rs index 1c24635a3..367007279 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1958,6 +1958,10 @@ pub(crate) fn add_contact_to_chat_ex( chat_id, msg_id: msg.id, }); + context.call_cb(Event::MemberAdded { + chat_id, + contact_id: contact.id, + }); } context.call_cb(Event::MsgsChanged { chat_id, diff --git a/src/events.rs b/src/events.rs index 37a0a4bc4..3d1eb140f 100644 --- a/src/events.rs +++ b/src/events.rs @@ -207,4 +207,10 @@ pub enum Event { /// @param data2 (int) contact_id #[strum(props(id = "2062"))] SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 }, + + /// This event is sent for each contact added to a chat. + /// @param data1 (int) chat_id + /// @param data2 (int) contact_id + #[strum(props(id = "2063"))] + MemberAdded { chat_id: ChatId, contact_id: u32 }, } diff --git a/src/securejoin.rs b/src/securejoin.rs index e50953e72..2964aba98 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -750,7 +750,7 @@ pub(crate) fn handle_securejoin_handshake( group: field_grpid.to_string(), } })?; - context.call_cb(Event::SecurejoinMemberAdded { + context.call_cb(Event::MemberAdded { chat_id: group_chat_id, contact_id, }); From 6a6a719ab682f27c1af9634c1d51433245c303a4 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 2 Mar 2020 15:15:22 +0100 Subject: [PATCH 026/156] shift pytest support code into deltachat package so deltabot can make use of the test infrastructure --- python/src/deltachat/eventlogger.py | 58 ++++++ python/src/deltachat/testplugin.py | 288 +++++++++++++++++++++++++++ python/tests/conftest.py | 297 +--------------------------- python/tests/ffi_event.py | 59 ------ python/tests/test_increation.py | 10 +- 5 files changed, 358 insertions(+), 354 deletions(-) create mode 100644 python/src/deltachat/testplugin.py delete mode 100644 python/tests/ffi_event.py diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 363f0359a..3ac47e290 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -1,6 +1,8 @@ import deltachat import threading import time +import re +from queue import Queue, Empty from .hookspec import account_hookimpl, global_hookimpl @@ -70,3 +72,59 @@ class FFIEventLogger: s = "{:2.2f} [{}] {}".format(elapsed, locname, message) with self._loglock: print(s) + + +class FFIEventTracker: + def __init__(self, account, timeout=None): + self.account = account + self._timeout = timeout + self._event_queue = Queue() + + @account_hookimpl + def process_ffi_event(self, ffi_event): + self._event_queue.put(ffi_event) + + def set_timeout(self, timeout): + self._timeout = timeout + + def consume_events(self, check_error=True): + while not self._event_queue.empty(): + self.get(check_error=check_error) + + def get(self, timeout=None, check_error=True): + timeout = timeout if timeout is not None else self._timeout + ev = self._event_queue.get(timeout=timeout) + if check_error and ev.name == "DC_EVENT_ERROR": + raise ValueError(str(ev)) + return ev + + def ensure_event_not_queued(self, event_name_regex): + __tracebackhide__ = True + rex = re.compile("(?:{}).*".format(event_name_regex)) + while 1: + try: + ev = self._event_queue.get(False) + except Empty: + break + else: + assert not rex.match(ev.name), "event found {}".format(ev) + + def get_matching(self, event_name_regex, check_error=True, timeout=None): + self.account.log_line("-- waiting for event with regex: {} --".format(event_name_regex)) + rex = re.compile("(?:{}).*".format(event_name_regex)) + while 1: + ev = self.get(timeout=timeout, check_error=check_error) + if rex.match(ev.name): + return ev + + def get_info_matching(self, regex): + rex = re.compile("(?:{}).*".format(regex)) + while 1: + ev = self.get_matching("DC_EVENT_INFO") + if rex.match(ev.data2): + return ev + + def wait_next_incoming_message(self): + """ wait for and return next incoming message. """ + ev = self.get_matching("DC_EVENT_INCOMING_MSG") + return self.account.get_message_by_id(ev.data2) diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py new file mode 100644 index 000000000..13d43d88a --- /dev/null +++ b/python/src/deltachat/testplugin.py @@ -0,0 +1,288 @@ +from __future__ import print_function +import os +import sys +import pytest +import requests +from contextlib import contextmanager +import time +from . import Account, const +from .tracker import ConfigureTracker +from .capi import lib +from .hookspec import account_hookimpl +from .eventlogger import FFIEventLogger, FFIEventTracker +from _pytest.monkeypatch import MonkeyPatch +import tempfile + + +def pytest_addoption(parser): + parser.addoption( + "--liveconfig", action="store", default=None, + help="a file with >=2 lines where each line " + "contains NAME=VALUE config settings for one account" + ) + parser.addoption( + "--ignored", action="store_true", + help="Also run tests marked with the ignored marker", + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used." + ) + cfg = config.getoption('--liveconfig') + if not cfg: + cfg = os.getenv('DCC_NEW_TMP_EMAIL') + if cfg: + config.option.liveconfig = cfg + + +def pytest_runtest_setup(item): + if (list(item.iter_markers(name="ignored")) + and not item.config.getoption("ignored")): + pytest.skip("Ignored tests not requested, use --ignored") + + +def pytest_report_header(config, startdir): + summary = [] + + t = tempfile.mktemp() + m = MonkeyPatch() + try: + m.setattr(sys.stdout, "write", lambda x: len(x)) + ac = Account(t) + info = ac.get_info() + ac.shutdown() + finally: + m.undo() + os.remove(t) + summary.extend(['Deltachat core={} sqlite={}'.format( + info['deltachat_core_version'], + info['sqlite_version'], + )]) + + cfg = config.option.liveconfig + if cfg: + if "#" in cfg: + url, token = cfg.split("#", 1) + summary.append('Liveconfig provider: {}#'.format(url)) + else: + summary.append('Liveconfig file: {}'.format(cfg)) + return summary + + +class SessionLiveConfigFromFile: + def __init__(self, fn): + self.fn = fn + self.configlist = [] + for line in open(fn): + if line.strip() and not line.strip().startswith('#'): + d = {} + for part in line.split(): + name, value = part.split("=") + d[name] = value + self.configlist.append(d) + + def get(self, index): + return self.configlist[index] + + def exists(self): + return bool(self.configlist) + + +class SessionLiveConfigFromURL: + def __init__(self, url): + self.configlist = [] + self.url = url + + def get(self, index): + try: + return self.configlist[index] + except IndexError: + assert index == len(self.configlist), index + res = requests.post(self.url) + if res.status_code != 200: + pytest.skip("creating newtmpuser failed {!r}".format(res)) + d = res.json() + config = dict(addr=d["email"], mail_pw=d["password"]) + self.configlist.append(config) + return config + + def exists(self): + return bool(self.configlist) + + +@pytest.fixture(scope="session") +def session_liveconfig(request): + liveconfig_opt = request.config.option.liveconfig + if liveconfig_opt: + if liveconfig_opt.startswith("http"): + return SessionLiveConfigFromURL(liveconfig_opt) + else: + return SessionLiveConfigFromFile(liveconfig_opt) + + +@pytest.fixture +def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): + + class AccountMaker: + def __init__(self): + self.live_count = 0 + self.offline_count = 0 + self._finalizers = [] + self.init_time = time.time() + self._generated_keys = ["alice", "bob", "charlie", + "dom", "elena", "fiona"] + + def finalize(self): + while self._finalizers: + fin = self._finalizers.pop() + fin() + + def make_account(self, path, logid): + ac = Account(path) + ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac)) + ac._configtracker = ac.add_account_plugin(ConfigureTracker()) + ac.add_account_plugin(FFIEventLogger(ac, logid=logid)) + self._finalizers.append(ac.shutdown) + return ac + + def get_unconfigured_account(self): + self.offline_count += 1 + tmpdb = tmpdir.join("offlinedb%d" % self.offline_count) + ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count)) + ac._evtracker.init_time = self.init_time + ac._evtracker.set_timeout(2) + return ac + + def _preconfigure_key(self, account, addr): + # Only set a key if we haven't used it yet for another account. + if self._generated_keys: + keyname = self._generated_keys.pop(0) + fname_pub = "key/{name}-public.asc".format(name=keyname) + fname_sec = "key/{name}-secret.asc".format(name=keyname) + account._preconfigure_keypair(addr, + datadir.join(fname_pub).read(), + datadir.join(fname_sec).read()) + + def get_configured_offline_account(self): + ac = self.get_unconfigured_account() + + # do a pseudo-configured account + addr = "addr{}@offline.org".format(self.offline_count) + ac.set_config("addr", addr) + self._preconfigure_key(ac, addr) + lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii")) + ac.set_config("mail_pw", "123") + lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123") + lib.dc_set_config(ac._dc_context, b"configured", b"1") + return ac + + def get_online_config(self, pre_generated_key=True): + if not session_liveconfig: + pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig") + configdict = session_liveconfig.get(self.live_count) + self.live_count += 1 + if "e2ee_enabled" not in configdict: + configdict["e2ee_enabled"] = "1" + + # Enable strict certificate checks for online accounts + configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT) + configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT) + + tmpdb = tmpdir.join("livedb%d" % self.live_count) + ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) + if pre_generated_key: + self._preconfigure_key(ac, configdict['addr']) + ac._evtracker.init_time = self.init_time + ac._evtracker.set_timeout(30) + return ac, dict(configdict) + + def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False, + pre_generated_key=True, config={}): + ac, configdict = self.get_online_config( + pre_generated_key=pre_generated_key) + configdict.update(config) + configdict["mvbox_watch"] = str(int(mvbox)) + configdict["mvbox_move"] = str(int(move)) + configdict["sentbox_watch"] = str(int(sentbox)) + ac.update_config(configdict) + ac.start() + return ac + + def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False): + ac1 = self.get_online_configuring_account( + pre_generated_key=pre_generated_key, mvbox=mvbox, move=move) + ac1._configtracker.wait_imap_connected() + ac1._configtracker.wait_smtp_connected() + ac1._configtracker.wait_finish() + return ac1 + + def get_two_online_accounts(self, move=False): + ac1 = self.get_online_configuring_account(move=True) + ac2 = self.get_online_configuring_account() + ac1._configtracker.wait_finish() + ac2._configtracker.wait_finish() + return ac1, ac2 + + def clone_online_account(self, account, pre_generated_key=True): + self.live_count += 1 + tmpdb = tmpdir.join("livedb%d" % self.live_count) + ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) + if pre_generated_key: + self._preconfigure_key(ac, account.get_config("addr")) + ac._evtracker.init_time = self.init_time + ac._evtracker.set_timeout(30) + ac.update_config(dict( + addr=account.get_config("addr"), + mail_pw=account.get_config("mail_pw"), + mvbox_watch=account.get_config("mvbox_watch"), + mvbox_move=account.get_config("mvbox_move"), + sentbox_watch=account.get_config("sentbox_watch"), + )) + ac.start() + return ac + + am = AccountMaker() + request.addfinalizer(am.finalize) + return am + + +@pytest.fixture +def tmp_db_path(tmpdir): + return tmpdir.join("test.db").strpath + + +@pytest.fixture +def lp(): + class Printer: + def sec(self, msg): + print() + print("=" * 10, msg, "=" * 10) + + def step(self, msg): + print("-" * 5, "step " + msg, "-" * 5) + return Printer() + + +@pytest.fixture +def make_plugin_recorder(): + @contextmanager + def make_plugin_recorder(account): + class HookImpl: + def __init__(self): + self.calls_member_added = [] + + @account_hookimpl + def member_added(self, chat, contact): + self.calls_member_added.append(dict(chat=chat, contact=contact)) + + def get_first(self, name): + val = getattr(self, "calls_" + name, None) + if val is not None: + return val.pop(0) + + with account.temp_plugin(HookImpl()) as plugin: + yield plugin + + return make_plugin_recorder diff --git a/python/tests/conftest.py b/python/tests/conftest.py index eb83d76b0..20a0fdd17 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -1,77 +1,11 @@ from __future__ import print_function + import os -import sys -import py import pytest -import requests -from contextlib import contextmanager -import time -from deltachat import Account -from deltachat.tracker import ConfigureTracker -from deltachat import const -from deltachat.capi import lib -from deltachat.hookspec import account_hookimpl -from deltachat.eventlogger import FFIEventLogger -from _pytest.monkeypatch import MonkeyPatch -from ffi_event import FFIEventTracker -import tempfile +import py -def pytest_addoption(parser): - parser.addoption( - "--liveconfig", action="store", default=None, - help="a file with >=2 lines where each line " - "contains NAME=VALUE config settings for one account" - ) - parser.addoption( - "--ignored", action="store_true", - help="Also run tests marked with the ignored marker", - ) - - -def pytest_configure(config): - config.addinivalue_line( - "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used." - ) - cfg = config.getoption('--liveconfig') - if not cfg: - cfg = os.getenv('DCC_NEW_TMP_EMAIL') - if cfg: - config.option.liveconfig = cfg - - -def pytest_runtest_setup(item): - if (list(item.iter_markers(name="ignored")) - and not item.config.getoption("ignored")): - pytest.skip("Ignored tests not requested, use --ignored") - - -def pytest_report_header(config, startdir): - summary = [] - - t = tempfile.mktemp() - m = MonkeyPatch() - try: - m.setattr(sys.stdout, "write", lambda x: len(x)) - ac = Account(t) - info = ac.get_info() - ac.shutdown() - finally: - m.undo() - os.remove(t) - summary.extend(['Deltachat core={} sqlite={}'.format( - info['deltachat_core_version'], - info['sqlite_version'], - )]) - - cfg = config.option.liveconfig - if cfg: - if "#" in cfg: - url, token = cfg.split("#", 1) - summary.append('Liveconfig provider: {}#'.format(url)) - else: - summary.append('Liveconfig file: {}'.format(cfg)) - return summary +from deltachat.testplugin import * # noqa @pytest.fixture(scope="session") @@ -87,57 +21,6 @@ def data(): return Data() -class SessionLiveConfigFromFile: - def __init__(self, fn): - self.fn = fn - self.configlist = [] - for line in open(fn): - if line.strip() and not line.strip().startswith('#'): - d = {} - for part in line.split(): - name, value = part.split("=") - d[name] = value - self.configlist.append(d) - - def get(self, index): - return self.configlist[index] - - def exists(self): - return bool(self.configlist) - - -class SessionLiveConfigFromURL: - def __init__(self, url): - self.configlist = [] - self.url = url - - def get(self, index): - try: - return self.configlist[index] - except IndexError: - assert index == len(self.configlist), index - res = requests.post(self.url) - if res.status_code != 200: - pytest.skip("creating newtmpuser failed {!r}".format(res)) - d = res.json() - config = dict(addr=d["email"], mail_pw=d["password"]) - self.configlist.append(config) - return config - - def exists(self): - return bool(self.configlist) - - -@pytest.fixture(scope="session") -def session_liveconfig(request): - liveconfig_opt = request.config.option.liveconfig - if liveconfig_opt: - if liveconfig_opt.startswith("http"): - return SessionLiveConfigFromURL(liveconfig_opt) - else: - return SessionLiveConfigFromFile(liveconfig_opt) - - @pytest.fixture(scope='session') def datadir(): """The py.path.local object of the test-data/ directory.""" @@ -149,172 +32,6 @@ def datadir(): pytest.skip('test-data directory not found') -@pytest.fixture -def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): - - class AccountMaker: - def __init__(self): - self.live_count = 0 - self.offline_count = 0 - self._finalizers = [] - self.init_time = time.time() - self._generated_keys = ["alice", "bob", "charlie", - "dom", "elena", "fiona"] - - def finalize(self): - while self._finalizers: - fin = self._finalizers.pop() - fin() - - def make_account(self, path, logid): - ac = Account(path) - ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac)) - ac._configtracker = ac.add_account_plugin(ConfigureTracker()) - ac.add_account_plugin(FFIEventLogger(ac, logid=logid)) - self._finalizers.append(ac.shutdown) - return ac - - def get_unconfigured_account(self): - self.offline_count += 1 - tmpdb = tmpdir.join("offlinedb%d" % self.offline_count) - ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count)) - ac._evtracker.init_time = self.init_time - ac._evtracker.set_timeout(2) - return ac - - def _preconfigure_key(self, account, addr): - # Only set a key if we haven't used it yet for another account. - if self._generated_keys: - keyname = self._generated_keys.pop(0) - fname_pub = "key/{name}-public.asc".format(name=keyname) - fname_sec = "key/{name}-secret.asc".format(name=keyname) - account._preconfigure_keypair(addr, - datadir.join(fname_pub).read(), - datadir.join(fname_sec).read()) - - def get_configured_offline_account(self): - ac = self.get_unconfigured_account() - - # do a pseudo-configured account - addr = "addr{}@offline.org".format(self.offline_count) - ac.set_config("addr", addr) - self._preconfigure_key(ac, addr) - lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii")) - ac.set_config("mail_pw", "123") - lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123") - lib.dc_set_config(ac._dc_context, b"configured", b"1") - return ac - - def get_online_config(self, pre_generated_key=True): - if not session_liveconfig: - pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig") - configdict = session_liveconfig.get(self.live_count) - self.live_count += 1 - if "e2ee_enabled" not in configdict: - configdict["e2ee_enabled"] = "1" - - # Enable strict certificate checks for online accounts - configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT) - configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT) - - tmpdb = tmpdir.join("livedb%d" % self.live_count) - ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) - if pre_generated_key: - self._preconfigure_key(ac, configdict['addr']) - ac._evtracker.init_time = self.init_time - ac._evtracker.set_timeout(30) - return ac, dict(configdict) - - def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False, - pre_generated_key=True, config={}): - ac, configdict = self.get_online_config( - pre_generated_key=pre_generated_key) - configdict.update(config) - configdict["mvbox_watch"] = str(int(mvbox)) - configdict["mvbox_move"] = str(int(move)) - configdict["sentbox_watch"] = str(int(sentbox)) - ac.update_config(configdict) - ac.start() - return ac - - def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False): - ac1 = self.get_online_configuring_account( - pre_generated_key=pre_generated_key, mvbox=mvbox, move=move) - ac1._configtracker.wait_imap_connected() - ac1._configtracker.wait_smtp_connected() - ac1._configtracker.wait_finish() - return ac1 - - def get_two_online_accounts(self, move=False): - ac1 = self.get_online_configuring_account(move=True) - ac2 = self.get_online_configuring_account() - ac1._configtracker.wait_finish() - ac2._configtracker.wait_finish() - return ac1, ac2 - - def clone_online_account(self, account, pre_generated_key=True): - self.live_count += 1 - tmpdb = tmpdir.join("livedb%d" % self.live_count) - ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) - if pre_generated_key: - self._preconfigure_key(ac, account.get_config("addr")) - ac._evtracker.init_time = self.init_time - ac._evtracker.set_timeout(30) - ac.update_config(dict( - addr=account.get_config("addr"), - mail_pw=account.get_config("mail_pw"), - mvbox_watch=account.get_config("mvbox_watch"), - mvbox_move=account.get_config("mvbox_move"), - sentbox_watch=account.get_config("sentbox_watch"), - )) - ac.start() - return ac - - am = AccountMaker() - request.addfinalizer(am.finalize) - return am - - -@pytest.fixture -def tmp_db_path(tmpdir): - return tmpdir.join("test.db").strpath - - -@pytest.fixture -def lp(): - class Printer: - def sec(self, msg): - print() - print("=" * 10, msg, "=" * 10) - - def step(self, msg): - print("-" * 5, "step " + msg, "-" * 5) - return Printer() - - -@pytest.fixture -def make_plugin_recorder(): - @contextmanager - def make_plugin_recorder(account): - class HookImpl: - def __init__(self): - self.calls_member_added = [] - - @account_hookimpl - def member_added(self, chat, contact): - self.calls_member_added.append(dict(chat=chat, contact=contact)) - - def get_first(self, name): - val = getattr(self, "calls_" + name, None) - if val is not None: - return val.pop(0) - - with account.temp_plugin(HookImpl()) as plugin: - yield plugin - - return make_plugin_recorder - - def wait_configuration_progress(account, min_target, max_target=1001): min_target = min(min_target, max_target) while 1: @@ -330,11 +47,3 @@ def wait_securejoin_inviter_progress(account, target): if event.data2 >= target: print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account) break - - -def wait_msgs_changed(account, chat_id, msg_id=None): - ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ev.data1 == chat_id - if msg_id is not None: - assert ev.data2 == msg_id - return ev.data2 diff --git a/python/tests/ffi_event.py b/python/tests/ffi_event.py deleted file mode 100644 index 8be2df8f3..000000000 --- a/python/tests/ffi_event.py +++ /dev/null @@ -1,59 +0,0 @@ -import re -from queue import Queue, Empty -from deltachat.hookspec import account_hookimpl - - -class FFIEventTracker: - def __init__(self, account, timeout=None): - self.account = account - self._timeout = timeout - self._event_queue = Queue() - - @account_hookimpl - def process_ffi_event(self, ffi_event): - self._event_queue.put(ffi_event) - - def set_timeout(self, timeout): - self._timeout = timeout - - def consume_events(self, check_error=True): - while not self._event_queue.empty(): - self.get(check_error=check_error) - - def get(self, timeout=None, check_error=True): - timeout = timeout if timeout is not None else self._timeout - ev = self._event_queue.get(timeout=timeout) - if check_error and ev.name == "DC_EVENT_ERROR": - raise ValueError(str(ev)) - return ev - - def ensure_event_not_queued(self, event_name_regex): - __tracebackhide__ = True - rex = re.compile("(?:{}).*".format(event_name_regex)) - while 1: - try: - ev = self._event_queue.get(False) - except Empty: - break - else: - assert not rex.match(ev.name), "event found {}".format(ev) - - def get_matching(self, event_name_regex, check_error=True, timeout=None): - self.account.log_line("-- waiting for event with regex: {} --".format(event_name_regex)) - rex = re.compile("(?:{}).*".format(event_name_regex)) - while 1: - ev = self.get(timeout=timeout, check_error=check_error) - if rex.match(ev.name): - return ev - - def get_info_matching(self, regex): - rex = re.compile("(?:{}).*".format(regex)) - while 1: - ev = self.get_matching("DC_EVENT_INFO") - if rex.match(ev.data2): - return ev - - def wait_next_incoming_message(self): - """ wait for and return next incoming message. """ - ev = self.get_matching("DC_EVENT_INCOMING_MSG") - return self.account.get_message_by_id(ev.data2) diff --git a/python/tests/test_increation.py b/python/tests/test_increation.py index d2f16285f..6fb96e1fe 100644 --- a/python/tests/test_increation.py +++ b/python/tests/test_increation.py @@ -6,10 +6,18 @@ import shutil import pytest from filecmp import cmp -from conftest import wait_configuration_progress, wait_msgs_changed +from conftest import wait_configuration_progress from deltachat import const +def wait_msgs_changed(account, chat_id, msg_id=None): + ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") + assert ev.data1 == chat_id + if msg_id is not None: + assert ev.data2 == msg_id + return ev.data2 + + class TestOnlineInCreation: def test_increation_not_blobdir(self, tmpdir, acfactory, lp): ac1 = acfactory.get_online_configuring_account() From a665d6de59189a9c6c7b2bd2af2047900dd74d38 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 3 Mar 2020 06:00:20 +0100 Subject: [PATCH 027/156] add chat.is_group() API to help callers avoid having to check with constants deprecate get_type() --- python/src/deltachat/chat.py | 12 +++++++++++- python/tests/test_account.py | 5 +++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index 09883f452..173ec8e84 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -51,6 +51,16 @@ class Chat(object): # ------ chat status/metadata API ------------------------------ + def is_group(self): + """ return true if this chat is a group chat. + + :returns: True if chat is a group-chat, false if it's a contact 1:1 chat. + """ + return lib.dc_chat_get_type(self._dc_chat) in ( + const.DC_CHAT_TYPE_GROUP, + const.DC_CHAT_TYPE_VERIFIED_GROUP + ) + def is_deaddrop(self): """ return true if this chat is a deaddrop chat. @@ -129,7 +139,7 @@ class Chat(object): return bool(lib.dc_chat_get_remaining_mute_duration(self.id)) def get_type(self): - """ return type of this chat. + """ (deprecated) return type of this chat. :returns: one of const.DC_CHAT_TYPE_* """ diff --git a/python/tests/test_account.py b/python/tests/test_account.py index c625b94c9..5c5fc8338 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -147,6 +147,9 @@ class TestOfflineChat: str(chat1) repr(chat1) + def test_is_group(self, chat1): + assert not chat1.is_group() + def test_chat_by_id(self, chat1): chat2 = chat1.account.get_chat_by_id(chat1.id) assert chat2 == chat1 @@ -169,6 +172,7 @@ class TestOfflineChat: def test_add_member_event(self, ac1, make_plugin_recorder): chat = ac1.create_group_chat(name="title1") + assert chat.is_group() # promote the chat chat.send_text("hello") contact1 = ac1.create_contact("some1@hello.com", name="some1") @@ -229,6 +233,7 @@ class TestOfflineChat: def test_group_chat_qr(self, acfactory, ac1, verified): ac2 = acfactory.get_configured_offline_account() chat = ac1.create_group_chat(name="title1", verified=verified) + assert chat.is_group() qr = chat.get_join_qr() assert ac2.check_qr(qr).is_ask_verifygroup From a1379f61daac283a157adb11a28785b564112d31 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 3 Mar 2020 06:29:05 +0100 Subject: [PATCH 028/156] fix up docs --- python/doc/api.rst | 4 ---- python/doc/plugins.rst | 12 +++++++++--- python/src/deltachat/__init__.py | 2 +- python/src/deltachat/account.py | 3 ++- python/src/deltachat/hookspec.py | 3 ++- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/python/doc/api.rst b/python/doc/api.rst index d82364b75..049bc2928 100644 --- a/python/doc/api.rst +++ b/python/doc/api.rst @@ -2,10 +2,6 @@ high level API reference ======================== -.. note:: - - This API is work in progress and may change in versions prior to 1.0. - - :class:`deltachat.account.Account` (your main entry point, creates the other classes) - :class:`deltachat.contact.Contact` diff --git a/python/doc/plugins.rst b/python/doc/plugins.rst index 6b37d19ea..1c4a5a619 100644 --- a/python/doc/plugins.rst +++ b/python/doc/plugins.rst @@ -4,15 +4,21 @@ Implementing Plugin Hooks The Delta Chat Python bindings use `pluggy `_ for managing global and per-account plugin registration, and performing -hook calls. +hook calls. There are two kinds of plugins: + +- Global plugins that are active for all accounts; they can implement + hooks at account-creation and account-shutdown time. + +- Account plugins that are only active during the lifetime of a + single Account instance. Registering a plugin -------------------- -.. autoclass:: deltachat.register_global_plugin +.. autofunction:: deltachat.register_global_plugin -.. autoclass:: deltachat.account.Account.add_account_plugin +.. automethod:: deltachat.account.Account.add_account_plugin Per-Account Hook specifications diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 4630c7ea6..ac27a4209 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -79,7 +79,7 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}): def register_global_plugin(plugin): """ Register a global plugin which implements one or more - of the :class:`deltachat.hookspec.Global` specs. + of the :class:`deltachat.hookspec.Global` hooks. """ gm = hookspec.Global._get_plugin_manager() gm.register(plugin) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 70f2eac8e..eaf8ee615 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -160,6 +160,7 @@ class Account(object): def update_config(self, kwargs): """ update config values. + :param kwargs: name=value config settings for this account. values need to be unicode. :returns: None @@ -529,7 +530,7 @@ class Account(object): def add_account_plugin(self, plugin): """ add an account plugin which implements one or more of - the :class:`deltachat.hookspec.PerAccount` specs. + the :class:`deltachat.hookspec.PerAccount` hooks. """ self._pm.register(plugin) self._pm.check_pending() diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 15bf7eb94..d8868f150 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -53,7 +53,8 @@ class PerAccount: class Global: - """ global hook specifications using a per-process singleton plugin manager instance. + """ global hook specifications using a per-process singleton + plugin manager instance. """ _plugin_manager = None From 91cdc764145eb43f9842f53563f57f8a2e51d60c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 3 Mar 2020 13:11:25 +0100 Subject: [PATCH 029/156] refactor docs and ffi/high level event handling to pass all tests again --- python/doc/capi.rst | 7 --- python/doc/examples.rst | 25 +++++++-- python/doc/index.rst | 1 + python/examples/echo_and_quit.py | 37 +++++++++++++ python/src/deltachat/account.py | 88 ++++++++++++++++++++++-------- python/src/deltachat/hookspec.py | 7 ++- python/src/deltachat/iothreads.py | 19 ++++++- python/src/deltachat/testplugin.py | 31 +---------- python/tests/test_account.py | 18 ++++-- 9 files changed, 161 insertions(+), 72 deletions(-) delete mode 100644 python/doc/capi.rst create mode 100644 python/examples/echo_and_quit.py diff --git a/python/doc/capi.rst b/python/doc/capi.rst deleted file mode 100644 index 67cbf0884..000000000 --- a/python/doc/capi.rst +++ /dev/null @@ -1,7 +0,0 @@ - -C deltachat interface -===================== - -See :doc:`lapi` for accessing many of the below functions -through the ``deltachat.capi.lib`` namespace. - diff --git a/python/doc/examples.rst b/python/doc/examples.rst index 9a489f783..9305fae3c 100644 --- a/python/doc/examples.rst +++ b/python/doc/examples.rst @@ -4,8 +4,8 @@ examples ======== -Playing around on the commandline ----------------------------------- +Sending a Chat message from the command line +--------------------------------------------- Once you have :doc:`installed deltachat bindings ` you can start playing from the python interpreter commandline. @@ -14,10 +14,10 @@ For example you can type ``python`` and then:: # instantiate and configure deltachat account import deltachat ac = deltachat.Account("/tmp/db") - ac.set_config("addr", "test2@hq5.merlinux.eu") + ac.set_config("addr", "address@example.org") ac.set_config("mail_pwd", "some password") - # start the IO threads and perform configuration + # start IO threads and perform configuration ac.start() # create a contact and send a message @@ -25,6 +25,23 @@ For example you can type ``python`` and then:: chat = ac.create_chat_by_contact(contact) chat.send_text("hi from the python interpreter command line") + # shutdown IO threads + ac.shutdown() + + +Checkout our :doc:`api` for the various high-level things you can do +to send/receive messages, create contacts and chats. + + +Receiving a Chat message from the command line +---------------------------------------------- + +Instantiate an account and register a plugin to process +incoming messages: + +.. include:: ../examples/echo_and_quit.py + :literal: + Checkout our :doc:`api` for the various high-level things you can do to send/receive messages, create contacts and chats. diff --git a/python/doc/index.rst b/python/doc/index.rst index fb93a56d4..b5643eee4 100644 --- a/python/doc/index.rst +++ b/python/doc/index.rst @@ -29,6 +29,7 @@ getting started changelog api lapi + plugins .. Indices and tables diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py new file mode 100644 index 000000000..d37ae2187 --- /dev/null +++ b/python/examples/echo_and_quit.py @@ -0,0 +1,37 @@ + +# instantiate and configure deltachat account +import deltachat +ac = deltachat.Account("/tmp/db") + +# to see low-level events in the console uncomment the following line +# ac.add_account_plugin(deltachat.eventlogger.FFIEventLogger(ac, "")) + +if not ac.is_configured(): + ac.set_config("addr", "tmpy.94mtm@testrun.org") + ac.set_config("mail_pw", "5CbD6VnjD/li") + ac.set_config("mvbox_watch", "0") + ac.set_config("sentbox_watch", "0") + +class MyPlugin: + @deltachat.hookspec.account_hookimpl + def process_incoming_message(self, message): + print("process_incoming message", message) + if message.text.strip() == "/quit": + print("shutting down") + ac.shutdown() + else: + ch = ac.create_chat_by_contact(message.get_sender_contact()) + ch.send_text("echoing {}".format(message.text)) + + @deltachat.hookspec.account_hookimpl + def process_message_delivered(self, message): + print("process_message_delivered", message) + +ac.add_account_plugin(MyPlugin()) + +# start IO threads and perform configuration +ac.start() + +print("waiting for /quit or other message on {}".format(ac.get_config("addr"))) + +ac.wait_shutdown() diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index eaf8ee615..4a41e0934 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -15,6 +15,7 @@ from .message import Message from .contact import Contact from .tracker import ImexTracker from . import hookspec, iothreads +from queue import Queue class MissingCredentials(ValueError): @@ -48,6 +49,9 @@ class Account(object): hook.account_init(account=self, db_path=db_path) self._threads = iothreads.IOThreads(self) + self._hook_event_queue = Queue() + self._in_use_iter_events = False + self._shutdown_event = Event() # open database if hasattr(db_path, "encode"): @@ -56,30 +60,13 @@ class Account(object): raise ValueError("Could not dc_open: {}".format(db_path)) self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) - self._shutdown_event = Event() @hookspec.account_hookimpl def process_ffi_event(self, ffi_event): - name = ffi_event.name - if name == "DC_EVENT_CONFIGURE_PROGRESS": - data1 = ffi_event.data1 - if data1 == 0 or data1 == 1000: - success = data1 == 1000 - self._pm.hook.configure_completed(success=success) - elif name == "DC_EVENT_INCOMING_MSG": - msg = self.get_message_by_id(ffi_event.data2) - self._pm.hook.process_incoming_message(message=msg) - elif name == "DC_EVENT_MSGS_CHANGED": - if ffi_event.data2 != 0: - msg = self.get_message_by_id(ffi_event.data2) - self._pm.hook.process_incoming_message(message=msg) - elif name == "DC_EVENT_MSG_DELIVERED": - msg = self.get_message_by_id(ffi_event.data2) - self._pm.hook.process_message_delivered(message=msg) - elif name == "DC_EVENT_MEMBER_ADDED": - chat = self.get_chat_by_id(ffi_event.data1) - contact = self.get_contact_by_id(ffi_event.data2) - self._pm.hook.member_added(chat=chat, contact=contact) + name, kwargs = self._map_ffi_event(ffi_event) + if name is not None: + ev = HookEvent(self, name=name, kwargs=kwargs) + self._hook_event_queue.put(ev) # def __del__(self): # self.shutdown() @@ -547,7 +534,7 @@ class Account(object): """ Stop ongoing securejoin, configuration or other core jobs. """ lib.dc_stop_ongoing_process(self._dc_context) - def start(self): + def start(self, callback_thread=True): """ start this account (activate imap/smtp threads etc.) and return immediately. @@ -565,7 +552,7 @@ class Account(object): if not self.get_config("addr") or not self.get_config("mail_pw"): raise MissingCredentials("addr or mail_pwd not set in config") lib.dc_configure(self._dc_context) - self._threads.start() + self._threads.start(callback_thread=callback_thread) def wait_shutdown(self): """ wait until shutdown of this account has completed. """ @@ -582,12 +569,51 @@ class Account(object): self.stop_ongoing() self._threads.stop(wait=False) lib.dc_close(dc_context) + self._hook_event_queue.put(None) self._threads.stop(wait=wait) # to wait for threads self._dc_context = None atexit.unregister(self.shutdown) + self._shutdown_event.set() hook = hookspec.Global._get_plugin_manager().hook hook.account_after_shutdown(account=self, dc_context=dc_context) - self._shutdown_event.set() + + def iter_events(self, timeout=None): + """ yield hook events until shutdown. + + It is not allowed to call iter_events() from multiple threads. + """ + if self._in_use_iter_events: + raise RuntimeError("can only call iter_events() from one thread") + self._in_use_iter_events = True + while 1: + event = self._hook_event_queue.get(timeout=timeout) + if event is None: + break + yield event + + def _map_ffi_event(self, ffi_event): + name = ffi_event.name + if name == "DC_EVENT_CONFIGURE_PROGRESS": + data1 = ffi_event.data1 + if data1 == 0 or data1 == 1000: + success = data1 == 1000 + return "configure_completed", dict(success=success) + elif name == "DC_EVENT_INCOMING_MSG": + msg = self.get_message_by_id(ffi_event.data2) + return "process_incoming_message", dict(message=msg) + elif name == "DC_EVENT_MSGS_CHANGED": + if ffi_event.data2 != 0: + msg = self.get_message_by_id(ffi_event.data2) + if msg.is_in_fresh(): + return "process_incoming_message", dict(message=msg) + elif name == "DC_EVENT_MSG_DELIVERED": + msg = self.get_message_by_id(ffi_event.data2) + return "process_message_delivered", dict(message=msg) + elif name == "DC_EVENT_MEMBER_ADDED": + chat = self.get_chat_by_id(ffi_event.data1) + contact = self.get_contact_by_id(ffi_event.data2) + return "member_added", dict(chat=chat, contact=contact) + return None, {} def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): @@ -614,3 +640,17 @@ class ScannedQRCode: @property def contact_id(self): return self._dc_lot.id() + + +class HookEvent: + def __init__(self, account, name, kwargs): + assert hasattr(account._pm.hook, name), name + self.account = account + self.name = name + self.kwargs = kwargs + + def call_hook(self): + hook = getattr(self.account._pm.hook, self.name, None) + if hook is None: + raise ValueError("event_name {} unknown".format(self.name)) + return hook(**self.kwargs) diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index d8868f150..092dd7232 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -15,7 +15,8 @@ global_hookimpl = pluggy.HookimplMarker(_global_name) class PerAccount: """ per-Account-instance hook specifications. - If you write a plugin you need to implement one of the following hooks. + Except for process_ffi_event all hooks are executed + in the thread which calls Account.wait_shutdown(). """ @classmethod def _make_plugin_manager(cls): @@ -29,6 +30,10 @@ class PerAccount: ffi_event has "name", "data1", "data2" values as specified with `DC_EVENT_* `_. + + DANGER: this hook is executed from the callback invoked by core. + Hook implementations need to be short running and can typically + not call back into core because this would easily cause recursion issues. """ @account_hookspec diff --git a/python/src/deltachat/iothreads.py b/python/src/deltachat/iothreads.py index 798483f74..da21ffc87 100644 --- a/python/src/deltachat/iothreads.py +++ b/python/src/deltachat/iothreads.py @@ -17,13 +17,17 @@ class IOThreads: def is_started(self): return len(self._name2thread) > 0 - def start(self): + def start(self, callback_thread): assert not self.is_started() self._start_one_thread("inbox", self.imap_thread_run) self._start_one_thread("smtp", self.smtp_thread_run) + if callback_thread: + self._start_one_thread("cb", self.cb_thread_run) + if int(self.account.get_config("mvbox_watch")): self._start_one_thread("mvbox", self.mvbox_thread_run) + if int(self.account.get_config("sentbox_watch")): self._start_one_thread("sentbox", self.sentbox_thread_run) @@ -53,7 +57,18 @@ class IOThreads: lib.dc_interrupt_sentbox_idle(self._dc_context) if wait: for name, thread in self._name2thread.items(): - thread.join() + if thread != threading.currentThread(): + thread.join() + + def cb_thread_run(self): + with self.log_execution("CALLBACK THREAD START"): + it = self.account.iter_events() + while not self._thread_quitflag: + try: + ev = next(it) + except StopIteration: + break + ev.call_hook() def imap_thread_run(self): with self.log_execution("INBOX THREAD START"): diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 13d43d88a..8bfebc4b2 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -3,12 +3,10 @@ import os import sys import pytest import requests -from contextlib import contextmanager import time from . import Account, const from .tracker import ConfigureTracker from .capi import lib -from .hookspec import account_hookimpl from .eventlogger import FFIEventLogger, FFIEventTracker from _pytest.monkeypatch import MonkeyPatch import tempfile @@ -63,9 +61,9 @@ def pytest_report_header(config, startdir): cfg = config.option.liveconfig if cfg: - if "#" in cfg: - url, token = cfg.split("#", 1) - summary.append('Liveconfig provider: {}#'.format(url)) + if "?" in cfg: + url, token = cfg.split("?", 1) + summary.append('Liveconfig provider: {}?'.format(url)) else: summary.append('Liveconfig file: {}'.format(cfg)) return summary @@ -263,26 +261,3 @@ def lp(): def step(self, msg): print("-" * 5, "step " + msg, "-" * 5) return Printer() - - -@pytest.fixture -def make_plugin_recorder(): - @contextmanager - def make_plugin_recorder(account): - class HookImpl: - def __init__(self): - self.calls_member_added = [] - - @account_hookimpl - def member_added(self, chat, contact): - self.calls_member_added.append(dict(chat=chat, contact=contact)) - - def get_first(self, name): - val = getattr(self, "calls_" + name, None) - if val is not None: - return val.pop(0) - - with account.temp_plugin(HookImpl()) as plugin: - yield plugin - - return make_plugin_recorder diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 5c5fc8338..cc26d8feb 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -170,18 +170,19 @@ class TestOfflineChat: else: pytest.fail("could not find chat") - def test_add_member_event(self, ac1, make_plugin_recorder): + def test_add_member_event(self, ac1): chat = ac1.create_group_chat(name="title1") assert chat.is_group() # promote the chat chat.send_text("hello") contact1 = ac1.create_contact("some1@hello.com", name="some1") - with make_plugin_recorder(ac1) as rec: - chat.add_contact(contact1) - kwargs = rec.get_first("member_added") - assert kwargs["chat"] == chat - assert kwargs["contact"] == contact1 + chat.add_contact(contact1) + for ev in ac1.iter_events(timeout=1): + if ev.name == "member_added": + assert ev.kwargs["chat"] == chat + assert ev.kwargs["contact"] == contact1 + break def test_group_chat_creation(self, ac1): contact1 = ac1.create_contact("some1@hello.com", name="some1") @@ -461,6 +462,11 @@ class TestOnlineAccount: ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr"))) return chat + def test_double_iter_events(self, acfactory): + ac1 = acfactory.get_one_online_account() + with pytest.raises(RuntimeError): + next(ac1.iter_events()) + @pytest.mark.ignored def test_configure_generate_key(self, acfactory, lp): # A slow test which will generate new keys. From f61b9f79648f2bf2bcb0c2bb711b01d53ac3fec9 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 3 Mar 2020 18:55:19 +0100 Subject: [PATCH 030/156] add a test echo_and_quit examples --- python/doc/examples.rst | 53 +++++++++------------------ python/doc/plugins.rst | 2 ++ python/examples/conftest.py | 2 ++ python/examples/echo_and_quit.py | 60 ++++++++++++++++++++----------- python/examples/test_examples.py | 41 +++++++++++++++++++++ python/src/deltachat/account.py | 1 + python/src/deltachat/iothreads.py | 1 + python/tox.ini | 3 +- 8 files changed, 106 insertions(+), 57 deletions(-) create mode 100644 python/examples/conftest.py create mode 100644 python/examples/test_examples.py diff --git a/python/doc/examples.rst b/python/doc/examples.rst index 9305fae3c..badb17725 100644 --- a/python/doc/examples.rst +++ b/python/doc/examples.rst @@ -4,52 +4,33 @@ examples ======== -Sending a Chat message from the command line ---------------------------------------------- - -Once you have :doc:`installed deltachat bindings ` -you can start playing from the python interpreter commandline. -For example you can type ``python`` and then:: - - # instantiate and configure deltachat account - import deltachat - ac = deltachat.Account("/tmp/db") - ac.set_config("addr", "address@example.org") - ac.set_config("mail_pwd", "some password") - - # start IO threads and perform configuration - ac.start() - - # create a contact and send a message - contact = ac.create_contact("someother@email.address") - chat = ac.create_chat_by_contact(contact) - chat.send_text("hi from the python interpreter command line") - - # shutdown IO threads - ac.shutdown() - - -Checkout our :doc:`api` for the various high-level things you can do -to send/receive messages, create contacts and chats. - - Receiving a Chat message from the command line ---------------------------------------------- -Instantiate an account and register a plugin to process -incoming messages: +Once you have :doc:`installed deltachat bindings ` +you can start playing from the python interpreter commandline. + +Here is a simple module that implements a bot that: + +- receives a message and sends back an "echo" message + +- terminates the bot if the message `/quit` is sent .. include:: ../examples/echo_and_quit.py :literal: -Checkout our :doc:`api` for the various high-level things you can do -to send/receive messages, create contacts and chats. +With this file in your working directory you can run the bot +by specifying a database path, an e-mail address and password of +a SMTP-IMAP account:: + python echo_and_quit.py --db /tmp/db --email ADDRESS --password PASSWORD -Looking at a real example +While this process is running you can start sending chat messages +to `ADDRESS`. + +Writing bots for real ------------------------- The `deltabot repository `_ -contains a real-life example of Python bindings usage. - +contains a little framework for writing deltachat bots in Python. diff --git a/python/doc/plugins.rst b/python/doc/plugins.rst index 1c4a5a619..f7bd45322 100644 --- a/python/doc/plugins.rst +++ b/python/doc/plugins.rst @@ -17,8 +17,10 @@ Registering a plugin -------------------- .. autofunction:: deltachat.register_global_plugin + :noindex: .. automethod:: deltachat.account.Account.add_account_plugin + :noindex: Per-Account Hook specifications diff --git a/python/examples/conftest.py b/python/examples/conftest.py new file mode 100644 index 000000000..48b66463e --- /dev/null +++ b/python/examples/conftest.py @@ -0,0 +1,2 @@ + +from deltachat.testplugin import * # noqa diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index d37ae2187..bc66c43e0 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -1,37 +1,57 @@ -# instantiate and configure deltachat account +# content of echo_and_quit.py + +import sys +import optparse import deltachat -ac = deltachat.Account("/tmp/db") -# to see low-level events in the console uncomment the following line -# ac.add_account_plugin(deltachat.eventlogger.FFIEventLogger(ac, "")) -if not ac.is_configured(): - ac.set_config("addr", "tmpy.94mtm@testrun.org") - ac.set_config("mail_pw", "5CbD6VnjD/li") - ac.set_config("mvbox_watch", "0") - ac.set_config("sentbox_watch", "0") - -class MyPlugin: +class SimpleEchoPlugin: @deltachat.hookspec.account_hookimpl def process_incoming_message(self, message): print("process_incoming message", message) if message.text.strip() == "/quit": - print("shutting down") - ac.shutdown() + message.account.shutdown() else: - ch = ac.create_chat_by_contact(message.get_sender_contact()) - ch.send_text("echoing {}".format(message.text)) + ch = message.account.create_chat_by_contact(message.get_sender_contact()) + ch.send_text("echoing from {}:\n{}".format(message.get_sender_contact().addr, message.text)) @deltachat.hookspec.account_hookimpl def process_message_delivered(self, message): print("process_message_delivered", message) -ac.add_account_plugin(MyPlugin()) -# start IO threads and perform configuration -ac.start() +def main(argv): + p = optparse.OptionParser("simple-echo") + p.add_option("-l", action="store_true", help="show low-level events") + p.add_option("--db", type="str", help="database file") + p.add_option("--email", type="str", help="email address") + p.add_option("--password", type="str", help="password") -print("waiting for /quit or other message on {}".format(ac.get_config("addr"))) + opts, posargs = p.parse_args(argv) -ac.wait_shutdown() + assert opts.db, "you must specify --db" + ac = deltachat.Account(opts.db) + + if opts.l: + ac.add_account_plugin(deltachat.eventlogger.FFIEventLogger(ac, "echo")) + + if not ac.is_configured(): + assert opts.email and opts.password, "you must specify --email and --password" + ac.set_config("addr", opts.email) + ac.set_config("mail_pw", opts.password) + ac.set_config("mvbox_watch", "0") + ac.set_config("sentbox_watch", "0") + + ac.add_account_plugin(SimpleEchoPlugin()) + + # start IO threads and perform configure if neccessary + ac.start(callback_thread=True) + + print("waiting for /quit or message to echo on: {}".format(ac.get_config("addr"))) + + ac.wait_shutdown() + + +if __name__ == "__main__": + main(sys.argv) diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py new file mode 100644 index 000000000..a9f47b02b --- /dev/null +++ b/python/examples/test_examples.py @@ -0,0 +1,41 @@ + +import threading +import pytest +import py +from echo_and_quit import main + + +@pytest.fixture(scope='session') +def datadir(): + """The py.path.local object of the test-data/ directory.""" + for path in reversed(py.path.local(__file__).parts()): + datadir = path.join('test-data') + if datadir.isdir(): + return datadir + else: + pytest.skip('test-data directory not found') + + +def test_echo_quit_plugin(acfactory): + bot_ac, bot_cfg = acfactory.get_online_config() + + def run_bot(): + print("*"*20 + " starting bot") + main([ + "-l", + "--email", bot_cfg["addr"], + "--password", bot_cfg["mail_pw"], + "--db", bot_ac.db_path + ]) + + t = threading.Thread(target=run_bot) + t.start() + + ac1 = acfactory.get_one_online_account() + bot_contact = ac1.create_contact(bot_cfg["addr"]) + ch1 = ac1.create_chat_by_contact(bot_contact) + ch1.send_text("hello") + reply = ac1._evtracker.wait_next_incoming_message() + assert "hello" in reply.text + ch1.send_text("/quit") + t.join() diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 4a41e0934..51349c8eb 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -58,6 +58,7 @@ class Account(object): db_path = db_path.encode("utf8") if not lib.dc_open(self._dc_context, db_path, ffi.NULL): raise ValueError("Could not dc_open: {}".format(db_path)) + self.db_path = db_path self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) diff --git a/python/src/deltachat/iothreads.py b/python/src/deltachat/iothreads.py index da21ffc87..fe1d1703d 100644 --- a/python/src/deltachat/iothreads.py +++ b/python/src/deltachat/iothreads.py @@ -68,6 +68,7 @@ class IOThreads: ev = next(it) except StopIteration: break + self.account.log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs)) ev.call_hook() def imap_thread_run(self): diff --git a/python/tox.ini b/python/tox.ini index 965607770..cfdab2957 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -8,6 +8,7 @@ envlist = [testenv] commands = pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs:tests} + pytest examples python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS @@ -41,7 +42,7 @@ deps = restructuredtext_lint commands = flake8 src/deltachat - flake8 tests/ + flake8 tests/ examples/ rst-lint --encoding 'utf-8' README.rst [testenv:doc] From d8e14d9993ce2667def35e90f0eddeb996c97a2f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 3 Mar 2020 19:15:50 +0100 Subject: [PATCH 031/156] refine example and make Contact accept Account object --- python/examples/echo_and_quit.py | 21 +++++++++++++-------- python/src/deltachat/account.py | 10 +++++----- python/src/deltachat/chat.py | 2 +- python/src/deltachat/contact.py | 9 +++++++-- python/src/deltachat/message.py | 9 ++++++++- python/tox.ini | 2 +- 6 files changed, 35 insertions(+), 18 deletions(-) diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index bc66c43e0..73a8bc099 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -13,8 +13,10 @@ class SimpleEchoPlugin: if message.text.strip() == "/quit": message.account.shutdown() else: - ch = message.account.create_chat_by_contact(message.get_sender_contact()) - ch.send_text("echoing from {}:\n{}".format(message.get_sender_contact().addr, message.text)) + ch = message.get_sender_chat() + addr = message.get_sender_contact().addr + text = message.text + ch.send_text("echoing from {}:\n{}".format(addr, text)) @deltachat.hookspec.account_hookimpl def process_message_delivered(self, message): @@ -23,7 +25,7 @@ class SimpleEchoPlugin: def main(argv): p = optparse.OptionParser("simple-echo") - p.add_option("-l", action="store_true", help="show low-level events") + p.add_option("-l", action="store_true", help="show ffi") p.add_option("--db", type="str", help="database file") p.add_option("--email", type="str", help="email address") p.add_option("--password", type="str", help="password") @@ -34,10 +36,13 @@ def main(argv): ac = deltachat.Account(opts.db) if opts.l: - ac.add_account_plugin(deltachat.eventlogger.FFIEventLogger(ac, "echo")) + log = deltachat.eventlogger.FFIEventLogger(ac, "echo") + ac.add_account_plugin(log) if not ac.is_configured(): - assert opts.email and opts.password, "you must specify --email and --password" + assert opts.email and opts.password, ( + "you must specify --email and --password" + ) ac.set_config("addr", opts.email) ac.set_config("mail_pw", opts.password) ac.set_config("mvbox_watch", "0") @@ -45,10 +50,10 @@ def main(argv): ac.add_account_plugin(SimpleEchoPlugin()) - # start IO threads and perform configure if neccessary - ac.start(callback_thread=True) + # start IO threads and configure if neccessary + ac.start() - print("waiting for /quit or message to echo on: {}".format(ac.get_config("addr"))) + print("{}: waiting for message".format(ac.get_config("addr"))) ac.wait_shutdown() diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 51349c8eb..853d13894 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -214,7 +214,7 @@ class Account(object): :returns: :class:`deltachat.contact.Contact` """ self.check_is_configured() - return Contact(self._dc_context, const.DC_CONTACT_ID_SELF) + return Contact(self, const.DC_CONTACT_ID_SELF) def create_contact(self, email, name=None): """ create a (new) Contact. If there already is a Contact @@ -229,7 +229,7 @@ class Account(object): email = as_dc_charpointer(email) contact_id = lib.dc_create_contact(self._dc_context, name, email) assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL - return Contact(self._dc_context, contact_id) + return Contact(self, contact_id) def delete_contact(self, contact): """ delete a Contact. @@ -261,7 +261,7 @@ class Account(object): lib.dc_get_contacts(self._dc_context, flags, query), lib.dc_array_unref ) - return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x))) + return list(iter_array(dc_array, lambda x: Contact(self, x))) def create_chat_by_contact(self, contact): """ create or get an existing 1:1 chat object for the specified contact or contact id. @@ -340,7 +340,7 @@ class Account(object): :param contact_id: integer id of this contact. :returns: None or :class:`deltachat.contact.Contact` instance. """ - return Contact(self._dc_context, contact_id) + return Contact(self, contact_id) def get_chat_by_id(self, chat_id): """ return Chat instance. @@ -542,7 +542,7 @@ class Account(object): If this account is not configured, an internal configuration job will be scheduled if config values are sufficiently specified. - You may call :method:`wait_shutdown` or `shutdown` after the + You may call `wait_shutdown` or `shutdown` after the account is in started mode. :raises MissingCredentials: if `addr` and `mail_pw` values are not set. diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index 173ec8e84..ea84c378c 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -363,7 +363,7 @@ class Chat(object): lib.dc_array_unref ) return list(iter_array( - dc_array, lambda id: Contact(self._dc_context, id)) + dc_array, lambda id: Contact(self.account, id)) ) def set_profile_image(self, img_path): diff --git a/python/src/deltachat/contact.py b/python/src/deltachat/contact.py index 9cebc829a..6e2cf1ae5 100644 --- a/python/src/deltachat/contact.py +++ b/python/src/deltachat/contact.py @@ -10,8 +10,9 @@ class Contact(object): You obtain instances of it through :class:`deltachat.account.Account`. """ - def __init__(self, dc_context, id): - self._dc_context = dc_context + def __init__(self, account, id): + self.account = account + self._dc_context = account._dc_context self.id = id def __eq__(self, other): @@ -57,3 +58,7 @@ class Contact(object): if dc_res == ffi.NULL: return None return from_dc_charpointer(dc_res) + + def get_chat(self): + """return 1:1 chat for this contact. """ + return self.account.create_chat_by_contact(self) diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index d8128bfb0..370863231 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -159,6 +159,13 @@ class Message(object): chat_id = lib.dc_msg_get_chat_id(self._dc_msg) return Chat(self.account, chat_id) + def get_sender_chat(self): + """return the 1:1 chat with the sender of this message. + + :returns: :class:`deltachat.chat.Chat` instance + """ + return self.get_sender_contact().get_chat() + def get_sender_contact(self): """return the contact of who wrote the message. @@ -166,7 +173,7 @@ class Message(object): """ from .contact import Contact contact_id = lib.dc_msg_get_from_id(self._dc_msg) - return Contact(self._dc_context, contact_id) + return Contact(self.account, contact_id) # # Message State query methods diff --git a/python/tox.ini b/python/tox.ini index cfdab2957..45810e40b 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -8,7 +8,7 @@ envlist = [testenv] commands = pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs:tests} - pytest examples + pytest examples/test_examples.py python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS From 33dd747ec7b6b48cb10d5eeedb9d335015548a13 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 4 Mar 2020 11:36:28 +0100 Subject: [PATCH 032/156] some more test setup refinements and make example testing part of tox runs --- python/examples/conftest.py | 2 -- python/setup.py | 5 +++++ python/tests/conftest.py | 3 --- python/tox.ini | 8 +++----- 4 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 python/examples/conftest.py diff --git a/python/examples/conftest.py b/python/examples/conftest.py deleted file mode 100644 index 48b66463e..000000000 --- a/python/examples/conftest.py +++ /dev/null @@ -1,2 +0,0 @@ - -from deltachat.testplugin import * # noqa diff --git a/python/setup.py b/python/setup.py index 8c8e63ecc..e2a79e4b1 100644 --- a/python/setup.py +++ b/python/setup.py @@ -22,6 +22,11 @@ def main(): packages=setuptools.find_packages('src'), package_dir={'': 'src'}, cffi_modules=['src/deltachat/_build.py:ffibuilder'], + entry_points = { + 'pytest11': [ + 'deltachat.testplugin = deltachat.testplugin', + ], + }, classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 20a0fdd17..2db21033e 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -5,9 +5,6 @@ import pytest import py -from deltachat.testplugin import * # noqa - - @pytest.fixture(scope="session") def data(): class Data: diff --git a/python/tox.ini b/python/tox.ini index 45810e40b..1b6f0dbd9 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -7,15 +7,14 @@ envlist = [testenv] commands = - pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs:tests} - pytest examples/test_examples.py - python tests/package_wheels.py {toxworkdir}/wheelhouse + pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples} + # python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS DCC_RS_DEV DCC_RS_TARGET DCC_PY_LIVECONFIG - DCC_NEW_TMP_EMAIL + DCC_NEW_TMP_EMAIL CARGO_TARGET_DIR RUSTC_WRAPPER deps = @@ -68,7 +67,6 @@ commands = [pytest] addopts = -v -ra --strict-markers -python_files = tests/test_*.py norecursedirs = .tox xfail_strict=true timeout = 90 From 36b50436d7ad616b0c9f9127fef9525570349245 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 4 Mar 2020 12:03:11 +0100 Subject: [PATCH 033/156] add Message.mark_seen shortcut --- python/src/deltachat/message.py | 4 ++++ python/tests/test_account.py | 2 +- python/tox.ini | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 370863231..e94ba0e1c 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -276,6 +276,10 @@ class Message(object): """ return True if it's a file message. """ return self._view_type == const.DC_MSG_FILE + def mark_seen(self): + """ mark this message as seen. """ + self.account.mark_seen_messages([self.id]) + # some code for handling DC_MSG_* view types diff --git a/python/tests/test_account.py b/python/tests/test_account.py index cc26d8feb..191178b84 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -805,7 +805,7 @@ class TestOnlineAccount: lp.sec("check that a second call to mark_seen does not create change or smtp job") ac2._evtracker.consume_events() - ac2.mark_seen_messages([msg_in]) + msg_in.mark_seen() try: ac2._evtracker.get_matching("DC_EVENT_MSG_READ", timeout=0.01) except queue.Empty: diff --git a/python/tox.ini b/python/tox.ini index 1b6f0dbd9..a1d26c4a1 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -8,7 +8,7 @@ envlist = [testenv] commands = pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples} - # python tests/package_wheels.py {toxworkdir}/wheelhouse + python tests/package_wheels.py {toxworkdir}/wheelhouse passenv = TRAVIS DCC_RS_DEV From d66829702f2d72445fddb2a1f5a7bd6b001a509b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 4 Mar 2020 14:34:26 +0100 Subject: [PATCH 034/156] fix #164 add MEMBER_REMOVED event and member_removed plugin python hook --- deltachat-ffi/deltachat.h | 9 ++++++++ deltachat-ffi/src/lib.rs | 4 ++++ python/src/deltachat/account.py | 18 +++++++++++++-- python/src/deltachat/const.py | 1 + python/src/deltachat/hookspec.py | 4 ++++ python/tests/test_account.py | 38 ++++++++++++++++++++++++++++++++ src/chat.rs | 22 +++++++++++------- src/events.rs | 6 +++++ 8 files changed, 92 insertions(+), 10 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 7e2d7c5b9..7a160d693 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4522,6 +4522,15 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); */ #define DC_EVENT_MEMBER_ADDED 2063 +/** + * This event is sent for each member that gets removed from a (verified or unverified) chat. + * + * @param data1 (int) chat_id + * @param data2 (int) contact_id + * @return 0 + */ +#define DC_EVENT_MEMBER_REMOVED 2064 + /** * @} */ diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 10cdfaccf..5d68aedd6 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -203,6 +203,10 @@ impl ContextWrapper { | Event::MemberAdded { chat_id, contact_id, + } + | Event::MemberRemoved { + chat_id, + contact_id, } => { ffi_cb( self, diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 853d13894..78b21686d 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -3,6 +3,7 @@ from __future__ import print_function import atexit from contextlib import contextmanager +import queue from threading import Event import os from array import array @@ -15,7 +16,6 @@ from .message import Message from .contact import Contact from .tracker import ImexTracker from . import hookspec, iothreads -from queue import Queue class MissingCredentials(ValueError): @@ -49,7 +49,7 @@ class Account(object): hook.account_init(account=self, db_path=db_path) self._threads = iothreads.IOThreads(self) - self._hook_event_queue = Queue() + self._hook_event_queue = queue.Queue() self._in_use_iter_events = False self._shutdown_event = Event() @@ -578,6 +578,16 @@ class Account(object): hook = hookspec.Global._get_plugin_manager().hook hook.account_after_shutdown(account=self, dc_context=dc_context) + def _handle_current_events(self): + """ handle all currently queued events and then return. """ + while 1: + try: + event = self._hook_event_queue.get(block=False) + except queue.Empty: + break + else: + event.call_hook() + def iter_events(self, timeout=None): """ yield hook events until shutdown. @@ -614,6 +624,10 @@ class Account(object): chat = self.get_chat_by_id(ffi_event.data1) contact = self.get_contact_by_id(ffi_event.data2) return "member_added", dict(chat=chat, contact=contact) + elif name == "DC_EVENT_MEMBER_REMOVED": + chat = self.get_chat_by_id(ffi_event.data1) + contact = self.get_contact_by_id(ffi_event.data2) + return "member_removed", dict(chat=chat, contact=contact) return None, {} diff --git a/python/src/deltachat/const.py b/python/src/deltachat/const.py index 09229ff75..249be9804 100644 --- a/python/src/deltachat/const.py +++ b/python/src/deltachat/const.py @@ -100,6 +100,7 @@ DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061 DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062 DC_EVENT_MEMBER_ADDED = 2063 +DC_EVENT_MEMBER_REMOVED = 2064 DC_EVENT_FILE_COPIED = 2055 DC_EVENT_IS_OFFLINE = 2081 DC_EVENT_GET_STRING = 2091 diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 092dd7232..a74a71237 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -56,6 +56,10 @@ class PerAccount: def member_added(self, chat, contact): """ Called for each contact added to a chat. """ + @account_hookspec + def member_removed(self, chat, contact): + """ Called for each contact removed from a chat. """ + class Global: """ global hook specifications using a per-process singleton diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 191178b84..ffecef9af 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -438,20 +438,56 @@ class TestOfflineChat: def test_group_chat_many_members_add_remove(self, ac1, lp): lp.sec("ac1: creating group chat with 10 other members") chat = ac1.create_group_chat(name="title1") + # promote chat + chat.send_text("hello") + assert chat.is_promoted() + + # activate local plugin + in_list = [] + + class InPlugin: + @account_hookimpl + def member_added(self, chat, contact): + in_list.append(("added", chat, contact)) + + @account_hookimpl + def member_removed(self, chat, contact): + in_list.append(("removed", chat, contact)) + + ac1.add_account_plugin(InPlugin()) + + # perform add contact many times contacts = [] for i in range(10): + lp.sec("create contact") contact = ac1.create_contact("some{}@example.org".format(i)) contacts.append(contact) + lp.sec("add contact") chat.add_contact(contact) num_contacts = len(chat.get_contacts()) assert num_contacts == 11 + # perform plugin hooks + ac1._handle_current_events() + + assert len(in_list) == 10 + for in_cmd, in_chat, in_contact in in_list: + assert in_cmd == "added" + assert in_chat == chat + assert in_contact in contacts + lp.sec("ac1: removing two contacts and checking things are right") chat.remove_contact(contacts[9]) chat.remove_contact(contacts[3]) assert len(chat.get_contacts()) == 9 + ac1._handle_current_events() + assert len(in_list) == 12 + assert in_list[-2][0] == "removed" + assert in_list[-2][1] == chat + assert in_list[-2][2] == contacts[9] + class TestOnlineAccount: def get_chat(self, ac1, ac2, both_created=False): @@ -1390,7 +1426,9 @@ class TestGroupStressTests: lp.sec("ac2: removing one contact") to_remove = contacts[-1] + msg.chat.remove_contact(to_remove) + ac2._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED") lp.sec("ac1: receiving system message about contact removal") sysmsg = ac1._evtracker.wait_next_incoming_message() diff --git a/src/chat.rs b/src/chat.rs index 367007279..1d09540b6 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1953,20 +1953,22 @@ pub(crate) fn add_contact_to_chat_ex( msg.param.set_cmd(SystemMessage::MemberAddedToGroup); msg.param.set(Param::Arg, contact.get_addr()); msg.param.set_int(Param::Arg2, from_handshake.into()); + msg.id = send_msg(context, chat_id, &mut msg)?; - context.call_cb(Event::MsgsChanged { - chat_id, - msg_id: msg.id, - }); + // send_msg sends MsgsChanged event + // so we only send an explicit MemberAdded one context.call_cb(Event::MemberAdded { chat_id, contact_id: contact.id, }); + } else { + // send an event for unpromoted groups + // XXX probably not neccessary because ChatModified should suffice + context.call_cb(Event::MsgsChanged { + chat_id, + msg_id: MsgId::new(0), + }); } - context.call_cb(Event::MsgsChanged { - chat_id, - msg_id: MsgId::new(0), - }); context.call_cb(Event::ChatModified(chat_id)); Ok(true) } @@ -2171,6 +2173,10 @@ pub fn remove_contact_from_chat( msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup); msg.param.set(Param::Arg, contact.get_addr()); msg.id = send_msg(context, chat_id, &mut msg)?; + context.call_cb(Event::MemberRemoved { + chat_id, + contact_id: contact.id, + }); context.call_cb(Event::MsgsChanged { chat_id, msg_id: msg.id, diff --git a/src/events.rs b/src/events.rs index 3d1eb140f..540fe3364 100644 --- a/src/events.rs +++ b/src/events.rs @@ -213,4 +213,10 @@ pub enum Event { /// @param data2 (int) contact_id #[strum(props(id = "2063"))] MemberAdded { chat_id: ChatId, contact_id: u32 }, + + /// This event is sent for each contact removed from a chat. + /// @param data1 (int) chat_id + /// @param data2 (int) contact_id + #[strum(props(id = "2064"))] + MemberRemoved { chat_id: ChatId, contact_id: u32 }, } From f38386d164c89636bde04f0f696ecd8febf7f675 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 4 Mar 2020 23:24:58 +0100 Subject: [PATCH 035/156] fix member_added/member_removed event with tests and and provide a group-tracking example --- deltachat-ffi/src/lib.rs | 6 ++- python/examples/group_tracking.py | 64 ++++++++++++++++++++++++++ python/tests/test_account.py | 75 ++++++++++++++++++++++++++++--- src/chat.rs | 55 ++++++++++++++--------- src/contact.rs | 9 ++-- src/dc_receive_imf.rs | 44 +++++++++--------- src/stock.rs | 2 +- 7 files changed, 198 insertions(+), 57 deletions(-) create mode 100644 python/examples/group_tracking.py diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 5d68aedd6..664866300 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -27,7 +27,7 @@ use num_traits::{FromPrimitive, ToPrimitive}; use deltachat::chat::{ChatId, ChatVisibility, MuteDuration}; use deltachat::constants::DC_MSG_ID_LAST_SPECIAL; -use deltachat::contact::Contact; +use deltachat::contact::{Contact, Origin}; use deltachat::context::Context; use deltachat::key::DcKey; use deltachat::message::MsgId; @@ -1665,7 +1665,9 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr( } let ffi_context = &*context; ffi_context - .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, to_string_lossy(addr))) + .with_inner(|ctx| { + Contact::lookup_id_by_addr(ctx, to_string_lossy(addr), Origin::IncomingReplyTo) + }) .unwrap_or(0) } diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py new file mode 100644 index 000000000..3e4390d3a --- /dev/null +++ b/python/examples/group_tracking.py @@ -0,0 +1,64 @@ + +# content of group_tracking.py + +import sys +import optparse +import deltachat + + +class GroupTrackingPlugin: + @deltachat.hookspec.account_hookimpl + def process_incoming_message(self, message): + print("*** process_incoming_message addr={} msg={!r}".format( + message.get_sender_contact().addr, message.text)) + for member in message.chat.get_contacts(): + print("chat member: {}".format(member.addr)) + + @deltachat.hookspec.account_hookimpl + def member_added(self, chat, contact): + print("*** member_added", contact.addr, "from", chat) + for member in chat.get_contacts(): + print("chat member: {}".format(member.addr)) + + @deltachat.hookspec.account_hookimpl + def member_removed(self, chat, contact): + print("*** member_removed", contact.addr, "from", chat) + + +def main(argv): + p = optparse.OptionParser("simple-echo") + p.add_option("-l", action="store_true", help="show ffi") + p.add_option("--db", type="str", help="database file") + p.add_option("--email", type="str", help="email address") + p.add_option("--password", type="str", help="password") + + opts, posargs = p.parse_args(argv) + + assert opts.db, "you must specify --db" + ac = deltachat.Account(opts.db) + + if opts.l: + log = deltachat.eventlogger.FFIEventLogger(ac, "group-tracking") + ac.add_account_plugin(log) + + if not ac.is_configured(): + assert opts.email and opts.password, ( + "you must specify --email and --password" + ) + ac.set_config("addr", opts.email) + ac.set_config("mail_pw", opts.password) + ac.set_config("mvbox_watch", "0") + ac.set_config("sentbox_watch", "0") + + ac.add_account_plugin(GroupTrackingPlugin()) + + # start IO threads and configure if neccessary + ac.start() + + print("{}: waiting for message".format(ac.get_config("addr"))) + + ac.wait_shutdown() + + +if __name__ == "__main__": + main(sys.argv) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index ffecef9af..a51cb38b1 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -173,14 +173,29 @@ class TestOfflineChat: def test_add_member_event(self, ac1): chat = ac1.create_group_chat(name="title1") assert chat.is_group() - # promote the chat - chat.send_text("hello") contact1 = ac1.create_contact("some1@hello.com", name="some1") chat.add_contact(contact1) for ev in ac1.iter_events(timeout=1): if ev.name == "member_added": assert ev.kwargs["chat"] == chat + if ev.kwargs["contact"] == ac1.get_self_contact(): + continue + assert ev.kwargs["contact"] == contact1 + break + + def test_remove_member_event(self, ac1): + chat = ac1.create_group_chat(name="title1") + assert chat.is_group() + contact1 = ac1.create_contact("some1@hello.com", name="some1") + chat.add_contact(contact1) + ac1._handle_current_events() + chat.remove_contact(contact1) + for ev in ac1.iter_events(timeout=1): + if ev.name == "member_removed": + assert ev.kwargs["chat"] == chat + if ev.kwargs["contact"] == ac1.get_self_contact(): + continue assert ev.kwargs["contact"] == contact1 break @@ -471,11 +486,13 @@ class TestOfflineChat: # perform plugin hooks ac1._handle_current_events() - assert len(in_list) == 10 + assert len(in_list) == 11 + chat_contacts = chat.get_contacts() for in_cmd, in_chat, in_contact in in_list: assert in_cmd == "added" assert in_chat == chat - assert in_contact in contacts + assert in_contact in chat_contacts + chat_contacts.remove(in_contact) lp.sec("ac1: removing two contacts and checking things are right") chat.remove_contact(contacts[9]) @@ -483,10 +500,13 @@ class TestOfflineChat: assert len(chat.get_contacts()) == 9 ac1._handle_current_events() - assert len(in_list) == 12 + assert len(in_list) == 13 assert in_list[-2][0] == "removed" assert in_list[-2][1] == chat assert in_list[-2][2] == contacts[9] + assert in_list[-1][0] == "removed" + assert in_list[-1][1] == chat + assert in_list[-1][2] == contacts[3] class TestOnlineAccount: @@ -1259,6 +1279,51 @@ class TestOnlineAccount: msg3 = ac2._evtracker.wait_next_incoming_message() assert msg3.get_sender_contact().get_profile_image() is None + def test_add_remove_member_remote_events(self, acfactory, lp): + ac1, ac2 = acfactory.get_two_online_accounts() + # activate local plugin for ac2 + in_list = queue.Queue() + + class InPlugin: + @account_hookimpl + def member_added(self, chat, contact): + in_list.put(("added", chat, contact)) + + @account_hookimpl + def member_removed(self, chat, contact): + in_list.put(("removed", chat, contact)) + + ac2.add_account_plugin(InPlugin()) + + lp.sec("ac1: create group chat with ac2") + chat = ac1.create_group_chat("hello") + contact = ac1.create_contact(email=ac2.get_config("addr")) + chat.add_contact(contact) + + lp.sec("ac1: send a message to group chat to promote the group") + chat.send_text("afterwards promoted") + ev1 = in_list.get() + ev2 = in_list.get() + assert ev1[2] == ac2.get_self_contact() + assert ev2[2].addr == ac1.get_config("addr") + + lp.sec("ac1: add address2") + contact2 = ac1.create_contact(email="not@example.org") + chat.add_contact(contact2) + ev1 = in_list.get() + assert ev1[2].addr == contact2.addr + + lp.sec("ac1: remove address2") + chat.remove_contact(contact2) + ev1 = in_list.get() + assert ev1[0] == "removed" + assert ev1[2].addr == contact2.addr + + lp.sec("ac1: remove ac2 contact from chat") + chat.remove_contact(contact) + ev1 = in_list.get() + assert ev1[2] == ac2.get_self_contact() + def test_set_get_group_image(self, acfactory, data, lp): ac1, ac2 = acfactory.get_two_online_accounts() diff --git a/src/chat.rs b/src/chat.rs index 1d09540b6..576185e7b 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1830,36 +1830,61 @@ pub fn create_group_chat( Ok(chat_id) } +/// add a contact to the chats_contact table +/// on success emit MemberAdded event and return true pub(crate) fn add_to_chat_contacts_table( context: &Context, chat_id: ChatId, contact_id: u32, ) -> bool { - // add a contact to a chat; the function does not check the type or if any of the record exist or are already - // added to the chat! - sql::execute( + match sql::execute( context, &context.sql, "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)", params![chat_id, contact_id as i32], - ) - .is_ok() + ) { + Ok(()) => { + context.call_cb(Event::MemberAdded { + chat_id, + contact_id, + }); + + true + } + Err(err) => { + error!( + context, + "could not add {} to chat {} table: {}", contact_id, chat_id, err + ); + + false + } + } } +/// remove a contact from the chats_contact table +/// on success emit MemberRemoved event and return true pub(crate) fn remove_from_chat_contacts_table( context: &Context, chat_id: ChatId, contact_id: u32, ) -> bool { - // remove a contact from the chats_contact table unconditionally - // the function does not check the type or if the record exist - sql::execute( + match sql::execute( context, &context.sql, "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?", params![chat_id, contact_id as i32], - ) - .is_ok() + ) { + Ok(()) => { + context.call_cb(Event::MemberRemoved { + chat_id, + contact_id, + }); + + true + } + Err(_) => false, + } } /// Adds a contact to the chat. @@ -1955,12 +1980,6 @@ pub(crate) fn add_contact_to_chat_ex( msg.param.set_int(Param::Arg2, from_handshake.into()); msg.id = send_msg(context, chat_id, &mut msg)?; - // send_msg sends MsgsChanged event - // so we only send an explicit MemberAdded one - context.call_cb(Event::MemberAdded { - chat_id, - contact_id: contact.id, - }); } else { // send an event for unpromoted groups // XXX probably not neccessary because ChatModified should suffice @@ -2173,10 +2192,6 @@ pub fn remove_contact_from_chat( msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup); msg.param.set(Param::Arg, contact.get_addr()); msg.id = send_msg(context, chat_id, &mut msg)?; - context.call_cb(Event::MemberRemoved { - chat_id, - contact_id: contact.id, - }); context.call_cb(Event::MsgsChanged { chat_id, msg_id: msg.id, diff --git a/src/contact.rs b/src/contact.rs index 65ff35ccf..c74ce8db9 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -24,9 +24,6 @@ use crate::peerstate::*; use crate::sql; use crate::stock::StockMessage; -/// Contacts with at least this origin value are shown in the contact list. -const DC_ORIGIN_MIN_CONTACT_LIST: i32 = 0x100; - /// An object representing a single contact in memory. /// /// The contact object is not updated. @@ -94,6 +91,7 @@ pub enum Origin { UnhandledQrScan = 0x80, /// Reply-To: of incoming message of known sender + /// Contacts with at least this origin value are shown in the contact list. IncomingReplyTo = 0x100, /// Cc: of incoming message of known sender @@ -274,7 +272,7 @@ impl Contact { /// /// To validate an e-mail address independently of the contact database /// use `dc_may_be_valid_addr()`. - pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef) -> u32 { + pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef, min_origin: Origin) -> u32 { if addr.as_ref().is_empty() { return 0; } @@ -287,14 +285,13 @@ impl Contact { if addr_cmp(addr_normalized, addr_self) { return DC_CONTACT_ID_SELF; } - context.sql.query_get_value( context, "SELECT id FROM contacts WHERE addr=?1 COLLATE NOCASE AND id>?2 AND origin>=?3 AND blocked=0;", params![ addr_normalized, DC_CONTACT_ID_LAST_SPECIAL as i32, - DC_ORIGIN_MIN_CONTACT_LIST, + min_origin as u32, ], ).unwrap_or_default() } diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 6620202db..53ae79c45 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -800,7 +800,6 @@ fn create_or_lookup_group( let mut chat_id_blocked = Blocked::Not; let mut recreate_member_list = false; let mut send_EVENT_CHAT_MODIFIED = false; - let mut X_MrRemoveFromGrp = None; let mut X_MrAddToGrp = None; let mut X_MrGrpNameChanged = false; let mut better_msg: String = From::from(""); @@ -848,22 +847,25 @@ fn create_or_lookup_group( // but we might not know about this group let grpname = mime_parser.get(HeaderDef::ChatGroupName).cloned(); + let mut removed_id = 0; - if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() { - X_MrRemoveFromGrp = Some(optional_field); - mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup; - let left_group = Contact::lookup_id_by_addr(context, X_MrRemoveFromGrp.as_ref().unwrap()) - == from_id as u32; - better_msg = context.stock_system_msg( - if left_group { - StockMessage::MsgGroupLeft - } else { - StockMessage::MsgDelMember - }, - X_MrRemoveFromGrp.as_ref().unwrap(), - "", - from_id as u32, - ) + if let Some(removed_addr) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() { + removed_id = Contact::lookup_id_by_addr(context, &removed_addr, Origin::Unknown); + if removed_id == 0 { + warn!(context, "removed {:?} has no contact_id", removed_addr); + } else { + mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup; + better_msg = context.stock_system_msg( + if removed_id == from_id as u32 { + StockMessage::MsgGroupLeft + } else { + StockMessage::MsgDelMember + }, + &removed_addr, + "", + from_id as u32, + ); + } } else { let field = mime_parser.get(HeaderDef::ChatGroupMemberAdded).cloned(); if let Some(optional_field) = field { @@ -949,7 +951,7 @@ fn create_or_lookup_group( && !grpid.is_empty() && grpname.is_some() // otherwise, a pending "quit" message may pop up - && X_MrRemoveFromGrp.is_none() + && removed_id == 0 // re-create explicitly left groups only if ourself is re-added && (!group_explicitly_left || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap())) @@ -1083,12 +1085,8 @@ fn create_or_lookup_group( } } send_EVENT_CHAT_MODIFIED = true; - } else if let Some(removed_addr) = X_MrRemoveFromGrp { - let contact_id = Contact::lookup_id_by_addr(context, removed_addr); - if contact_id != 0 { - info!(context, "remove {:?} from chat id={}", contact_id, chat_id); - chat::remove_from_chat_contacts_table(context, chat_id, contact_id); - } + } else if removed_id > 0 { + chat::remove_from_chat_contacts_table(context, chat_id, removed_id); send_EVENT_CHAT_MODIFIED = true; } diff --git a/src/stock.rs b/src/stock.rs index 1653b4855..d536d522e 100644 --- a/src/stock.rs +++ b/src/stock.rs @@ -314,7 +314,7 @@ impl Context { from_id: u32, ) -> String { let insert1 = if id == StockMessage::MsgAddMember || id == StockMessage::MsgDelMember { - let contact_id = Contact::lookup_id_by_addr(self, param1.as_ref()); + let contact_id = Contact::lookup_id_by_addr(self, param1.as_ref(), Origin::Unknown); if contact_id != 0 { Contact::get_by_id(self, contact_id) .map(|contact| contact.get_name_n_addr()) From 6f8067ffd315adaa2ea3d95edb8811d912bdd026 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 6 Mar 2020 10:47:07 +0100 Subject: [PATCH 036/156] address @adbenitez and @r10s comments --- deltachat-ffi/deltachat.h | 13 ++----------- python/CHANGELOG | 4 +++- python/src/deltachat/account.py | 2 +- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 7a160d693..bbaf3ecc1 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4504,15 +4504,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS 2061 -/** - * (DEPRECATED) - * - * @param data1 (int) chat_id - * @param data2 (int) contact_id - * @return 0 - */ -#define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062 - /** * This event is sent for each member that gets added to a (verified or unverified) chat. * @@ -4520,7 +4511,7 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data2 (int) contact_id * @return 0 */ -#define DC_EVENT_MEMBER_ADDED 2063 +#define DC_EVENT_MEMBER_ADDED 2062 /** * This event is sent for each member that gets removed from a (verified or unverified) chat. @@ -4529,7 +4520,7 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data2 (int) contact_id * @return 0 */ -#define DC_EVENT_MEMBER_REMOVED 2064 +#define DC_EVENT_MEMBER_REMOVED 2063 /** * @} diff --git a/python/CHANGELOG b/python/CHANGELOG index 82b6402cb..f972ed346 100644 --- a/python/CHANGELOG +++ b/python/CHANGELOG @@ -5,8 +5,10 @@ - introduced PerAccount and Global hooks that plugins can implement -- introduced `member_added()` plugin event. +- introduced `member_added()` and `member_removed` plugin events. +- introduced two documented examples for an echo and a group-membership + tracking plugin. 0.800.0 ------- diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 78b21686d..80f68449a 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -46,7 +46,7 @@ class Account(object): ) hook = hookspec.Global._get_plugin_manager().hook - hook.account_init(account=self, db_path=db_path) + hook.account_init(account=self) self._threads = iothreads.IOThreads(self) self._hook_event_queue = queue.Queue() From 724e1ea97e6d971c5587af7c44af5f1f20dfc83d Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 6 Mar 2020 11:28:47 +0100 Subject: [PATCH 037/156] simplify example --- python/examples/echo_and_quit.py | 41 ++----------------------------- python/src/deltachat/__init__.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index 73a8bc099..cf9ee8c27 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -1,11 +1,8 @@ # content of echo_and_quit.py -import sys -import optparse import deltachat - class SimpleEchoPlugin: @deltachat.hookspec.account_hookimpl def process_incoming_message(self, message): @@ -23,40 +20,6 @@ class SimpleEchoPlugin: print("process_message_delivered", message) -def main(argv): - p = optparse.OptionParser("simple-echo") - p.add_option("-l", action="store_true", help="show ffi") - p.add_option("--db", type="str", help="database file") - p.add_option("--email", type="str", help="email address") - p.add_option("--password", type="str", help="password") +if __name__ = "__main__": + deltachat.run_simple_cmdline(account_plugins=[SimpleEchoPlugin()]) - opts, posargs = p.parse_args(argv) - - assert opts.db, "you must specify --db" - ac = deltachat.Account(opts.db) - - if opts.l: - log = deltachat.eventlogger.FFIEventLogger(ac, "echo") - ac.add_account_plugin(log) - - if not ac.is_configured(): - assert opts.email and opts.password, ( - "you must specify --email and --password" - ) - ac.set_config("addr", opts.email) - ac.set_config("mail_pw", opts.password) - ac.set_config("mvbox_watch", "0") - ac.set_config("sentbox_watch", "0") - - ac.add_account_plugin(SimpleEchoPlugin()) - - # start IO threads and configure if neccessary - ac.start() - - print("{}: waiting for message".format(ac.get_config("addr"))) - - ac.wait_shutdown() - - -if __name__ == "__main__": - main(sys.argv) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index ac27a4209..13ac63f75 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -2,6 +2,7 @@ from . import capi, const, hookspec from .capi import ffi from .account import Account # noqa from . import eventlogger +from .util import lazydecorator from pkg_resources import get_distribution, DistributionNotFound try: @@ -92,3 +93,44 @@ def unregister_global_plugin(plugin): register_global_plugin(eventlogger) + + +def run_cmdline(argv=None, account_plugins=None): + """ Run a simple default command line app, registering the specified + account plugins. """ + import argparse + if argv is None: + argv = sys.argv + + parser = argparse.ArgumentParser(prog="simple-echo") + parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events") + parser.add_argument("--db", action="store", help="database file") + parser.add_argument("--email", action="store", help="email address") + parser.add_argument("--password", action="store", help="password") + + args = parser.parse_args(argv[1:]) + + assert args.db, "you must specify --db" + ac = deltachat.Account(args.db) + + if args.show_ffi: + log = deltachat.eventlogger.FFIEventLogger(ac, "echo") + ac.add_account_plugin(log) + + if not ac.is_configured(): + assert args.email and args.password, ( + "you must specify --email and --password once to configure this database/account" + ) + ac.set_config("addr", args.email) + ac.set_config("mail_pw", args.password) + ac.set_config("mvbox_watch", "0") + ac.set_config("sentbox_watch", "0") + + ac.add_account_plugin(SimpleEchoPlugin()) + + # start IO threads and configure if neccessary + ac.start() + + print("{}: waiting for message".format(ac.get_config("addr"))) + + ac.wait_shutdown() From a1d5120e5850b1e6d633b0e6e57a128b8e09a34c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 6 Mar 2020 12:09:23 +0100 Subject: [PATCH 038/156] sipmlify plugins and tests and remove superflous core event --- python/examples/echo_and_quit.py | 10 +++++--- python/examples/group_tracking.py | 39 +++--------------------------- python/examples/test_examples.py | 40 +++++++++++++++++++++++++++---- python/src/deltachat/__init__.py | 10 ++++---- python/src/deltachat/account.py | 2 +- src/chat.rs | 7 ------ 6 files changed, 53 insertions(+), 55 deletions(-) diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index cf9ee8c27..2795923f1 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -3,7 +3,8 @@ import deltachat -class SimpleEchoPlugin: + +class EchoPlugin: @deltachat.hookspec.account_hookimpl def process_incoming_message(self, message): print("process_incoming message", message) @@ -20,6 +21,9 @@ class SimpleEchoPlugin: print("process_message_delivered", message) -if __name__ = "__main__": - deltachat.run_simple_cmdline(account_plugins=[SimpleEchoPlugin()]) +def main(argv=None): + deltachat.run_cmdline(argv=argv, account_plugins=[EchoPlugin()]) + +if __name__ == "__main__": + main() diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index 3e4390d3a..d8e159e93 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -1,8 +1,6 @@ # content of group_tracking.py -import sys -import optparse import deltachat @@ -25,40 +23,9 @@ class GroupTrackingPlugin: print("*** member_removed", contact.addr, "from", chat) -def main(argv): - p = optparse.OptionParser("simple-echo") - p.add_option("-l", action="store_true", help="show ffi") - p.add_option("--db", type="str", help="database file") - p.add_option("--email", type="str", help="email address") - p.add_option("--password", type="str", help="password") - - opts, posargs = p.parse_args(argv) - - assert opts.db, "you must specify --db" - ac = deltachat.Account(opts.db) - - if opts.l: - log = deltachat.eventlogger.FFIEventLogger(ac, "group-tracking") - ac.add_account_plugin(log) - - if not ac.is_configured(): - assert opts.email and opts.password, ( - "you must specify --email and --password" - ) - ac.set_config("addr", opts.email) - ac.set_config("mail_pw", opts.password) - ac.set_config("mvbox_watch", "0") - ac.set_config("sentbox_watch", "0") - - ac.add_account_plugin(GroupTrackingPlugin()) - - # start IO threads and configure if neccessary - ac.start() - - print("{}: waiting for message".format(ac.get_config("addr"))) - - ac.wait_shutdown() +def main(argv=None): + deltachat.run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()]) if __name__ == "__main__": - main(sys.argv) + main() diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index a9f47b02b..66d28e828 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -2,7 +2,8 @@ import threading import pytest import py -from echo_and_quit import main +import echo_and_quit +import group_tracking @pytest.fixture(scope='session') @@ -21,11 +22,13 @@ def test_echo_quit_plugin(acfactory): def run_bot(): print("*"*20 + " starting bot") - main([ - "-l", + print("*"*20 + " bot_ac.dbpath", bot_ac.db_path) + echo_and_quit.main([ + "echo", + "--show-ffi", + "--db", bot_ac.db_path, "--email", bot_cfg["addr"], "--password", bot_cfg["mail_pw"], - "--db", bot_ac.db_path ]) t = threading.Thread(target=run_bot) @@ -39,3 +42,32 @@ def test_echo_quit_plugin(acfactory): assert "hello" in reply.text ch1.send_text("/quit") t.join() + + +@pytest.mark.skip(reason="not implemented") +def test_group_tracking_plugin(acfactory): + bot_ac, bot_cfg = acfactory.get_online_config() + + def run_bot(): + print("*"*20 + " starting bot") + print("*"*20 + " bot_ac.dbpath", bot_ac.db_path) + group_tracking.main([ + "group-tracking", + "--show-ffi", bot_ac.db_path, + "--db", bot_ac.db_path, + "--email", bot_cfg["addr"], + "--password", bot_cfg["mail_pw"], + ]) + + t = threading.Thread(target=run_bot) + t.setDaemon(1) + t.start() + + ac1 = acfactory.get_one_online_account() + bot_contact = ac1.create_contact(bot_cfg["addr"]) + ch1 = ac1.create_chat_by_contact(bot_contact) + ch1.send_text("hello") + ch1.add_contact(ac1.create_contact("x@example.org")) + + # XXX wait for bot to receive things + t.join() diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 13ac63f75..f1e140685 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -1,8 +1,9 @@ +import sys + from . import capi, const, hookspec from .capi import ffi from .account import Account # noqa from . import eventlogger -from .util import lazydecorator from pkg_resources import get_distribution, DistributionNotFound try: @@ -111,10 +112,10 @@ def run_cmdline(argv=None, account_plugins=None): args = parser.parse_args(argv[1:]) assert args.db, "you must specify --db" - ac = deltachat.Account(args.db) + ac = Account(args.db) if args.show_ffi: - log = deltachat.eventlogger.FFIEventLogger(ac, "echo") + log = eventlogger.FFIEventLogger(ac, "echo") ac.add_account_plugin(log) if not ac.is_configured(): @@ -126,7 +127,8 @@ def run_cmdline(argv=None, account_plugins=None): ac.set_config("mvbox_watch", "0") ac.set_config("sentbox_watch", "0") - ac.add_account_plugin(SimpleEchoPlugin()) + for plugin in account_plugins or []: + ac.add_account_plugin(plugin) # start IO threads and configure if neccessary ac.start() diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 80f68449a..5c23cbc28 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -54,11 +54,11 @@ class Account(object): self._shutdown_event = Event() # open database + self.db_path = db_path if hasattr(db_path, "encode"): db_path = db_path.encode("utf8") if not lib.dc_open(self._dc_context, db_path, ffi.NULL): raise ValueError("Could not dc_open: {}".format(db_path)) - self.db_path = db_path self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) diff --git a/src/chat.rs b/src/chat.rs index 576185e7b..b1772b741 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1980,13 +1980,6 @@ pub(crate) fn add_contact_to_chat_ex( msg.param.set_int(Param::Arg2, from_handshake.into()); msg.id = send_msg(context, chat_id, &mut msg)?; - } else { - // send an event for unpromoted groups - // XXX probably not neccessary because ChatModified should suffice - context.call_cb(Event::MsgsChanged { - chat_id, - msg_id: MsgId::new(0), - }); } context.call_cb(Event::ChatModified(chat_id)); Ok(true) From d4e1c1b109079f9d6ed776a20a809f1eb4d5dbb2 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 6 Mar 2020 19:11:21 +0100 Subject: [PATCH 039/156] refine handling of accepted contacts in example --- python/examples/echo_and_quit.py | 5 +++-- python/examples/test_examples.py | 2 +- python/src/deltachat/account.py | 3 +++ python/src/deltachat/eventlogger.py | 7 +++++++ python/src/deltachat/message.py | 10 ++++++++++ python/tests/test_account.py | 15 ++++++++++++--- 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index 2795923f1..2ad9450e4 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -11,10 +11,11 @@ class EchoPlugin: if message.text.strip() == "/quit": message.account.shutdown() else: - ch = message.get_sender_chat() + # unconditionally accept the chat + message.accept_sender_contact() addr = message.get_sender_contact().addr text = message.text - ch.send_text("echoing from {}:\n{}".format(addr, text)) + message.chat.send_text("echoing from {}:\n{}".format(addr, text)) @deltachat.hookspec.account_hookimpl def process_message_delivered(self, message): diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index 66d28e828..dc074db82 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -22,7 +22,6 @@ def test_echo_quit_plugin(acfactory): def run_bot(): print("*"*20 + " starting bot") - print("*"*20 + " bot_ac.dbpath", bot_ac.db_path) echo_and_quit.main([ "echo", "--show-ffi", @@ -40,6 +39,7 @@ def test_echo_quit_plugin(acfactory): ch1.send_text("hello") reply = ac1._evtracker.wait_next_incoming_message() assert "hello" in reply.text + assert reply.chat == ch1 ch1.send_text("/quit") t.join() diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 5c23cbc28..c70c54deb 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -283,6 +283,9 @@ class Account(object): """ create or get an existing chat object for the the specified message. + If this message is in the deaddrop chat then + the sender will become an accepted contact. + :param message: messsage id or message instance. :returns: a :class:`deltachat.chat.Chat` object. """ diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 3ac47e290..4be38c63f 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -128,3 +128,10 @@ class FFIEventTracker: """ wait for and return next incoming message. """ ev = self.get_matching("DC_EVENT_INCOMING_MSG") return self.account.get_message_by_id(ev.data2) + + def wait_next_messages_changed(self): + """ wait for and return next message-changed message or None + if the event contains no msgid""" + ev = self.get_matching("DC_EVENT_MSGS_CHANGED") + if ev.data2 > 0: + return self.account.get_message_by_id(ev.data2) diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index e94ba0e1c..fc63636de 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -50,6 +50,16 @@ class Message(object): lib.dc_msg_unref )) + def accept_sender_contact(self): + """ ensure that the sender is an accepted contact + and that the message has a non-deaddrop chat object. + """ + self.account.create_chat_by_message(self) + self._dc_msg = ffi.gc( + lib.dc_get_msg(self._dc_context, self.id), + lib.dc_msg_unref + ) + @props.with_doc def text(self): """unicode text of this messages (might be empty if not a text message). """ diff --git a/python/tests/test_account.py b/python/tests/test_account.py index a51cb38b1..b653159cd 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -276,7 +276,7 @@ class TestOfflineChat: def test_delete_and_send_fails(self, ac1, chat1): chat1.delete() - ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") + ac1._evtracker.wait_next_messages_changed() with pytest.raises(ValueError): chat1.send_text("msg1") @@ -625,8 +625,8 @@ class TestOnlineAccount: ev = ac1._evtracker.get_matching("DC_EVENT_DELETED_BLOB_FILE") # Second client receives only second message, but not the first - ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_CHANGED") - assert ac1_clone.get_message_by_id(ev.data2).text == msg_out.text + ev_msg = ac1_clone._evtracker.wait_next_messages_changed() + assert ev_msg.text == msg_out.text def test_send_file_twice_unicode_filename_mangling(self, tmpdir, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -1377,6 +1377,15 @@ class TestOnlineAccount: assert chat1b.get_profile_image() is None assert chat.get_profile_image() is None + def test_accept_sender_contact(self, acfactory, lp): + ac1, ac2 = acfactory.get_two_online_accounts() + ch = ac1.create_chat_by_contact(ac1.create_contact(ac2.get_config("addr"))) + ch.send_text("hello") + msg = ac2._evtracker.wait_next_messages_changed() + assert msg.chat.is_deaddrop() + msg.accept_sender_contact() + assert not msg.chat.is_deaddrop() + def test_send_receive_locations(self, acfactory, lp): now = datetime.utcnow() ac1, ac2 = acfactory.get_two_online_accounts() From 6c3a8448cfc4e5859d6875b0de4a446c4b05d8ea Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 7 Mar 2020 08:47:36 +0100 Subject: [PATCH 040/156] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit some fixes thanks to @adbenitez Co-Authored-By: Asiel Díaz Benítez --- python/CHANGELOG | 2 +- python/src/deltachat/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/CHANGELOG b/python/CHANGELOG index f972ed346..52daab3ca 100644 --- a/python/CHANGELOG +++ b/python/CHANGELOG @@ -5,7 +5,7 @@ - introduced PerAccount and Global hooks that plugins can implement -- introduced `member_added()` and `member_removed` plugin events. +- introduced `member_added()` and `member_removed()` plugin events. - introduced two documented examples for an echo and a group-membership tracking plugin. diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index f1e140685..e54a95cbf 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -103,7 +103,7 @@ def run_cmdline(argv=None, account_plugins=None): if argv is None: argv = sys.argv - parser = argparse.ArgumentParser(prog="simple-echo") + parser = argparse.ArgumentParser(prog=argv[0] if argv else None) parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events") parser.add_argument("--db", action="store", help="database file") parser.add_argument("--email", action="store", help="email address") From d4ba09c7538a857ccd0f1398b973176d5c4b1a2e Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sat, 7 Mar 2020 09:45:39 +0100 Subject: [PATCH 041/156] refactor bot-setup and testing into a helper function --- python/examples/test_examples.py | 51 ++++++++---------------------- python/src/deltachat/testplugin.py | 30 ++++++++++++++++++ 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index dc074db82..eaed592be 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -1,5 +1,4 @@ -import threading import pytest import py import echo_and_quit @@ -18,56 +17,32 @@ def datadir(): def test_echo_quit_plugin(acfactory): - bot_ac, bot_cfg = acfactory.get_online_config() - - def run_bot(): - print("*"*20 + " starting bot") - echo_and_quit.main([ - "echo", - "--show-ffi", - "--db", bot_ac.db_path, - "--email", bot_cfg["addr"], - "--password", bot_cfg["mail_pw"], - ]) - - t = threading.Thread(target=run_bot) - t.start() + botproc = acfactory.run_bot_process(echo_and_quit) ac1 = acfactory.get_one_online_account() - bot_contact = ac1.create_contact(bot_cfg["addr"]) + bot_contact = ac1.create_contact(botproc.addr) ch1 = ac1.create_chat_by_contact(bot_contact) ch1.send_text("hello") reply = ac1._evtracker.wait_next_incoming_message() assert "hello" in reply.text assert reply.chat == ch1 ch1.send_text("/quit") - t.join() + botproc.wait() -@pytest.mark.skip(reason="not implemented") +@pytest.mark.skip(reason="botproc-matching not implementing") def test_group_tracking_plugin(acfactory): - bot_ac, bot_cfg = acfactory.get_online_config() - - def run_bot(): - print("*"*20 + " starting bot") - print("*"*20 + " bot_ac.dbpath", bot_ac.db_path) - group_tracking.main([ - "group-tracking", - "--show-ffi", bot_ac.db_path, - "--db", bot_ac.db_path, - "--email", bot_cfg["addr"], - "--password", bot_cfg["mail_pw"], - ]) - - t = threading.Thread(target=run_bot) - t.setDaemon(1) - t.start() + botproc = acfactory.run_bot_process(group_tracking) ac1 = acfactory.get_one_online_account() - bot_contact = ac1.create_contact(bot_cfg["addr"]) - ch1 = ac1.create_chat_by_contact(bot_contact) + bot_contact = ac1.create_contact(botproc.addr) + ch1 = ac1.create_group_chat("bot test group") + ch1.add_contact(bot_contact) ch1.send_text("hello") ch1.add_contact(ac1.create_contact("x@example.org")) - # XXX wait for bot to receive things - t.join() + botproc.fnmatch_lines(""" + *member_added x@example.org* + """) + + botproc.kill() diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 8bfebc4b2..c065cae0a 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import sys +import subprocess import pytest import requests import time @@ -241,11 +242,40 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): ac.start() return ac + def run_bot_process(self, module): + fn = module.__file__ + + bot_ac, bot_cfg = self.get_online_config() + + popen = subprocess.Popen([ + sys.executable, + fn, + "--show-ffi", + "--db", bot_ac.db_path, + "--email", bot_cfg["addr"], + "--password", bot_cfg["mail_pw"], + ]) + bot = BotProcess(popen, bot_cfg) + self._finalizers.append(bot.kill) + return bot + am = AccountMaker() request.addfinalizer(am.finalize) return am +class BotProcess: + def __init__(self, popen, bot_cfg): + self.popen = popen + self.addr = bot_cfg["addr"] + + def kill(self): + self.popen.kill() + + def wait(self, timeout=30): + self.popen.wait(timeout=timeout) + + @pytest.fixture def tmp_db_path(tmpdir): return tmpdir.join("test.db").strpath From 57f879a6ba40752857391f2066a704ca9659ba0c Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 25 Mar 2020 06:34:37 +0100 Subject: [PATCH 042/156] fix buffer handling so that the group-tracking bot example passes --- python/examples/group_tracking.py | 4 ++ python/examples/test_examples.py | 41 +++++++++++++---- python/src/deltachat/__init__.py | 3 +- python/src/deltachat/eventlogger.py | 2 +- python/src/deltachat/testplugin.py | 71 ++++++++++++++++++++++++----- 5 files changed, 98 insertions(+), 23 deletions(-) diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index d8e159e93..290af5eea 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -12,6 +12,10 @@ class GroupTrackingPlugin: for member in message.chat.get_contacts(): print("chat member: {}".format(member.addr)) + @deltachat.hookspec.account_hookimpl + def configure_completed(self, success): + print("*** configure_completed:", success) + @deltachat.hookspec.account_hookimpl def member_added(self, chat, contact): print("*** member_added", contact.addr, "from", chat) diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index eaed592be..572b4ac68 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -3,6 +3,7 @@ import pytest import py import echo_and_quit import group_tracking +from deltachat.eventlogger import FFIEventLogger @pytest.fixture(scope='session') @@ -30,19 +31,39 @@ def test_echo_quit_plugin(acfactory): botproc.wait() -@pytest.mark.skip(reason="botproc-matching not implementing") -def test_group_tracking_plugin(acfactory): +def test_group_tracking_plugin(acfactory, lp): + lp.sec("creating one group-tracking bot and two temp accounts") botproc = acfactory.run_bot_process(group_tracking) - ac1 = acfactory.get_one_online_account() - bot_contact = ac1.create_contact(botproc.addr) - ch1 = ac1.create_group_chat("bot test group") - ch1.add_contact(bot_contact) - ch1.send_text("hello") - ch1.add_contact(ac1.create_contact("x@example.org")) + ac1, ac2 = acfactory.get_two_online_accounts(quiet=True) botproc.fnmatch_lines(""" - *member_added x@example.org* + *configure_completed: True* """) + ac1.add_account_plugin(FFIEventLogger(ac1, "ac1")) + ac2.add_account_plugin(FFIEventLogger(ac2, "ac2")) - botproc.kill() + lp.sec("creating bot test group with all three") + bot_contact = ac1.create_contact(botproc.addr) + ch = ac1.create_group_chat("bot test group") + ch.add_contact(bot_contact) + ch.send_text("hello") + + botproc.fnmatch_lines(""" + *member_added {}* + """.format(ac1.get_config("addr"))) + + lp.sec("adding third member {}".format(ac2.get_config("addr"))) + contact3 = ac1.create_contact(ac2.get_config("addr")) + ch.add_contact(contact3) + + lp.sec("now looking at what the bot received") + botproc.fnmatch_lines(""" + *member_added {}* + """.format(contact3.addr)) + + lp.sec("contact successfully added, now removing") + ch.remove_contact(contact3) + botproc.fnmatch_lines(""" + *member_removed {}* + """.format(contact3.addr)) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index e54a95cbf..2df36bd2b 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -115,7 +115,7 @@ def run_cmdline(argv=None, account_plugins=None): ac = Account(args.db) if args.show_ffi: - log = eventlogger.FFIEventLogger(ac, "echo") + log = eventlogger.FFIEventLogger(ac, "bot") ac.add_account_plugin(log) if not ac.is_configured(): @@ -124,6 +124,7 @@ def run_cmdline(argv=None, account_plugins=None): ) ac.set_config("addr", args.email) ac.set_config("mail_pw", args.password) + ac.set_config("mvbox_move", "0") ac.set_config("mvbox_watch", "0") ac.set_config("sentbox_watch", "0") diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index 4be38c63f..e90d483d0 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -71,7 +71,7 @@ class FFIEventLogger: locname += "-" + self.logid s = "{:2.2f} [{}] {}".format(elapsed, locname, message) with self._loglock: - print(s) + print(s, flush=True) class FFIEventTracker: diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index c065cae0a..bbdc7eaeb 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -2,6 +2,9 @@ from __future__ import print_function import os import sys import subprocess +import queue +import threading +import fnmatch import pytest import requests import time @@ -10,6 +13,8 @@ from .tracker import ConfigureTracker from .capi import lib from .eventlogger import FFIEventLogger, FFIEventTracker from _pytest.monkeypatch import MonkeyPatch +from _pytest._code import Source + import tempfile @@ -138,11 +143,12 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): fin = self._finalizers.pop() fin() - def make_account(self, path, logid): + def make_account(self, path, logid, quiet=False): ac = Account(path) ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac)) ac._configtracker = ac.add_account_plugin(ConfigureTracker()) - ac.add_account_plugin(FFIEventLogger(ac, logid=logid)) + if not quiet: + ac.add_account_plugin(FFIEventLogger(ac, logid=logid)) self._finalizers.append(ac.shutdown) return ac @@ -177,7 +183,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): lib.dc_set_config(ac._dc_context, b"configured", b"1") return ac - def get_online_config(self, pre_generated_key=True): + def get_online_config(self, pre_generated_key=True, quiet=False): if not session_liveconfig: pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig") configdict = session_liveconfig.get(self.live_count) @@ -190,7 +196,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT) tmpdb = tmpdir.join("livedb%d" % self.live_count) - ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count)) + ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet) if pre_generated_key: self._preconfigure_key(ac, configdict['addr']) ac._evtracker.init_time = self.init_time @@ -198,9 +204,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): return ac, dict(configdict) def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False, - pre_generated_key=True, config={}): + pre_generated_key=True, quiet=False, config={}): ac, configdict = self.get_online_config( - pre_generated_key=pre_generated_key) + pre_generated_key=pre_generated_key, quiet=quiet) configdict.update(config) configdict["mvbox_watch"] = str(int(mvbox)) configdict["mvbox_move"] = str(int(move)) @@ -217,9 +223,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): ac1._configtracker.wait_finish() return ac1 - def get_two_online_accounts(self, move=False): - ac1 = self.get_online_configuring_account(move=True) - ac2 = self.get_online_configuring_account() + def get_two_online_accounts(self, move=False, quiet=False): + ac1 = self.get_online_configuring_account(move=True, quiet=quiet) + ac2 = self.get_online_configuring_account(quiet=quiet) ac1._configtracker.wait_finish() ac2._configtracker.wait_finish() return ac1, ac2 @@ -247,14 +253,24 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): bot_ac, bot_cfg = self.get_online_config() - popen = subprocess.Popen([ + args = [ sys.executable, fn, "--show-ffi", "--db", bot_ac.db_path, "--email", bot_cfg["addr"], "--password", bot_cfg["mail_pw"], - ]) + ] + print("$", " ".join(args)) + popen = subprocess.Popen( + args=args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, # combine stdout/stderr in one stream + bufsize=1, # line buffering + close_fds=True, # close all FDs other than 0/1/2 + universal_newlines=True # give back text + ) bot = BotProcess(popen, bot_cfg) self._finalizers.append(bot.kill) return bot @@ -269,12 +285,45 @@ class BotProcess: self.popen = popen self.addr = bot_cfg["addr"] + # we read stdout as quickly as we can in a thread and make + # the (unicode) lines available for readers through a queue. + self.stdout_queue = queue.Queue() + self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread") + t.setDaemon(1) + t.start() + + def _run_stdout_thread(self): + try: + while 1: + line = self.popen.stdout.readline() + if not line: + break + line = line.strip() + print("QUEUING:", repr(line)) + self.stdout_queue.put(line) + finally: + self.stdout_queue.put(None) + def kill(self): self.popen.kill() def wait(self, timeout=30): self.popen.wait(timeout=timeout) + def fnmatch_lines(self, pattern_lines): + patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()] + for next_pattern in patterns: + print("+++FNMATCH:", next_pattern) + while 1: + line = self.stdout_queue.get(timeout=15) + if line is None: + raise IOError("BOT stdout-thread terminated") + if fnmatch.fnmatch(line, next_pattern): + print("+++MATCHED:", line) + break + else: + print("+++IGN:", line) + @pytest.fixture def tmp_db_path(tmpdir): From 2a34022619b4307d60327e1c31101c4e184e2e5e Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 25 Mar 2020 07:48:21 +0100 Subject: [PATCH 043/156] refine example doc and address https://github.com/deltachat/deltachat-core-rust/pull/1307#pullrequestreview-380876587 --- python/doc/examples.rst | 38 ++++++++++++++++++++++++------ python/examples/echo_and_quit.py | 8 +++---- python/examples/group_tracking.py | 25 ++++++++++++-------- python/examples/test_examples.py | 3 +++ python/src/deltachat/__init__.py | 4 ++-- python/src/deltachat/testplugin.py | 3 +-- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/python/doc/examples.rst b/python/doc/examples.rst index badb17725..dd8f2a023 100644 --- a/python/doc/examples.rst +++ b/python/doc/examples.rst @@ -1,18 +1,18 @@ - examples ======== +Once you have :doc:`installed deltachat bindings ` +you need email/password credentials for an IMAP/SMTP account. +Delta Chat developers and the CI system use a special URL to create +temporary e-mail accounts on [testrun.org](https://testrun.org) for testing. Receiving a Chat message from the command line ---------------------------------------------- -Once you have :doc:`installed deltachat bindings ` -you can start playing from the python interpreter commandline. +Here is a simple bot that: -Here is a simple module that implements a bot that: - -- receives a message and sends back an "echo" message +- receives a message and sends back ("echoes") a message - terminates the bot if the message `/quit` is sent @@ -23,11 +23,35 @@ With this file in your working directory you can run the bot by specifying a database path, an e-mail address and password of a SMTP-IMAP account:: - python echo_and_quit.py --db /tmp/db --email ADDRESS --password PASSWORD + $ cd examples + $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD While this process is running you can start sending chat messages to `ADDRESS`. +Track member additions and removals in a group +---------------------------------------------- + +Here is a simple bot that: + +- echoes messages sent to it + +- tracks if configuration completed + +- tracks member additions and removals for all chat groups + +.. include:: ../examples/group_tracking.py + :literal: + +With this file in your working directory you can run the bot +by specifying a database path, an e-mail address and password of +a SMTP-IMAP account:: + + python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db + +When this process is running you can start sending chat messages +to `ADDRESS`. + Writing bots for real ------------------------- diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index 2ad9450e4..eea1ffafb 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -1,11 +1,11 @@ # content of echo_and_quit.py -import deltachat +from deltachat import account_hookimpl, run_cmdline class EchoPlugin: - @deltachat.hookspec.account_hookimpl + @account_hookimpl def process_incoming_message(self, message): print("process_incoming message", message) if message.text.strip() == "/quit": @@ -17,13 +17,13 @@ class EchoPlugin: text = message.text message.chat.send_text("echoing from {}:\n{}".format(addr, text)) - @deltachat.hookspec.account_hookimpl + @account_hookimpl def process_message_delivered(self, message): print("process_message_delivered", message) def main(argv=None): - deltachat.run_cmdline(argv=argv, account_plugins=[EchoPlugin()]) + run_cmdline(argv=argv, account_plugins=[EchoPlugin()]) if __name__ == "__main__": diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index 290af5eea..da6c07031 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -1,34 +1,39 @@ # content of group_tracking.py -import deltachat +from deltachat import account_hookimpl, run_cmdline class GroupTrackingPlugin: - @deltachat.hookspec.account_hookimpl + @account_hookimpl def process_incoming_message(self, message): - print("*** process_incoming_message addr={} msg={!r}".format( - message.get_sender_contact().addr, message.text)) - for member in message.chat.get_contacts(): - print("chat member: {}".format(member.addr)) + print("process_incoming message", message) + if message.text.strip() == "/quit": + message.account.shutdown() + else: + # unconditionally accept the chat + message.accept_sender_contact() + addr = message.get_sender_contact().addr + text = message.text + message.chat.send_text("echoing from {}:\n{}".format(addr, text)) - @deltachat.hookspec.account_hookimpl + @account_hookimpl def configure_completed(self, success): print("*** configure_completed:", success) - @deltachat.hookspec.account_hookimpl + @account_hookimpl def member_added(self, chat, contact): print("*** member_added", contact.addr, "from", chat) for member in chat.get_contacts(): print("chat member: {}".format(member.addr)) - @deltachat.hookspec.account_hookimpl + @account_hookimpl def member_removed(self, chat, contact): print("*** member_removed", contact.addr, "from", chat) def main(argv=None): - deltachat.run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()]) + run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()]) if __name__ == "__main__": diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index 572b4ac68..343528aa2 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -57,6 +57,9 @@ def test_group_tracking_plugin(acfactory, lp): contact3 = ac1.create_contact(ac2.get_config("addr")) ch.add_contact(contact3) + reply = ac1._evtracker.wait_next_incoming_message() + assert "hello" in reply.text + lp.sec("now looking at what the bot received") botproc.fnmatch_lines(""" *member_added {}* diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 2df36bd2b..7def90569 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -4,6 +4,7 @@ from . import capi, const, hookspec from .capi import ffi from .account import Account # noqa from . import eventlogger +from .hookspec import account_hookimpl, global_hookimpl # noqa from pkg_resources import get_distribution, DistributionNotFound try: @@ -104,14 +105,13 @@ def run_cmdline(argv=None, account_plugins=None): argv = sys.argv parser = argparse.ArgumentParser(prog=argv[0] if argv else None) + parser.add_argument("db", action="store", help="database file") parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events") - parser.add_argument("--db", action="store", help="database file") parser.add_argument("--email", action="store", help="email address") parser.add_argument("--password", action="store", help="password") args = parser.parse_args(argv[1:]) - assert args.db, "you must specify --db" ac = Account(args.db) if args.show_ffi: diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index bbdc7eaeb..6f1b3437e 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -257,9 +257,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): sys.executable, fn, "--show-ffi", - "--db", bot_ac.db_path, "--email", bot_cfg["addr"], "--password", bot_cfg["mail_pw"], + bot_ac.db_path, ] print("$", " ".join(args)) popen = subprocess.Popen( @@ -299,7 +299,6 @@ class BotProcess: if not line: break line = line.strip() - print("QUEUING:", repr(line)) self.stdout_queue.put(line) finally: self.stdout_queue.put(None) From f98d0bbc1f1179b64ddec515f5ceb2e4266ed72b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 26 Mar 2020 09:45:39 +0100 Subject: [PATCH 044/156] fix failing sync test --- python/tests/test_account.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index b653159cd..69b3f3b0b 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -1563,7 +1563,7 @@ class TestGroupStressTests: # send a message to get the contact key via autocrypt header chat1.send_text("hi") - msg = ac1.wait_next_incoming_message() + msg = ac1._evtracker.wait_next_incoming_message() assert msg.text == "hi" ac2, ac3 = accounts @@ -1584,7 +1584,7 @@ class TestGroupStressTests: lp.sec("checking that the chat arrived correctly") for ac in accounts: - msg = ac.wait_next_incoming_message() + msg = ac._evtracker.wait_next_incoming_message() assert msg.text == "hello" print("chat is", msg.chat) assert len(msg.chat.get_contacts()) == 3 @@ -1593,7 +1593,7 @@ class TestGroupStressTests: chat.remove_contact(contacts[0]) lp.sec("ac2: wait for a message about removal from the chat") - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() lp.sec("ac1: removing ac3") chat.remove_contact(contacts[1]) @@ -1604,7 +1604,7 @@ class TestGroupStressTests: chat.add_contact(contacts[0]) lp.sec("ac2: check that ac3 is removed") - msg = ac2.wait_next_incoming_message() + msg = ac2._evtracker.wait_next_incoming_message() assert len(msg.chat.get_contacts()) == len(chat.get_contacts()) From 7e1470ea4613b3880b2b185e83fe1176f3450d36 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 27 Mar 2020 09:27:43 +0100 Subject: [PATCH 045/156] refactor preconfigure handling to not break deltabot's usage of deltachat's test fixtures and relax timestamp comparisons --- python/src/deltachat/account.py | 6 ++-- python/src/deltachat/testplugin.py | 44 ++++++++++++++++++++++++++---- python/tests/conftest.py | 28 ------------------- python/tests/data/key | 1 + python/tests/test_account.py | 11 ++++---- 5 files changed, 48 insertions(+), 42 deletions(-) create mode 120000 python/tests/data/key diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index c70c54deb..f9b71ed0a 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -46,7 +46,6 @@ class Account(object): ) hook = hookspec.Global._get_plugin_manager().hook - hook.account_init(account=self) self._threads = iothreads.IOThreads(self) self._hook_event_queue = queue.Queue() @@ -61,6 +60,7 @@ class Account(object): raise ValueError("Could not dc_open: {}".format(db_path)) self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) + hook.account_init(account=self) @hookspec.account_hookimpl def process_ffi_event(self, ffi_event): @@ -519,11 +519,11 @@ class Account(object): # meta API for start/stop and event based processing # - def add_account_plugin(self, plugin): + def add_account_plugin(self, plugin, name=None): """ add an account plugin which implements one or more of the :class:`deltachat.hookspec.PerAccount` hooks. """ - self._pm.register(plugin) + self._pm.register(plugin, name=name) self._pm.check_pending() return plugin diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 6f1b3437e..c3e92f0eb 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -127,7 +127,37 @@ def session_liveconfig(request): @pytest.fixture -def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): +def data(request): + class Data: + def __init__(self): + # trying to find test data heuristically + # because we are run from a dev-setup with pytest direct, + # through tox, and then maybe also from deltachat-binding + # users like "deltabot". + self.paths = [os.path.normpath(x) for x in [ + os.path.join(os.path.dirname(request.fspath.strpath), "data"), + os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data") + ]] + + def get_path(self, bn): + """ return path of file or None if it doesn't exist. """ + for path in self.paths: + fn = os.path.join(path, *bn.split("/")) + if os.path.exists(fn): + return fn + print("WARNING: path does not exist: {!r}".format(fn)) + + def read_path(self, bn, mode="r"): + fn = self.get_path(bn) + if fn is not None: + with open(fn, mode) as f: + return f.read() + + return Data() + + +@pytest.fixture +def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): class AccountMaker: def __init__(self): @@ -164,11 +194,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir): # Only set a key if we haven't used it yet for another account. if self._generated_keys: keyname = self._generated_keys.pop(0) - fname_pub = "key/{name}-public.asc".format(name=keyname) - fname_sec = "key/{name}-secret.asc".format(name=keyname) - account._preconfigure_keypair(addr, - datadir.join(fname_pub).read(), - datadir.join(fname_sec).read()) + fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname)) + fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname)) + if fname_pub and fname_sec: + account._preconfigure_keypair(addr, fname_pub, fname_sec) + return True + else: + print("WARN: could not use preconfigured keys for {!r}".format(addr)) def get_configured_offline_account(self): ac = self.get_unconfigured_account() diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 2db21033e..b32e2fd0d 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -1,33 +1,5 @@ from __future__ import print_function -import os -import pytest -import py - - -@pytest.fixture(scope="session") -def data(): - class Data: - def __init__(self): - self.path = os.path.join(os.path.dirname(__file__), "data") - - def get_path(self, bn): - fn = os.path.join(self.path, bn) - assert os.path.exists(fn) - return fn - return Data() - - -@pytest.fixture(scope='session') -def datadir(): - """The py.path.local object of the test-data/ directory.""" - for path in reversed(py.path.local(__file__).parts()): - datadir = path.join('test-data') - if datadir.isdir(): - return datadir - else: - pytest.skip('test-data directory not found') - def wait_configuration_progress(account, min_target, max_target=1001): min_target = min(min_target, max_target) diff --git a/python/tests/data/key b/python/tests/data/key new file mode 120000 index 000000000..0351f725d --- /dev/null +++ b/python/tests/data/key @@ -0,0 +1 @@ +../../../test-data/key \ No newline at end of file diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 69b3f3b0b..a86170f8b 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -26,11 +26,12 @@ class TestOfflineAccountBasic: ac1 = Account(p.strpath, os_name="solarpunk") ac1.get_info() - def test_preconfigure_keypair(self, acfactory, datadir): + def test_preconfigure_keypair(self, acfactory, data): ac = acfactory.get_unconfigured_account() - ac._preconfigure_keypair("alice@example.com", - datadir.join("key/alice-public.asc").read(), - datadir.join("key/alice-secret.asc").read()) + alice_public = data.read_path("key/alice-public.asc") + alice_secret = data.read_path("key/alice-secret.asc") + assert alice_public and alice_secret + ac._preconfigure_keypair("alice@example.com", alice_public, alice_secret) def test_getinfo(self, acfactory): ac1 = acfactory.get_unconfigured_account() @@ -827,7 +828,7 @@ class TestOnlineAccount: assert msg_in in chat2.get_messages() assert chat2.is_deaddrop() assert chat2.count_fresh_messages() == 0 - assert msg_in.time_received > msg_in.time_sent + assert msg_in.time_received >= msg_out.time_sent lp.sec("create new chat with contact and verify it's proper") chat2b = ac2.create_chat_by_message(msg_in) From ca88c5b41c7f21f55a03a8cdd8cb562def3f40b8 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 29 Mar 2020 07:34:48 +0200 Subject: [PATCH 046/156] rename hooks to use "ac_" (account) and "dc_" (global) --- python/CHANGELOG | 2 +- python/examples/echo_and_quit.py | 6 ++--- python/examples/group_tracking.py | 14 +++++------ python/examples/test_examples.py | 8 +++---- python/src/deltachat/account.py | 22 +++++++++--------- python/src/deltachat/eventlogger.py | 16 ++++++------- python/src/deltachat/hookspec.py | 36 ++++++++++++++--------------- python/src/deltachat/iothreads.py | 6 ++--- python/src/deltachat/tracker.py | 6 ++--- python/tests/test_account.py | 16 ++++++------- python/tests/test_lowlevel.py | 2 +- 11 files changed, 67 insertions(+), 67 deletions(-) diff --git a/python/CHANGELOG b/python/CHANGELOG index 52daab3ca..a32c2def9 100644 --- a/python/CHANGELOG +++ b/python/CHANGELOG @@ -5,7 +5,7 @@ - introduced PerAccount and Global hooks that plugins can implement -- introduced `member_added()` and `member_removed()` plugin events. +- introduced `ac_member_added()` and `ac_member_removed()` plugin events. - introduced two documented examples for an echo and a group-membership tracking plugin. diff --git a/python/examples/echo_and_quit.py b/python/examples/echo_and_quit.py index eea1ffafb..367d1fd58 100644 --- a/python/examples/echo_and_quit.py +++ b/python/examples/echo_and_quit.py @@ -6,7 +6,7 @@ from deltachat import account_hookimpl, run_cmdline class EchoPlugin: @account_hookimpl - def process_incoming_message(self, message): + def ac_incoming_message(self, message): print("process_incoming message", message) if message.text.strip() == "/quit": message.account.shutdown() @@ -18,8 +18,8 @@ class EchoPlugin: message.chat.send_text("echoing from {}:\n{}".format(addr, text)) @account_hookimpl - def process_message_delivered(self, message): - print("process_message_delivered", message) + def ac_message_delivered(self, message): + print("ac_message_delivered", message) def main(argv=None): diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index da6c07031..baed3e951 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -6,7 +6,7 @@ from deltachat import account_hookimpl, run_cmdline class GroupTrackingPlugin: @account_hookimpl - def process_incoming_message(self, message): + def ac_incoming_message(self, message): print("process_incoming message", message) if message.text.strip() == "/quit": message.account.shutdown() @@ -18,18 +18,18 @@ class GroupTrackingPlugin: message.chat.send_text("echoing from {}:\n{}".format(addr, text)) @account_hookimpl - def configure_completed(self, success): - print("*** configure_completed:", success) + def ac_configure_completed(self, success): + print("*** ac_configure_completed:", success) @account_hookimpl - def member_added(self, chat, contact): - print("*** member_added", contact.addr, "from", chat) + def ac_member_added(self, chat, contact): + print("*** ac_member_added", contact.addr, "from", chat) for member in chat.get_contacts(): print("chat member: {}".format(member.addr)) @account_hookimpl - def member_removed(self, chat, contact): - print("*** member_removed", contact.addr, "from", chat) + def ac_member_removed(self, chat, contact): + print("*** ac_member_removed", contact.addr, "from", chat) def main(argv=None): diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index 343528aa2..f3feb0e58 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -38,7 +38,7 @@ def test_group_tracking_plugin(acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts(quiet=True) botproc.fnmatch_lines(""" - *configure_completed: True* + *ac_configure_completed: True* """) ac1.add_account_plugin(FFIEventLogger(ac1, "ac1")) ac2.add_account_plugin(FFIEventLogger(ac2, "ac2")) @@ -50,7 +50,7 @@ def test_group_tracking_plugin(acfactory, lp): ch.send_text("hello") botproc.fnmatch_lines(""" - *member_added {}* + *ac_member_added {}* """.format(ac1.get_config("addr"))) lp.sec("adding third member {}".format(ac2.get_config("addr"))) @@ -62,11 +62,11 @@ def test_group_tracking_plugin(acfactory, lp): lp.sec("now looking at what the bot received") botproc.fnmatch_lines(""" - *member_added {}* + *ac_member_added {}* """.format(contact3.addr)) lp.sec("contact successfully added, now removing") ch.remove_contact(contact3) botproc.fnmatch_lines(""" - *member_removed {}* + *ac_member_removed {}* """.format(contact3.addr)) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index f9b71ed0a..ab724b436 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -60,10 +60,10 @@ class Account(object): raise ValueError("Could not dc_open: {}".format(db_path)) self._configkeys = self.get_config("sys.config_keys").split() atexit.register(self.shutdown) - hook.account_init(account=self) + hook.dc_account_init(account=self) @hookspec.account_hookimpl - def process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event): name, kwargs = self._map_ffi_event(ffi_event) if name is not None: ev = HookEvent(self, name=name, kwargs=kwargs) @@ -72,8 +72,8 @@ class Account(object): # def __del__(self): # self.shutdown() - def log_line(self, msg): - self._pm.hook.log_line(message=msg) + def ac_log_line(self, msg): + self._pm.hook.ac_log_line(message=msg) def _check_config_key(self, name): if name not in self._configkeys: @@ -579,7 +579,7 @@ class Account(object): atexit.unregister(self.shutdown) self._shutdown_event.set() hook = hookspec.Global._get_plugin_manager().hook - hook.account_after_shutdown(account=self, dc_context=dc_context) + hook.dc_account_after_shutdown(account=self, dc_context=dc_context) def _handle_current_events(self): """ handle all currently queued events and then return. """ @@ -611,26 +611,26 @@ class Account(object): data1 = ffi_event.data1 if data1 == 0 or data1 == 1000: success = data1 == 1000 - return "configure_completed", dict(success=success) + return "ac_configure_completed", dict(success=success) elif name == "DC_EVENT_INCOMING_MSG": msg = self.get_message_by_id(ffi_event.data2) - return "process_incoming_message", dict(message=msg) + return "ac_incoming_message", dict(message=msg) elif name == "DC_EVENT_MSGS_CHANGED": if ffi_event.data2 != 0: msg = self.get_message_by_id(ffi_event.data2) if msg.is_in_fresh(): - return "process_incoming_message", dict(message=msg) + return "ac_incoming_message", dict(message=msg) elif name == "DC_EVENT_MSG_DELIVERED": msg = self.get_message_by_id(ffi_event.data2) - return "process_message_delivered", dict(message=msg) + return "ac_message_delivered", dict(message=msg) elif name == "DC_EVENT_MEMBER_ADDED": chat = self.get_chat_by_id(ffi_event.data1) contact = self.get_contact_by_id(ffi_event.data2) - return "member_added", dict(chat=chat, contact=contact) + return "ac_member_added", dict(chat=chat, contact=contact) elif name == "DC_EVENT_MEMBER_REMOVED": chat = self.get_chat_by_id(ffi_event.data1) contact = self.get_contact_by_id(ffi_event.data2) - return "member_removed", dict(chat=chat, contact=contact) + return "ac_member_removed", dict(chat=chat, contact=contact) return None, {} diff --git a/python/src/deltachat/eventlogger.py b/python/src/deltachat/eventlogger.py index e90d483d0..5490e7e72 100644 --- a/python/src/deltachat/eventlogger.py +++ b/python/src/deltachat/eventlogger.py @@ -7,19 +7,19 @@ from .hookspec import account_hookimpl, global_hookimpl @global_hookimpl -def account_init(account): +def dc_account_init(account): # send all FFI events for this account to a plugin hook def _ll_event(ctx, evt_name, data1, data2): assert ctx == account._dc_context ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2) - account._pm.hook.process_ffi_event( + account._pm.hook.ac_process_ffi_event( account=account, ffi_event=ffi_event ) deltachat.set_context_callback(account._dc_context, _ll_event) @global_hookimpl -def account_after_shutdown(dc_context): +def dc_account_after_shutdown(dc_context): deltachat.clear_context_callback(dc_context) @@ -50,17 +50,17 @@ class FFIEventLogger: self.init_time = time.time() @account_hookimpl - def process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event): self._log_event(ffi_event) def _log_event(self, ffi_event): # don't show events that are anyway empty impls now if ffi_event.name == "DC_EVENT_GET_STRING": return - self.account.log_line(str(ffi_event)) + self.account.ac_log_line(str(ffi_event)) @account_hookimpl - def log_line(self, message): + def ac_log_line(self, message): t = threading.currentThread() tname = getattr(t, "name", t) if tname == "MainThread": @@ -81,7 +81,7 @@ class FFIEventTracker: self._event_queue = Queue() @account_hookimpl - def process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event): self._event_queue.put(ffi_event) def set_timeout(self, timeout): @@ -110,7 +110,7 @@ class FFIEventTracker: assert not rex.match(ev.name), "event found {}".format(ev) def get_matching(self, event_name_regex, check_error=True, timeout=None): - self.account.log_line("-- waiting for event with regex: {} --".format(event_name_regex)) + self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex)) rex = re.compile("(?:{}).*".format(event_name_regex)) while 1: ev = self.get(timeout=timeout, check_error=check_error) diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index a74a71237..1512d031e 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -3,29 +3,29 @@ import pluggy -_account_name = "deltachat-account" -account_hookspec = pluggy.HookspecMarker(_account_name) -account_hookimpl = pluggy.HookimplMarker(_account_name) +account_spec_name = "deltachat-account" +account_hookspec = pluggy.HookspecMarker(account_spec_name) +account_hookimpl = pluggy.HookimplMarker(account_spec_name) -_global_name = "deltachat-global" -global_hookspec = pluggy.HookspecMarker(_global_name) -global_hookimpl = pluggy.HookimplMarker(_global_name) +global_spec_name = "deltachat-global" +global_hookspec = pluggy.HookspecMarker(global_spec_name) +global_hookimpl = pluggy.HookimplMarker(global_spec_name) class PerAccount: """ per-Account-instance hook specifications. - Except for process_ffi_event all hooks are executed + Except for ac_process_ffi_event all hooks are executed in the thread which calls Account.wait_shutdown(). """ @classmethod def _make_plugin_manager(cls): - pm = pluggy.PluginManager(_account_name) + pm = pluggy.PluginManager(account_spec_name) pm.add_hookspecs(cls) return pm @account_hookspec - def process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event): """ process a CFFI low level events for a given account. ffi_event has "name", "data1", "data2" values as specified @@ -37,27 +37,27 @@ class PerAccount: """ @account_hookspec - def log_line(self, message): + def ac_log_line(self, message): """ log a message related to the account. """ @account_hookspec - def configure_completed(self, success): + def ac_configure_completed(self, success): """ Called when a configure process completed. """ @account_hookspec - def process_incoming_message(self, message): + def ac_incoming_message(self, message): """ Called on any incoming message (to deaddrop or chat). """ @account_hookspec - def process_message_delivered(self, message): + def ac_message_delivered(self, message): """ Called when an outgoing message has been delivered to SMTP. """ @account_hookspec - def member_added(self, chat, contact): + def ac_member_added(self, chat, contact): """ Called for each contact added to a chat. """ @account_hookspec - def member_removed(self, chat, contact): + def ac_member_removed(self, chat, contact): """ Called for each contact removed from a chat. """ @@ -71,14 +71,14 @@ class Global: @classmethod def _get_plugin_manager(cls): if cls._plugin_manager is None: - cls._plugin_manager = pm = pluggy.PluginManager(_global_name) + cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name) pm.add_hookspecs(cls) return cls._plugin_manager @global_hookspec - def account_init(self, account): + def dc_account_init(self, account): """ called when `Account::__init__()` function starts executing. """ @global_hookspec - def account_after_shutdown(self, account, dc_context): + def dc_account_after_shutdown(self, account, dc_context): """ Called after the account has been shutdown. """ diff --git a/python/src/deltachat/iothreads.py b/python/src/deltachat/iothreads.py index fe1d1703d..1e9a864f1 100644 --- a/python/src/deltachat/iothreads.py +++ b/python/src/deltachat/iothreads.py @@ -38,9 +38,9 @@ class IOThreads: @contextmanager def log_execution(self, message): - self.account.log_line(message + " START") + self.account.ac_log_line(message + " START") yield - self.account.log_line(message + " FINISHED") + self.account.ac_log_line(message + " FINISHED") def stop(self, wait=False): self._thread_quitflag = True @@ -68,7 +68,7 @@ class IOThreads: ev = next(it) except StopIteration: break - self.account.log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs)) + self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs)) ev.call_hook() def imap_thread_run(self): diff --git a/python/src/deltachat/tracker.py b/python/src/deltachat/tracker.py index 815005e97..4570f3d4c 100644 --- a/python/src/deltachat/tracker.py +++ b/python/src/deltachat/tracker.py @@ -14,7 +14,7 @@ class ImexTracker: self._imex_events = Queue() @account_hookimpl - def process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event): if ffi_event.name == "DC_EVENT_IMEX_PROGRESS": self._imex_events.put(ffi_event.data1) elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN": @@ -47,7 +47,7 @@ class ConfigureTracker: self._ffi_events = [] @account_hookimpl - def process_ffi_event(self, ffi_event): + def ac_process_ffi_event(self, ffi_event): self._ffi_events.append(ffi_event) if ffi_event.name == "DC_EVENT_SMTP_CONNECTED": self._smtp_finished.set() @@ -55,7 +55,7 @@ class ConfigureTracker: self._imap_finished.set() @account_hookimpl - def configure_completed(self, success): + def ac_configure_completed(self, success): self._configure_events.put(success) def wait_smtp_connected(self): diff --git a/python/tests/test_account.py b/python/tests/test_account.py index a86170f8b..c1f506186 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -178,7 +178,7 @@ class TestOfflineChat: chat.add_contact(contact1) for ev in ac1.iter_events(timeout=1): - if ev.name == "member_added": + if ev.name == "ac_member_added": assert ev.kwargs["chat"] == chat if ev.kwargs["contact"] == ac1.get_self_contact(): continue @@ -193,7 +193,7 @@ class TestOfflineChat: ac1._handle_current_events() chat.remove_contact(contact1) for ev in ac1.iter_events(timeout=1): - if ev.name == "member_removed": + if ev.name == "ac_member_removed": assert ev.kwargs["chat"] == chat if ev.kwargs["contact"] == ac1.get_self_contact(): continue @@ -463,11 +463,11 @@ class TestOfflineChat: class InPlugin: @account_hookimpl - def member_added(self, chat, contact): + def ac_member_added(self, chat, contact): in_list.append(("added", chat, contact)) @account_hookimpl - def member_removed(self, chat, contact): + def ac_member_removed(self, chat, contact): in_list.append(("removed", chat, contact)) ac1.add_account_plugin(InPlugin()) @@ -1051,14 +1051,14 @@ class TestOnlineAccount: class InPlugin: @account_hookimpl - def process_incoming_message(self, message): + def ac_incoming_message(self, message): message_queue.put(message) delivered = queue.Queue() class OutPlugin: @account_hookimpl - def process_message_delivered(self, message): + def ac_message_delivered(self, message): delivered.put(message) ac1.add_account_plugin(OutPlugin()) @@ -1287,11 +1287,11 @@ class TestOnlineAccount: class InPlugin: @account_hookimpl - def member_added(self, chat, contact): + def ac_member_added(self, chat, contact): in_list.put(("added", chat, contact)) @account_hookimpl - def member_removed(self, chat, contact): + def ac_member_removed(self, chat, contact): in_list.put(("removed", chat, contact)) ac2.add_account_plugin(InPlugin()) diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index 37dfd5625..48865ed44 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -26,7 +26,7 @@ def test_dc_close_events(tmpdir, acfactory): class ShutdownPlugin: @global_hookimpl - def account_after_shutdown(self, account): + def dc_account_after_shutdown(self, account): assert account._dc_context is None shutdowns.append(account) register_global_plugin(ShutdownPlugin()) From 1b858393c50f76da923ac6f7fbd5f981ad1f6450 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 29 Mar 2020 13:53:04 +0200 Subject: [PATCH 047/156] make create_contact accept email addresses that parse into routable_email and display_name --- python/src/deltachat/account.py | 10 +++++++--- python/tests/test_account.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index ab724b436..0e12cc921 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -3,6 +3,7 @@ from __future__ import print_function import atexit from contextlib import contextmanager +from email.utils import parseaddr import queue from threading import Event import os @@ -225,9 +226,12 @@ class Account(object): :param name: display name for this contact (optional) :returns: :class:`deltachat.contact.Contact` instance. """ - name = as_dc_charpointer(name) - email = as_dc_charpointer(email) - contact_id = lib.dc_create_contact(self._dc_context, name, email) + realname, addr = parseaddr(email) + if name: + realname = name + realname = as_dc_charpointer(realname) + addr = as_dc_charpointer(addr) + contact_id = lib.dc_create_contact(self._dc_context, realname, addr) assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL return Contact(self, contact_id) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index c1f506186..7eb66799c 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -368,6 +368,17 @@ class TestOfflineChat: assert msg2 != msg assert msg2.filename != msg.filename + def test_create_contact(self, acfactory): + ac1 = acfactory.get_configured_offline_account() + email = "hello " + contact1 = ac1.create_contact(email) + assert contact1.addr == "hello@example.org" + assert contact1.display_name == "hello" + contact1 = ac1.create_contact(email, name="world") + assert contact1.display_name == "world" + contact2 = ac1.create_contact("display1 ", "real") + assert contact2.display_name == "real" + def test_create_chat_mismatch(self, acfactory): ac1 = acfactory.get_configured_offline_account() ac2 = acfactory.get_configured_offline_account() From 323d996d5f9b881fefd17ea4c81e441e49ee2b23 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 29 Mar 2020 14:49:28 +0200 Subject: [PATCH 048/156] a few streamlinings --- python/src/deltachat/__init__.py | 5 ++++- python/src/deltachat/account.py | 1 - python/tests/test_account.py | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/src/deltachat/__init__.py b/python/src/deltachat/__init__.py index 7def90569..41c513145 100644 --- a/python/src/deltachat/__init__.py +++ b/python/src/deltachat/__init__.py @@ -3,8 +3,11 @@ import sys from . import capi, const, hookspec from .capi import ffi from .account import Account # noqa -from . import eventlogger +from .message import Message # noqa +from .contact import Contact # noqa +from .chat import Chat # noqa from .hookspec import account_hookimpl, global_hookimpl # noqa +from . import eventlogger from pkg_resources import get_distribution, DistributionNotFound try: diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 0e12cc921..7f881ebb7 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -214,7 +214,6 @@ class Account(object): :returns: :class:`deltachat.contact.Contact` """ - self.check_is_configured() return Contact(self, const.DC_CONTACT_ID_SELF) def create_contact(self, email, name=None): diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 7eb66799c..84bfc98e1 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -64,8 +64,7 @@ class TestOfflineAccountBasic: def test_selfcontact_if_unconfigured(self, acfactory): ac1 = acfactory.get_unconfigured_account() - with pytest.raises(ValueError): - ac1.get_self_contact() + assert not ac1.get_self_contact().addr def test_selfcontact_configured(self, acfactory): ac1 = acfactory.get_configured_offline_account() From 1855f84fe0f48f79a51dd1be09a9978e36ef189a Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 13 Apr 2020 11:56:36 +0200 Subject: [PATCH 049/156] fix bug in that remove-contact failed on new groups where we didn't have the peerstate of the removed-contact yet --- python/tests/test_account.py | 4 ++++ src/chat.rs | 25 ++++++++++++++++++++----- src/mimefactory.rs | 15 --------------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 84bfc98e1..7a7d53db9 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -1210,6 +1210,10 @@ class TestOnlineAccount: wait_securejoin_inviter_progress(ac1, 1000) ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED") + ch.remove_contact(ac1.get_self_contact()) + ac2._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED") + ac1._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED") + def test_qr_verified_group_and_chatting(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() lp.sec("ac1: create verified-group QR, ac2 scans and joins") diff --git a/src/chat.rs b/src/chat.rs index b1772b741..78dc0175f 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1883,7 +1883,14 @@ pub(crate) fn remove_from_chat_contacts_table( true } - Err(_) => false, + Err(_) => { + warn!( + context, + "could not remove contact {:?} from chat {:?}", contact_id, chat_id + ); + + false + } } } @@ -2161,8 +2168,7 @@ pub fn remove_contact_from_chat( "Cannot remove contact from chat; self not in group.".into() ) ); - } else if remove_from_chat_contacts_table(context, chat_id, contact_id) { - /* we should respect this - whatever we send to the group, it gets discarded anyway! */ + } else { if let Ok(contact) = Contact::get_by_id(context, contact_id) { if chat.is_promoted() { msg.viewtype = Viewtype::Text; @@ -2191,9 +2197,18 @@ pub fn remove_contact_from_chat( }); } } - + // we remove the member from the chat after constructing the + // to-be-send message. If between send_msg() and here the + // process dies the user will have to re-do the action. It's + // better than the other way round: you removed + // someone from DB but no peer or device gets to know about it and + // group membership is thus different on different devices. + // Note also that sending a message needs all recipients + // in order to correctly determine encryption so if we + // removed it first, it would complicate the + // check/encryption logic. + success = remove_from_chat_contacts_table(context, chat_id, contact_id); context.call_cb(Event::ChatModified(chat_id)); - success = true; } } } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 8a370cf57..58217face 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -106,21 +106,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> { let command = msg.param.get_cmd(); - /* for added members, the list is just fine */ - if command == SystemMessage::MemberRemovedFromGroup { - let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default(); - - let self_addr = context - .get_config(Config::ConfiguredAddr) - .unwrap_or_default(); - - if !email_to_remove.is_empty() - && !addr_cmp(email_to_remove, self_addr) - && !recipients_contain_addr(&recipients, &email_to_remove) - { - recipients.push(("".to_string(), email_to_remove.to_string())); - } - } if command != SystemMessage::AutocryptSetupMessage && command != SystemMessage::SecurejoinMessage && context.get_config_bool(Config::MdnsEnabled) From ea455323d8149a4c3f6be0b83b3ced09bd92e8a7 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 13 Apr 2020 12:21:11 +0200 Subject: [PATCH 050/156] increase timeout while waiting for rsa2048 keygen -- default timeout is 30secs, and it sometimes takes 31 or more on the CI machines --- python/tests/test_account.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 7a7d53db9..c8eeddc6d 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -545,6 +545,8 @@ class TestOnlineAccount: pre_generated_key=False, config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)} ) + # rsa key gen can be slow especially on CI, adjust timeout + ac1._evtracker.set_timeout(120) wait_configuration_progress(ac1, 1000) wait_configuration_progress(ac2, 1000) chat = self.get_chat(ac1, ac2, both_created=True) From 76b93274e84d193647387f43ec2c655c1eaeb40b Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 13 Apr 2020 13:04:36 +0200 Subject: [PATCH 051/156] use latest sphinx -- seems to work again --- python/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tox.ini b/python/tox.ini index a1d26c4a1..3309b08a3 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -47,7 +47,7 @@ commands = [testenv:doc] changedir=doc deps = - sphinx==2.2.0 + sphinx breathe commands = sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html From 134b09dba57178242e4f3a7dd8d1e8b523763410 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 6 Apr 2020 14:02:56 +0200 Subject: [PATCH 052/156] Fix #1373, ignore incorrect html close tags --- src/dehtml.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/dehtml.rs b/src/dehtml.rs index 0b2f5435f..5639e3af4 100644 --- a/src/dehtml.rs +++ b/src/dehtml.rs @@ -35,6 +35,7 @@ pub fn dehtml(buf: &str) -> String { }; let mut reader = quick_xml::Reader::from_str(buf); + reader.check_end_names(false); let mut buf = Vec::new(); @@ -225,4 +226,23 @@ mod tests { "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}" ); } + + #[test] + fn test_unclosed_tags() { + let input = r##" + + + + Hi + + + + lots of text + + + "##; + let txt = dehtml(input); + assert_eq!(txt.trim(), "lots of text"); + } } From 960c8745d907cb207041441b237a1b90a6fd5307 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sat, 4 Apr 2020 19:00:00 +0200 Subject: [PATCH 053/156] For smtp error 5.5.0, try again. For others, send info message (doesn't work) --- src/job.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/job.rs b/src/job.rs index d53e84224..482bc6adf 100644 --- a/src/job.rs +++ b/src/job.rs @@ -9,6 +9,9 @@ use deltachat_derive::{FromSql, ToSql}; use itertools::Itertools; use rand::{thread_rng, Rng}; +use async_smtp::smtp::response::Category; +use async_smtp::smtp::response::Code; +use async_smtp::smtp::response::Detail; use async_std::task; use crate::blob::BlobObject; @@ -200,8 +203,28 @@ impl Job { self.pending_error = Some(err.to_string()); let res = match err { - async_smtp::smtp::error::Error::Permanent(_) => { - Status::Finished(Err(format_err!("Permanent SMTP error: {}", err))) + async_smtp::smtp::error::Error::Permanent(ref response) => { + match response.code { + // Sometimes servers send a permanent error when actually it is a temporary error + // For documentation see https://tools.ietf.org/html/rfc3463 + + // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2 + Code { + severity: _, + category: Category::MailSystem, + detail: Detail::Zero, + } => Status::RetryLater, + _ => { + // If we do not retry, add an info message to the chat + // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM. + chat::add_info_msg( + context, + ChatId::new(self.foreign_id), + err.to_string(), + ); + Status::Finished(Err(format_err!("Permanent SMTP error: {}", err))) + } + } } async_smtp::smtp::error::Error::Transient(_) => { // We got a transient 4xx response from SMTP server. From fdc091319bf79aa97cdbe2e29baa1254652f4152 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 5 Apr 2020 10:43:52 +0200 Subject: [PATCH 054/156] Make Clippy happy --- src/job.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index 482bc6adf..369b6580c 100644 --- a/src/job.rs +++ b/src/job.rs @@ -210,10 +210,11 @@ impl Job { // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2 Code { - severity: _, category: Category::MailSystem, detail: Detail::Zero, + .. } => Status::RetryLater, + _ => { // If we do not retry, add an info message to the chat // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM. From 6d89638ca4b22dfe08e2b0e3af5e97303ac8e1f6 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 6 Apr 2020 10:50:10 +0200 Subject: [PATCH 055/156] Get ChatId from Message --- src/job.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/job.rs b/src/job.rs index 369b6580c..e12f32cb1 100644 --- a/src/job.rs +++ b/src/job.rs @@ -218,11 +218,19 @@ impl Job { _ => { // If we do not retry, add an info message to the chat // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM. - chat::add_info_msg( - context, - ChatId::new(self.foreign_id), - err.to_string(), - ); + match Message::load_from_db(context, MsgId::new(self.foreign_id)) { + Ok(message) => chat::add_info_msg( + context, + message.chat_id, + err.to_string(), + ), + Err(e) => warn!( + context, + "couldn't load chat_id to inform user about SMTP error: {}", + e + ), + }; + Status::Finished(Err(format_err!("Permanent SMTP error: {}", err))) } } From 016a7806320362b2b26f7d3d3c23c40e92ecbf41 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Wed, 1 Apr 2020 00:43:37 +0200 Subject: [PATCH 056/156] check that incoming read-receipts do not unarchive chats --- src/dc_receive_imf.rs | 119 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 53ae79c45..e5ef249dd 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -1655,6 +1655,7 @@ fn dc_create_incoming_rfc724_mid( #[cfg(test)] mod tests { use super::*; + use crate::chat::ChatVisibility; use crate::chatlist::Chatlist; use crate::message::Message; use crate::test_utils::{dummy_context, TestContext}; @@ -1882,4 +1883,122 @@ mod tests { assert_eq!(chat.name, "group with Alice, Bob and Claire"); assert_eq!(chat::get_chat_contacts(&t.ctx, chat_id).len(), 3); } + + #[test] + fn test_read_receipt_and_unarchive() { + // create alice's account + let t = configured_offline_context(); + + // create one-to-one with bob, archive one-to-one + let bob_id = Contact::create(&t.ctx, "bob", "bob@exampel.org").unwrap(); + let one2one_id = chat::create_by_contact_id(&t.ctx, bob_id).unwrap(); + one2one_id + .set_visibility(&t.ctx, ChatVisibility::Archived) + .unwrap(); + let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap(); + assert!(one2one.get_visibility() == ChatVisibility::Archived); + + // create a group with bob, archive group + let group_id = chat::create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap(); + chat::add_contact_to_chat(&t.ctx, group_id, bob_id); + assert_eq!(chat::get_chat_msgs(&t.ctx, group_id, 0, None).len(), 0); + group_id + .set_visibility(&t.ctx, ChatVisibility::Archived) + .unwrap(); + let group = Chat::load_from_db(&t.ctx, group_id).unwrap(); + assert!(group.get_visibility() == ChatVisibility::Archived); + + // everything archived, chatlist should be empty + assert_eq!( + Chatlist::try_load(&t.ctx, DC_GCL_NO_SPECIALS, None, None) + .unwrap() + .len(), + 0 + ); + + // send a message to group with bob + dc_receive_imf( + &t.ctx, + format!( + "From: alice@example.org\n\ + To: bob@example.org\n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Chat-Group-ID: {}\n\ + Chat-Group-Name: foo\n\ + Chat-Disposition-Notification-To: alice@example.org\n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ + \n\ + hello\n", + group.grpid, group.grpid + ) + .as_bytes(), + "INBOX", + 1, + false, + ) + .unwrap(); + let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None); + assert_eq!(msgs.len(), 1); + let msg_id = msgs.first().unwrap(); + let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap(); + assert_eq!(msg.is_dc_message, MessengerMessage::Yes); + assert_eq!(msg.text.unwrap(), "hello"); + assert_eq!(msg.state, MessageState::OutDelivered); + let group = Chat::load_from_db(&t.ctx, group_id).unwrap(); + assert!(group.get_visibility() == ChatVisibility::Normal); + + // bob sends a read receipt to the group + dc_receive_imf( + &t.ctx, + format!( + "From: bob@example.org\n\ + To: alice@example.org\n\ + Subject: message opened\n\ + Date: Sun, 22 Mar 2020 23:37:57 +0000\n\ + Chat-Version: 1.0\n\ + Message-ID: \n\ + Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\ + \n\ + \n\ + --SNIPP\n\ + Content-Type: text/plain; charset=utf-8\n\ + \n\ + Read receipts do not guarantee sth. was read.\n\ + \n\ + \n\ + --SNIPP\n\ + Content-Type: message/disposition-notification\n\ + \n\ + Reporting-UA: Delta Chat 1.28.0\n\ + Original-Recipient: rfc822;bob@example.org\n\ + Final-Recipient: rfc822;bob@example.org\n\ + Original-Message-ID: \n\ + Disposition: manual-action/MDN-sent-automatically; displayed\n\ + \n\ + \n\ + --SNIPP--", + group.grpid + ) + .as_bytes(), + "INBOX", + 1, + false, + ) + .unwrap(); + assert_eq!(chat::get_chat_msgs(&t.ctx, group_id, 0, None).len(), 1); + let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap(); + assert_eq!(msg.state, MessageState::OutMdnRcvd); + + // check, the read-receipt has not unarchived the one2one + assert_eq!( + Chatlist::try_load(&t.ctx, DC_GCL_NO_SPECIALS, None, None) + .unwrap() + .len(), + 1 + ); + let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap(); + assert!(one2one.get_visibility() == ChatVisibility::Archived); + } } From 24730e7ad66a4a88cf01d7d81ed0c363971ba7c7 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 28 Feb 2020 14:58:29 +0100 Subject: [PATCH 057/156] do not delete handshake-messages needed for a multi-device-verification --- src/securejoin.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/securejoin.rs b/src/securejoin.rs index 2964aba98..a7167b0bf 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -622,7 +622,7 @@ pub(crate) fn handle_securejoin_handshake( send_handshake_msg(context, contact_chat_id, "vc-contact-confirm", "", None, ""); inviter_progress!(context, contact_id, 1000); } - Ok(HandshakeMessage::Done) + Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed) } "vg-member-added" | "vc-contact-confirm" => { /*======================================================= @@ -723,7 +723,7 @@ pub(crate) fn handle_securejoin_handshake( Ok(if join_vg { HandshakeMessage::Propagate } else { - HandshakeMessage::Done + HandshakeMessage::Ignore // "Done" deletes the message and breaks multi-device }) } "vg-member-added-received" => { @@ -754,7 +754,7 @@ pub(crate) fn handle_securejoin_handshake( chat_id: group_chat_id, contact_id, }); - Ok(HandshakeMessage::Done) + Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device } else { warn!(context, "vg-member-added-received invalid.",); Ok(HandshakeMessage::Ignore) From 5ded8fb40014bafdb0ae505e593263d61f6c560e Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 28 Feb 2020 15:01:15 +0100 Subject: [PATCH 058/156] add vc-contact-confirm-received message needed for multi-device-verification --- src/securejoin.rs | 58 ++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/src/securejoin.rs b/src/securejoin.rs index a7167b0bf..b6f6e62b3 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -707,17 +707,21 @@ pub(crate) fn handle_securejoin_handshake( } secure_connection_established(context, contact_chat_id); context.bob.write().unwrap().expects = 0; - if join_vg { - // Bob -> Alice - send_handshake_msg( - context, - contact_chat_id, - "vg-member-added-received", - "", - None, - "", - ); - } + + // Bob -> Alice + send_handshake_msg( + context, + contact_chat_id, + if join_vg { + "vg-member-added-received" + } else { + "vc-contact-confirm-received" // only for observe_securejoin_on_other_device() + }, + "", + None, + "", + ); + context.bob.write().unwrap().status = 1; context.stop_ongoing(); Ok(if join_vg { @@ -726,7 +730,7 @@ pub(crate) fn handle_securejoin_handshake( HandshakeMessage::Ignore // "Done" deletes the message and breaks multi-device }) } - "vg-member-added-received" => { + "vg-member-added-received" | "vc-contact-confirm-received" => { /*========================================================== ==== Alice - the inviter side ==== ==== Step 8 in "Out-of-band verified groups" protocol ==== @@ -734,29 +738,31 @@ pub(crate) fn handle_securejoin_handshake( if let Ok(contact) = Contact::get_by_id(context, contact_id) { if contact.is_verified(context) == VerifiedStatus::Unverified { - warn!(context, "vg-member-added-received invalid.",); + warn!(context, "{} invalid.", step); return Ok(HandshakeMessage::Ignore); } - inviter_progress!(context, contact_id, 800); - inviter_progress!(context, contact_id, 1000); - let field_grpid = mime_message - .get(HeaderDef::SecureJoinGroup) - .map(|s| s.as_str()) - .unwrap_or_else(|| ""); - let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid) - .map_err(|err| { + if join_vg { + inviter_progress!(context, contact_id, 800); + inviter_progress!(context, contact_id, 1000); + let field_grpid = mime_message + .get(HeaderDef::SecureJoinGroup) + .map(|s| s.as_str()) + .unwrap_or_else(|| ""); + let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid) + .map_err(|err| { warn!(context, "Failed to lookup chat_id from grpid: {}", err); HandshakeError::ChatNotFound { group: field_grpid.to_string(), } })?; - context.call_cb(Event::MemberAdded { - chat_id: group_chat_id, - contact_id, - }); + context.call_cb(Event::MemberAdded { + chat_id: group_chat_id, + contact_id, + }); + } Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device } else { - warn!(context, "vg-member-added-received invalid.",); + warn!(context, "{} invalid.", step); Ok(HandshakeMessage::Ignore) } } From 278454287c1cc31263b5025ebd57b85207c51adc Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 28 Feb 2020 18:28:47 +0100 Subject: [PATCH 059/156] make sure, Secure-Join-Fingerprint is not accepted unencrypted --- src/mimeparser.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index cfd241d3b..513697ff9 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -92,6 +92,9 @@ impl MimeMessage { // init known headers with what mailparse provided us MimeMessage::merge_headers(&mut headers, &mail.headers); + // remove headers that are allowed _only_ in the encrypted part + headers.remove("secure-join-fingerprint"); + // Memory location for a possible decrypted message. let mail_raw; let mut gossipped_addr = Default::default(); @@ -1174,14 +1177,16 @@ mod tests { Content-Type: multipart/mixed; boundary=\"==break==\";\n\ Subject: outer-subject\n\ Secure-Join-Group: no\n\ - Test-Header: Bar\nChat-Version: 0.0\n\ + Secure-Join-Fingerprint: 123456\n\ + Test-Header: Bar\n\ + chat-VERSION: 0.0\n\ \n\ --==break==\n\ Content-Type: text/plain; protected-headers=\"v1\";\n\ Subject: inner-subject\n\ SecureBar-Join-Group: yes\n\ Test-Header: Xy\n\ - Chat-Version: 1.0\n\ + chat-VERSION: 1.0\n\ \n\ test1\n\ \n\ @@ -1200,12 +1205,17 @@ mod tests { // the following fields would bubble up // if the test would really use encryption for the protected part - // however, as this is not the case, the outer things stay valid + // however, as this is not the case, the outer things stay valid. + // for Chat-Version, also the case-insensivity is tested. assert_eq!(mimeparser.get_subject(), Some("outer-subject".into())); let of = mimeparser.get(HeaderDef::ChatVersion).unwrap(); assert_eq!(of, "0.0"); assert_eq!(mimeparser.parts.len(), 1); + + // make sure, headers that are only allowed in the encrypted part + // cannot be set from the outer part + assert!(mimeparser.get(HeaderDef::SecureJoinFingerprint).is_none()); } #[test] From 52442017e23da66837bc954842a1d516347e0af3 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 28 Feb 2020 18:50:02 +0100 Subject: [PATCH 060/156] mark contacts as verified if a secure-join for them was observed on another device --- src/securejoin.rs | 97 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 16 deletions(-) diff --git a/src/securejoin.rs b/src/securejoin.rs index b6f6e62b3..bb77e8335 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -390,8 +390,6 @@ pub(crate) fn handle_securejoin_handshake( mime_message: &MimeMessage, contact_id: u32, ) -> Result { - let own_fingerprint: String; - if contact_id <= DC_CONTACT_ID_LAST_SPECIAL { return Err(HandshakeError::SpecialContactId); } @@ -507,7 +505,7 @@ pub(crate) fn handle_securejoin_handshake( return Ok(HandshakeMessage::Ignore); } info!(context, "Fingerprint verified.",); - own_fingerprint = get_self_fingerprint(context).unwrap(); + let own_fingerprint = get_self_fingerprint(context).unwrap(); joiner_progress!(context, contact_id, 400); context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM; @@ -619,7 +617,14 @@ pub(crate) fn handle_securejoin_handshake( } } else { // Alice -> Bob - send_handshake_msg(context, contact_chat_id, "vc-contact-confirm", "", None, ""); + send_handshake_msg( + context, + contact_chat_id, + "vc-contact-confirm", + "", + Some(fingerprint.clone()), + "", + ); inviter_progress!(context, contact_id, 1000); } Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed) @@ -718,7 +723,7 @@ pub(crate) fn handle_securejoin_handshake( "vc-contact-confirm-received" // only for observe_securejoin_on_other_device() }, "", - None, + Some(scanned_fingerprint_of_alice), "", ); @@ -774,8 +779,6 @@ pub(crate) fn handle_securejoin_handshake( } /// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen. -/// currently, the message is only ignored, in the future, -/// we may mark peers as verified accross devices: /// /// in a multi-device-setup, there may be other devices that "see" the handshake messages. /// if the seen messages seen are self-sent messages encrypted+signed correctly with our key, @@ -792,17 +795,79 @@ pub(crate) fn handle_securejoin_handshake( /// the joining device has marked the peer as verified on vg-member-added/vc-contact-confirm /// before sending vg-member-added-received - so, if we observe vg-member-added-received, /// we can mark the peer as verified as well. -/// -/// to make this work, (a) some messages must not be deleted, -/// (b) we need a vc-contact-confirm-received message if bcc_self is set, -/// (c) we should make sure, we do not only rely on the unencrypted To:-header for identifying the peer -/// (in handle_securejoin_handshake() we have the oob information for that) pub(crate) fn observe_securejoin_on_other_device( - _context: &Context, - _mime_message: &MimeMessage, - _contact_id: u32, + context: &Context, + mime_message: &MimeMessage, + contact_id: u32, ) -> Result { - Ok(HandshakeMessage::Ignore) + if contact_id <= DC_CONTACT_ID_LAST_SPECIAL { + return Err(HandshakeError::SpecialContactId); + } + let step = mime_message + .get(HeaderDef::SecureJoin) + .ok_or(HandshakeError::NotSecureJoinMsg)?; + info!(context, "observing secure-join message \'{}\'", step); + + let contact_chat_id = + match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not) { + Ok((chat_id, blocked)) => { + if blocked != Blocked::Not { + chat_id.unblock(context); + } + chat_id + } + Err(err) => { + return Err(HandshakeError::NoChat { + contact_id, + cause: err, + }); + } + }; + + match step.as_str() { + "vg-member-added" + | "vc-contact-confirm" + | "vg-member-added-received" + | "vc-contact-confirm-received" => { + if !encrypted_and_signed( + context, + mime_message, + get_self_fingerprint(context).unwrap_or_default(), + ) { + could_not_establish_secure_connection( + context, + contact_chat_id, + "Message not encrypted correctly.", + ); + return Ok(HandshakeMessage::Ignore); + } + let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) { + Some(fp) => fp, + None => { + could_not_establish_secure_connection( + context, + contact_chat_id, + "Fingerprint not provided, please update Delta Chat on all your devices.", + ); + return Ok(HandshakeMessage::Ignore); + } + }; + if mark_peer_as_verified(context, fingerprint).is_err() { + could_not_establish_secure_connection( + context, + contact_chat_id, + format!("Fingerprint mismatch on observing {}.", step).as_ref(), + ); + return Ok(HandshakeMessage::Ignore); + } + Ok(if step.as_str() == "vg-member-added" { + HandshakeMessage::Propagate + } else { + HandshakeMessage::Ignore + }) + } + _ => Ok(HandshakeMessage::Ignore), + } } fn secure_connection_established(context: &Context, contact_chat_id: ChatId) { From 32bd6109e390f358a92b654622b6cc945d2d0c40 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:08:42 +0300 Subject: [PATCH 061/156] Update rust-toolchain and proptest It is required for stabilized subslice patterns. proptest 0.9.6 fixes compatibility with the latest rustc nightly. --- .github/workflows/code-quality.yml | 8 ++++---- Cargo.lock | 6 +++--- appveyor.yml | 2 +- ci_scripts/docker-coredeps/deps/build_rust.sh | 4 ++-- rust-toolchain | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index bda5d7001..c0084e9a2 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -10,7 +10,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2019-11-06 + toolchain: nightly-2020-04-10 override: true - uses: actions-rs/cargo@v1 with: @@ -25,7 +25,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2019-11-06 + toolchain: nightly-2020-04-10 override: true - run: rustup component add rustfmt - uses: actions-rs/cargo@v1 @@ -39,10 +39,10 @@ jobs: - uses: actions/checkout@v1 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2019-11-06 + toolchain: nightly-2020-04-10 components: clippy override: true - uses: actions-rs/clippy-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features \ No newline at end of file + args: --all-features diff --git a/Cargo.lock b/Cargo.lock index 012ee63e0..80c9b4d9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,7 +663,7 @@ dependencies = [ "pgp 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "proptest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)", + "proptest 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", "quick-xml 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)", "r2d2 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)", "r2d2_sqlite 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1958,7 +1958,7 @@ dependencies = [ [[package]] name = "proptest" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3329,7 +3329,7 @@ dependencies = [ "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" "checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" "checksum proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" -"checksum proptest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bf6147d103a7c9d7598f4105cf049b15c99e2ecd93179bf024f0fd349be5ada4" +"checksum proptest 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)" = "01c477819b845fe023d33583ebf10c9f62518c8d79a0960ba5c36d6ac8a55a5b" "checksum pulldown-cmark 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eef52fac62d0ea7b9b4dc7da092aa64ea7ec3d90af6679422d3d7e0e14b6ee15" "checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" "checksum quick-xml 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fe1e430bdcf30c9fdc25053b9c459bb1a4672af4617b6c783d7d91dc17c6bbb0" diff --git a/appveyor.yml b/appveyor.yml index 7276acac1..e47e1f971 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ environment: install: - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe - - rustup-init -yv --default-toolchain nightly-2019-07-10 + - rustup-init -yv --default-toolchain nightly-2020-04-10 - set PATH=%PATH%;%USERPROFILE%\.cargo\bin - rustc -vV - cargo -vV diff --git a/ci_scripts/docker-coredeps/deps/build_rust.sh b/ci_scripts/docker-coredeps/deps/build_rust.sh index 3e0a232c4..e614994b6 100755 --- a/ci_scripts/docker-coredeps/deps/build_rust.sh +++ b/ci_scripts/docker-coredeps/deps/build_rust.sh @@ -3,9 +3,9 @@ set -e -x # Install Rust -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-04-10 -y export PATH=/root/.cargo/bin:$PATH rustc --version # remove some 300-400 MB that we don't need for automated builds -rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/ +rm -rf /root/.rustup/toolchains/nightly-2020-04-10-x86_64-unknown-linux-gnu/share/ diff --git a/rust-toolchain b/rust-toolchain index 22e904890..66822d49c 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2019-11-06 +nightly-2020-04-10 From 24aa3c781b9ddd6da40bf5dc6903e9b6208e4567 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 02:23:54 +0300 Subject: [PATCH 062/156] Remove indexing in Aheader::from_str --- src/aheader.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/aheader.rs b/src/aheader.rs index 628de89f7..6dee60eeb 100644 --- a/src/aheader.rs +++ b/src/aheader.rs @@ -127,14 +127,10 @@ impl str::FromStr for Aheader { .split(';') .filter_map(|a| { let attribute: Vec<&str> = a.trim().splitn(2, '=').collect(); - if attribute.len() < 2 { - return None; + match &attribute[..] { + [key, value] => Some((key.trim().to_string(), value.trim().to_string())), + _ => None, } - - Some(( - attribute[0].trim().to_string(), - attribute[1].trim().to_string(), - )) }) .collect(); From 92f1e6da1e6b61f7ef86a0d7863349ceef3af2a4 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 02:35:00 +0300 Subject: [PATCH 063/156] get_next_media: enumerate() instead of indexing --- src/chat.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 78dc0175f..4197b4142 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1738,8 +1738,8 @@ pub fn get_next_media( msg_type2, msg_type3, ); - for i in 0..list.len() { - if curr_msg_id == list[i] { + for (i, msg_id) in list.iter().enumerate() { + if curr_msg_id == *msg_id { match direction { Direction::Forward => { if i + 1 < list.len() { From 5f574cf283816bb5b745a761ffb4d2e923fbc7bf Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 02:38:10 +0300 Subject: [PATCH 064/156] get_next_media: replace indexing with .get() --- src/chat.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 4197b4142..ffce31db3 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1743,12 +1743,12 @@ pub fn get_next_media( match direction { Direction::Forward => { if i + 1 < list.len() { - ret = Some(list[i + 1]); + ret = list.get(i + 1).copied(); } } Direction::Backward => { if i >= 1 { - ret = Some(list[i - 1]); + ret = list.get(i - 1).copied(); } } } From 1c21d4f35670a8b9dd51fa4f826d50bd8ba3ea33 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:09:22 +0300 Subject: [PATCH 065/156] constants: remove unnecessary parenthesis Rust warns about them now --- src/constants.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/constants.rs b/src/constants.rs index 50345caa1..56d289f49 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -197,13 +197,13 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000; pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000; /// if none of these flags are set, the default is chosen -pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL); +pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL; /// if none of these flags are set, the default is chosen pub const DC_LP_IMAP_SOCKET_FLAGS: i32 = - (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN); + DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN; /// if none of these flags are set, the default is chosen pub const DC_LP_SMTP_SOCKET_FLAGS: usize = - (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN); + DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN; // QR code scanning (view from Bob, the joiner) pub const DC_VC_AUTH_REQUIRED: i32 = 2; From d997bbc0812b6e64d6f19805c0d4985473881149 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:11:24 +0300 Subject: [PATCH 066/156] skip_forward_header: get rid of indexing with subslice patterns --- src/simplify.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index 142ca7349..2fb803907 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -67,14 +67,13 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) { /// Returns message body lines and a boolean indicating whether /// a message is forwarded or not. fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) { - if lines.len() >= 3 - && lines[0] == "---------- Forwarded message ----------" - && lines[1].starts_with("From: ") - && lines[2].is_empty() - { - (&lines[3..], true) - } else { - (lines, false) + match lines { + ["---------- Forwarded message ----------", first_line, "", rest @ ..] + if first_line.starts_with("From: ") => + { + (rest, true) + } + _ => (lines, false), } } From 324e5d0258c623af56b4d71ac18453379c271664 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:12:36 +0300 Subject: [PATCH 067/156] Use slice pattern to parse KML coordinates --- src/location.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/location.rs b/src/location.rs index 5d86b3028..4c16fcbe6 100644 --- a/src/location.rs +++ b/src/location.rs @@ -123,9 +123,9 @@ impl Kml { } } else if self.tag.contains(KmlTag::COORDINATES) { let parts = val.splitn(2, ',').collect::>(); - if parts.len() == 2 { - self.curr.longitude = parts[0].parse().unwrap_or_default(); - self.curr.latitude = parts[1].parse().unwrap_or_default(); + if let [longitude, latitude] = &parts[..] { + self.curr.longitude = longitude.parse().unwrap_or_default(); + self.curr.latitude = latitude.parse().unwrap_or_default(); } } } From 76b7e7408a643e5118c59ef1a15a2463ef072ebe Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:13:05 +0300 Subject: [PATCH 068/156] chatlist: remove indexing in get_{chat,msg}_id() --- src/chatlist.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/chatlist.rs b/src/chatlist.rs index cec5b520c..a681febfa 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -5,7 +5,7 @@ use crate::chat::*; use crate::constants::*; use crate::contact::*; use crate::context::*; -use crate::error::{ensure, Result}; +use crate::error::{bail, ensure, Result}; use crate::lot::Lot; use crate::message::{Message, MessageState, MsgId}; use crate::stock::StockMessage; @@ -275,18 +275,20 @@ impl Chatlist { /// /// To get the message object from the message ID, use dc_get_chat(). pub fn get_chat_id(&self, index: usize) -> ChatId { - if index >= self.ids.len() { - return ChatId::new(0); + match self.ids.get(index) { + Some((chat_id, _msg_id)) => *chat_id, + None => ChatId::new(0), } - self.ids[index].0 } /// Get a single message ID of a chatlist. /// /// To get the message object from the message ID, use dc_get_msg(). pub fn get_msg_id(&self, index: usize) -> Result { - ensure!(index < self.ids.len(), "Chatlist index out of range"); - Ok(self.ids[index].1) + match self.ids.get(index) { + Some((_chat_id, msg_id)) => Ok(*msg_id), + None => bail!("Chatlist index out of range"), + } } /// Get a summary for a chatlist index. From db5b5d321b15df7fe8a851caa29e4dcc347a83df Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:14:54 +0300 Subject: [PATCH 069/156] clippy: remove redundant imports --- deltachat_derive/src/lib.rs | 1 - src/configure/auto_mozilla.rs | 1 - src/configure/auto_outlook.rs | 1 - src/contact.rs | 1 - src/dehtml.rs | 1 - src/location.rs | 1 - 6 files changed, 6 deletions(-) diff --git a/deltachat_derive/src/lib.rs b/deltachat_derive/src/lib.rs index f911b9fa9..664581464 100644 --- a/deltachat_derive/src/lib.rs +++ b/deltachat_derive/src/lib.rs @@ -3,7 +3,6 @@ extern crate proc_macro; use crate::proc_macro::TokenStream; use quote::quote; -use syn; // For now, assume (not check) that these macroses are applied to enum without // data. If this assumption is violated, compiler error will point to diff --git a/src/configure/auto_mozilla.rs b/src/configure/auto_mozilla.rs index 348a24fd9..3a25c280b 100644 --- a/src/configure/auto_mozilla.rs +++ b/src/configure/auto_mozilla.rs @@ -1,7 +1,6 @@ //! # Thunderbird's Autoconfiguration implementation //! //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */ -use quick_xml; use quick_xml::events::{BytesEnd, BytesStart, BytesText}; use crate::constants::*; diff --git a/src/configure/auto_outlook.rs b/src/configure/auto_outlook.rs index 1e8cbba10..96b22e2ab 100644 --- a/src/configure/auto_outlook.rs +++ b/src/configure/auto_outlook.rs @@ -1,6 +1,5 @@ //! Outlook's Autodiscover -use quick_xml; use quick_xml::events::BytesEnd; use crate::constants::*; diff --git a/src/contact.rs b/src/contact.rs index c74ce8db9..b7ad24015 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -4,7 +4,6 @@ use std::path::PathBuf; use deltachat_derive::*; use itertools::Itertools; -use rusqlite; use crate::aheader::EncryptPreference; use crate::chat::ChatId; diff --git a/src/dehtml.rs b/src/dehtml.rs index 5639e3af4..71864b9ec 100644 --- a/src/dehtml.rs +++ b/src/dehtml.rs @@ -3,7 +3,6 @@ //! A module to remove HTML tags from the email text use lazy_static::lazy_static; -use quick_xml; use quick_xml::events::{BytesEnd, BytesStart, BytesText}; lazy_static! { diff --git a/src/location.rs b/src/location.rs index 4c16fcbe6..c77396b62 100644 --- a/src/location.rs +++ b/src/location.rs @@ -1,7 +1,6 @@ //! Location handling use bitflags::bitflags; -use quick_xml; use quick_xml::events::{BytesEnd, BytesStart, BytesText}; use crate::chat::{self, ChatId}; From 3a91c87c73a584ef5ac21a5b61779433a82950b2 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:16:07 +0300 Subject: [PATCH 070/156] clippy: use as_deref() --- src/chat.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chat.rs b/src/chat.rs index ffce31db3..fd90d2c7d 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -324,7 +324,7 @@ impl ChatId { time(), msg.viewtype, MessageState::OutDraft, - msg.text.as_ref().map(String::as_str).unwrap_or(""), + msg.text.as_deref().unwrap_or(""), msg.param.to_string(), 1, ], From 0327000f8dcd2f6d3c019a707daa9a3ba5863825 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:18:12 +0300 Subject: [PATCH 071/156] squash_attachment_parts: use slice patterns --- src/mimeparser.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 513697ff9..e498e1453 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -203,10 +203,12 @@ impl MimeMessage { /// Delta Chat sends attachments, such as images, in two-part messages, with the first message /// containing an explanation. If such a message is detected, first part can be safely dropped. fn squash_attachment_parts(&mut self) { - if self.has_chat_version() && self.parts.len() == 2 { + if !self.has_chat_version() { + return; + } + + if let [textpart, filepart] = &self.parts[..] { let need_drop = { - let textpart = &self.parts[0]; - let filepart = &self.parts[1]; textpart.typ == Viewtype::Text && (filepart.typ == Viewtype::Image || filepart.typ == Viewtype::Gif From cb92579461f65172d4993655000683875a8038d1 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:24:36 +0300 Subject: [PATCH 072/156] Use slice patterns in EmailAddress::from_str() --- src/dc_tools.rs | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 19c7a6746..e510be1d1 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -494,24 +494,25 @@ impl FromStr for EmailAddress { ensure!(!input.is_empty(), "empty string is not valid"); let parts: Vec<&str> = input.rsplitn(2, '@').collect(); - ensure!(parts.len() > 1, "missing '@' character"); - let local = parts[1]; - let domain = parts[0]; + match &parts[..] { + [domain, local] => { + ensure!( + !local.is_empty(), + "empty string is not valid for local part" + ); + ensure!(domain.len() > 3, "domain is too short"); - ensure!( - !local.is_empty(), - "empty string is not valid for local part" - ); - ensure!(domain.len() > 3, "domain is too short"); + let dot = domain.find('.'); + ensure!(dot.is_some(), "invalid domain"); + ensure!(dot.unwrap() < domain.len() - 2, "invalid domain"); - let dot = domain.find('.'); - ensure!(dot.is_some(), "invalid domain"); - ensure!(dot.unwrap() < domain.len() - 2, "invalid domain"); - - Ok(EmailAddress { - local: local.to_string(), - domain: domain.to_string(), - }) + Ok(EmailAddress { + local: (*local).to_string(), + domain: (*domain).to_string(), + }) + } + _ => bail!("missing '@' character"), + } } } From 9fcb30ac334edc7cb17d40c05fccc5df606e3adb Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:29:36 +0300 Subject: [PATCH 073/156] Remove indexing in chatlist.get_summary() --- src/chatlist.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/chatlist.rs b/src/chatlist.rs index a681febfa..f6ed9a003 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -312,25 +312,27 @@ impl Chatlist { // Also, sth. as "No messages" would not work if the summary comes from a message. let mut ret = Lot::new(); - if index >= self.ids.len() { - ret.text2 = Some("ErrBadChatlistIndex".to_string()); - return ret; - } + let (chat_id, lastmsg_id) = match self.ids.get(index) { + Some(ids) => ids, + None => { + ret.text2 = Some("ErrBadChatlistIndex".to_string()); + return ret; + } + }; let chat_loaded: Chat; let chat = if let Some(chat) = chat { chat - } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) { + } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) { chat_loaded = chat; &chat_loaded } else { return ret; }; - let lastmsg_id = self.ids[index].1; let mut lastcontact = None; - let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) { + let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) { if lastmsg.from_id != DC_CONTACT_ID_SELF && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup) { From ec089faf3a6c0b4dc9a2503f3d02d1b2b361af5c Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:35:00 +0300 Subject: [PATCH 074/156] configure/auto_mozilla: remove indexing --- src/configure/auto_mozilla.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/configure/auto_mozilla.rs b/src/configure/auto_mozilla.rs index 3a25c280b..dbadc9543 100644 --- a/src/configure/auto_mozilla.rs +++ b/src/configure/auto_mozilla.rs @@ -43,11 +43,11 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result { reader.trim_text(true); // Split address into local part and domain part. - let p = in_emailaddr - .find('@') - .ok_or_else(|| Error::InvalidEmailAddress(in_emailaddr.to_string()))?; - let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p); - let in_emaildomain = &in_emaildomain[1..]; + let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect(); + let (in_emaillocalpart, in_emaildomain) = match &parts[..] { + [domain, local] => (local, domain), + _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())), + }; let mut moz_ac = MozAutoconfigure { in_emailaddr, From e909b8199b98016a9c9f0b5f48168285ea0a4454 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:48:20 +0300 Subject: [PATCH 075/156] lot: use as_deref() --- src/lot.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lot.rs b/src/lot.rs index c1f27eae0..480562aa4 100644 --- a/src/lot.rs +++ b/src/lot.rs @@ -40,11 +40,11 @@ impl Lot { } pub fn get_text1(&self) -> Option<&str> { - self.text1.as_ref().map(|s| s.as_str()) + self.text1.as_deref() } pub fn get_text2(&self) -> Option<&str> { - self.text2.as_ref().map(|s| s.as_str()) + self.text2.as_deref() } pub fn get_text1_meaning(&self) -> Meaning { From daf40fde8219e6950beb61fa1c3f5c45ef778b1b Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 28 Mar 2020 03:51:02 +0300 Subject: [PATCH 076/156] mimefactory: use .next() instead of .nth(0) --- src/mimefactory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 58217face..127a23111 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -1095,7 +1095,7 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool { fn render_rfc724_mid(rfc724_mid: &str) -> String { let rfc724_mid = rfc724_mid.trim().to_string(); - if rfc724_mid.chars().nth(0).unwrap_or_default() == '<' { + if rfc724_mid.chars().next().unwrap_or_default() == '<' { rfc724_mid } else { format!("<{}>", rfc724_mid) From 1760740a4c9d11c1b42e39ef61cc741c9b3e8a97 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Mon, 13 Apr 2020 23:10:30 +0300 Subject: [PATCH 077/156] Parse inline attachments from non-DC email clients Some mobile email clients, such as apple mail, attach or inline images after description, just like Delta Chat. It is better to display them instead of ignoring. --- src/mimeparser.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index e498e1453..36bec66e1 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -203,10 +203,6 @@ impl MimeMessage { /// Delta Chat sends attachments, such as images, in two-part messages, with the first message /// containing an explanation. If such a message is detected, first part can be safely dropped. fn squash_attachment_parts(&mut self) { - if !self.has_chat_version() { - return; - } - if let [textpart, filepart] = &self.parts[..] { let need_drop = { textpart.typ == Viewtype::Text @@ -1509,6 +1505,8 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== Some("Mail with inline attachment".to_string()) ); - assert_eq!(message.parts.len(), 2); + assert_eq!(message.parts.len(), 1); + assert_eq!(message.parts[0].typ, Viewtype::File); + assert_eq!(message.parts[0].msg, "Hello!"); } } From e85cdc8c9f922867945e1995eb9ce8970f68647a Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Mon, 13 Apr 2020 23:12:02 +0300 Subject: [PATCH 078/156] mimeparser: test parsing of inline images --- src/mimeparser.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 36bec66e1..8e0bd0004 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -1509,4 +1509,48 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg== assert_eq!(message.parts[0].typ, Viewtype::File); assert_eq!(message.parts[0].msg, "Hello!"); } + + #[test] + fn parse_inline_image() { + let context = dummy_context(); + let raw = br#"Message-ID: +From: foo +Subject: example +To: bar@example.org +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="--11019878869865180" + +----11019878869865180 +Content-Type: text/plain; charset=utf-8 + +Test + +----11019878869865180 +Content-Type: image/jpeg; + name="JPEG_filename.jpg" +Content-Transfer-Encoding: base64 +Content-Disposition: inline; + filename="JPEG_filename.jpg" + +ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG +acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT +/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal +n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C +xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2 +rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU +8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI +K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4 +CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= + + +----11019878869865180-- +"#; + + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap(); + assert_eq!(message.get_subject(), Some("example".to_string())); + + assert_eq!(message.parts.len(), 1); + assert_eq!(message.parts[0].typ, Viewtype::Image); + assert_eq!(message.parts[0].msg, "Test"); + } } From 13dd88b7adcd48aff4f569f5500be46dc2ce1eff Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Wed, 15 Apr 2020 02:36:29 +0300 Subject: [PATCH 079/156] mimeparser: display inline images received from Thunderbird --- src/mimeparser.rs | 83 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 10 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 8e0bd0004..cc3c4198b 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -472,7 +472,9 @@ impl MimeMessage { apple mail: "plaintext" as an alternative to "html+PDF attachment") */ (mime::MULTIPART, "alternative") => { for cur_data in &mail.subparts { - if get_mime_type(cur_data)?.0 == "multipart/mixed" { + if get_mime_type(cur_data)?.0 == "multipart/mixed" + || get_mime_type(cur_data)?.0 == "multipart/related" + { any_part_added = self.parse_mime_recursive(context, cur_data)?; break; } @@ -496,15 +498,6 @@ impl MimeMessage { } } } - (mime::MULTIPART, "related") => { - /* add the "root part" - the other parts may be referenced which is - not interesting for us (eg. embedded images) we assume he "root part" - being the first one, which may not be always true ... - however, most times it seems okay. */ - if let Some(first) = mail.subparts.iter().next() { - any_part_added = self.parse_mime_recursive(context, first)?; - } - } (mime::MULTIPART, "encrypted") => { // we currently do not try to decrypt non-autocrypt messages // at all. If we see an encrypted part, we set @@ -1553,4 +1546,74 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= assert_eq!(message.parts[0].typ, Viewtype::Image); assert_eq!(message.parts[0].msg, "Test"); } + + #[test] + fn parse_thunderbird_html_embedded_image() { + let context = dummy_context(); + let raw = br#"To: Alice +From: Bob +Subject: Test subject +Message-ID: +User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 + Thunderbird/68.7.0 +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="------------779C1631600DF3DB8C02E53A" +Content-Language: en-US + +This is a multi-part message in MIME format. +--------------779C1631600DF3DB8C02E53A +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 7bit + +Test + + +--------------779C1631600DF3DB8C02E53A +Content-Type: multipart/related; + boundary="------------10CC6C2609EB38DA782C5CA9" + + +--------------10CC6C2609EB38DA782C5CA9 +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: 7bit + + + + + + +Test
+

+ + + +--------------10CC6C2609EB38DA782C5CA9 +Content-Type: image/png; + name="1.png" +Content-Transfer-Encoding: base64 +Content-ID: +Content-Disposition: inline; + filename="1.png" + +ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG +acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT +/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal +n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C +xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2 +rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU +8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI +K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4 +CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= +--------------10CC6C2609EB38DA782C5CA9-- + +--------------779C1631600DF3DB8C02E53A--"#; + + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap(); + assert_eq!(message.get_subject(), Some("Test subject".to_string())); + + assert_eq!(message.parts.len(), 1); + assert_eq!(message.parts[0].typ, Viewtype::Image); + assert_eq!(message.parts[0].msg, "Test"); + } } From 857a384d8b8ce6155083476a61f850f03b32654f Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Wed, 15 Apr 2020 22:27:39 +0300 Subject: [PATCH 080/156] Switch to nightly-2020-03-12 --- .github/workflows/code-quality.yml | 6 +++--- appveyor.yml | 2 +- ci_scripts/docker-coredeps/deps/build_rust.sh | 4 ++-- rust-toolchain | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index c0084e9a2..90bf4fdc3 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -10,7 +10,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2020-04-10 + toolchain: nightly-2020-03-12 override: true - uses: actions-rs/cargo@v1 with: @@ -25,7 +25,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: profile: minimal - toolchain: nightly-2020-04-10 + toolchain: nightly-2020-03-12 override: true - run: rustup component add rustfmt - uses: actions-rs/cargo@v1 @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions-rs/toolchain@v1 with: - toolchain: nightly-2020-04-10 + toolchain: nightly-2020-03-12 components: clippy override: true - uses: actions-rs/clippy-check@v1 diff --git a/appveyor.yml b/appveyor.yml index e47e1f971..dfb5274aa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ environment: install: - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe - - rustup-init -yv --default-toolchain nightly-2020-04-10 + - rustup-init -yv --default-toolchain nightly-2020-03-12 - set PATH=%PATH%;%USERPROFILE%\.cargo\bin - rustc -vV - cargo -vV diff --git a/ci_scripts/docker-coredeps/deps/build_rust.sh b/ci_scripts/docker-coredeps/deps/build_rust.sh index e614994b6..239040eb7 100755 --- a/ci_scripts/docker-coredeps/deps/build_rust.sh +++ b/ci_scripts/docker-coredeps/deps/build_rust.sh @@ -3,9 +3,9 @@ set -e -x # Install Rust -curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-04-10 -y +curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y export PATH=/root/.cargo/bin:$PATH rustc --version # remove some 300-400 MB that we don't need for automated builds -rm -rf /root/.rustup/toolchains/nightly-2020-04-10-x86_64-unknown-linux-gnu/share/ +rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/ diff --git a/rust-toolchain b/rust-toolchain index 66822d49c..7b70b3322 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2020-04-10 +nightly-2020-03-12 From cc6ce72f6e97fdcfdb2462ebf920df823e0d6405 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Thu, 16 Apr 2020 21:48:51 +0200 Subject: [PATCH 081/156] remove unused dc_get_version_str() api --- deltachat-ffi/deltachat.h | 1 - deltachat-ffi/src/lib.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index bbaf3ecc1..39963644b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4537,7 +4537,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); #define DC_EVENT_DATA2_IS_STRING(e) ((e)>=100 && (e)<=499) #define DC_EVENT_RETURNS_INT(e) ((e)==DC_EVENT_IS_OFFLINE) // not used anymore #define DC_EVENT_RETURNS_STRING(e) ((e)==DC_EVENT_GET_STRING) // not used anymore -char* dc_get_version_str (void); // deprecated void dc_array_add_id (dc_array_t*, uint32_t); // deprecated #define dc_archive_chat(a,b,c) dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore #define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0) // not used anymore diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 664866300..86fe8a651 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -484,11 +484,6 @@ pub unsafe extern "C" fn dc_get_oauth2_url( .unwrap_or_else(|_| ptr::null_mut()) } -#[no_mangle] -pub unsafe extern "C" fn dc_get_version_str() -> *mut libc::c_char { - context::get_version_str().strdup() -} - #[no_mangle] pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) { if context.is_null() { From 0b6b8ced922b4fdf75233b8e9f4508dbac78889c Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Thu, 16 Apr 2020 22:06:47 +0200 Subject: [PATCH 082/156] remove unused dc_chat_get_subtitle() api --- CHANGELOG.md | 5 +++++ deltachat-ffi/deltachat.h | 16 --------------- deltachat-ffi/src/lib.rs | 13 ------------ python/src/deltachat/chat.py | 6 ------ python/src/deltachat/const.py | 3 --- python/tests/test_account.py | 1 - src/chat.rs | 37 ----------------------------------- src/constants.rs | 3 --- src/dc_receive_imf.rs | 6 +++--- src/stock.rs | 18 +++++------------ 10 files changed, 13 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cecfd6723..0468cd6fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## UNRELEASED + +- removed api: dc_chat_get_subtitle(), dc_get_version_str() + + ## 1.28.0 - new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist() diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 39963644b..afe2bc213 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -2827,19 +2827,6 @@ int dc_chat_get_type (const dc_chat_t* chat); char* dc_chat_get_name (const dc_chat_t* chat); -/* - * Get a subtitle for a chat. The subtitle is eg. the email-address or the - * number of group members. - * - * Deprecated function. Subtitles should be created in the ui - * where plural forms and other specials can be handled more gracefully. - * - * @param chat The chat object to calulate the subtitle for. - * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL. - */ -char* dc_chat_get_subtitle (const dc_chat_t* chat); - - /** * Get the chat's profile image. * For groups, this is the image set by any group member @@ -4667,8 +4654,6 @@ void dc_array_add_id (dc_array_t*, uint32_t); // depreca #define DC_STR_NOMESSAGES 1 #define DC_STR_SELF 2 #define DC_STR_DRAFT 3 -#define DC_STR_MEMBER 4 -#define DC_STR_CONTACT 6 #define DC_STR_VOICEMESSAGE 7 #define DC_STR_DEADDROP 8 #define DC_STR_IMAGE 9 @@ -4700,7 +4685,6 @@ void dc_array_add_id (dc_array_t*, uint32_t); // depreca #define DC_STR_STARREDMSGS 41 #define DC_STR_AC_SETUP_MSG_SUBJECT 42 #define DC_STR_AC_SETUP_MSG_BODY 43 -#define DC_STR_SELFTALK_SUBTITLE 50 #define DC_STR_CANNOT_LOGIN 60 #define DC_STR_SERVER_RESPONSE 61 #define DC_STR_MSGACTIONBYUSER 62 diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 86fe8a651..8dd8e11f0 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -2436,19 +2436,6 @@ pub unsafe extern "C" fn dc_chat_get_name(chat: *mut dc_chat_t) -> *mut libc::c_ ffi_chat.chat.get_name().strdup() } -#[no_mangle] -pub unsafe extern "C" fn dc_chat_get_subtitle(chat: *mut dc_chat_t) -> *mut libc::c_char { - if chat.is_null() { - eprintln!("ignoring careless call to dc_chat_get_subtitle()"); - return "".strdup(); - } - let ffi_chat = &*chat; - let ffi_context: &ContextWrapper = &*ffi_chat.context; - ffi_context - .with_inner(|ctx| ffi_chat.chat.get_subtitle(ctx).strdup()) - .unwrap_or_else(|_| "".strdup()) -} - #[no_mangle] pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut libc::c_char { if chat.is_null() { diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index ea84c378c..0c07dae11 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -415,12 +415,6 @@ class Chat(object): """ return lib.dc_chat_get_color(self._dc_chat) - def get_subtitle(self): - """return the subtitle of the chat - :returns: the subtitle - """ - return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat)) - # ------ location streaming API ------------------------------ def is_sending_locations(self): diff --git a/python/src/deltachat/const.py b/python/src/deltachat/const.py index 249be9804..13b638bb8 100644 --- a/python/src/deltachat/const.py +++ b/python/src/deltachat/const.py @@ -117,8 +117,6 @@ DC_CHAT_VISIBILITY_PINNED = 2 DC_STR_NOMESSAGES = 1 DC_STR_SELF = 2 DC_STR_DRAFT = 3 -DC_STR_MEMBER = 4 -DC_STR_CONTACT = 6 DC_STR_VOICEMESSAGE = 7 DC_STR_DEADDROP = 8 DC_STR_IMAGE = 9 @@ -150,7 +148,6 @@ DC_STR_ARCHIVEDCHATS = 40 DC_STR_STARREDMSGS = 41 DC_STR_AC_SETUP_MSG_SUBJECT = 42 DC_STR_AC_SETUP_MSG_BODY = 43 -DC_STR_SELFTALK_SUBTITLE = 50 DC_STR_CANNOT_LOGIN = 60 DC_STR_SERVER_RESPONSE = 61 DC_STR_MSGACTIONBYUSER = 62 diff --git a/python/tests/test_account.py b/python/tests/test_account.py index c8eeddc6d..4cbbab238 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -221,7 +221,6 @@ class TestOfflineChat: # assert d["param"] == chat.param assert d["color"] == chat.get_color() assert d["profile_image"] == "" if chat.get_profile_image() is None else chat.get_profile_image() - assert d["subtitle"] == chat.get_subtitle() assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft() def test_group_chat_creation_with_translation(self, ac1): diff --git a/src/chat.rs b/src/chat.rs index fd90d2c7d..8685e3743 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -604,38 +604,6 @@ impl Chat { &self.name } - pub fn get_subtitle(&self, context: &Context) -> String { - // returns either the address or the number of chat members - - if self.typ == Chattype::Single && self.param.exists(Param::Selftalk) { - return context.stock_str(StockMessage::SelfTalkSubTitle).into(); - } - - if self.typ == Chattype::Single { - return context - .sql - .query_get_value( - context, - "SELECT c.addr - FROM chats_contacts cc - LEFT JOIN contacts c ON c.id=cc.contact_id - WHERE cc.chat_id=?;", - params![self.id], - ) - .unwrap_or_else(|| "Err".into()); - } - - if self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup { - if self.id.is_deaddrop() { - return context.stock_str(StockMessage::DeadDrop).into(); - } - let cnt = get_chat_contact_cnt(context, self.id); - return context.stock_string_repl_int(StockMessage::Member, cnt as i32); - } - - "Err".to_string() - } - pub fn get_profile_image(&self, context: &Context) -> Option { if let Some(image_rel) = self.param.get(Param::ProfileImage) { if !image_rel.is_empty() { @@ -693,7 +661,6 @@ impl Chat { is_sending_locations: self.is_sending_locations, color: self.get_color(context), profile_image: self.get_profile_image(context).unwrap_or_else(PathBuf::new), - subtitle: self.get_subtitle(context), draft, is_muted: self.is_muted(), }) @@ -1036,9 +1003,6 @@ pub struct ChatInfo { /// currently. pub profile_image: PathBuf, - /// Subtitle for the chat. - pub subtitle: String, - /// The draft message text. /// /// If the chat has not draft this is an empty string. @@ -2653,7 +2617,6 @@ mod tests { "is_sending_locations": false, "color": 15895624, "profile_image": "", - "subtitle": "bob@example.com", "draft": "", "is_muted": false } diff --git a/src/constants.rs b/src/constants.rs index 56d289f49..64ce2cd59 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -314,8 +314,6 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated; const DC_STR_NOMESSAGES: usize = 1; const DC_STR_SELF: usize = 2; const DC_STR_DRAFT: usize = 3; -const DC_STR_MEMBER: usize = 4; -const DC_STR_CONTACT: usize = 6; const DC_STR_VOICEMESSAGE: usize = 7; const DC_STR_DEADDROP: usize = 8; const DC_STR_IMAGE: usize = 9; @@ -347,7 +345,6 @@ const DC_STR_ARCHIVEDCHATS: usize = 40; const DC_STR_STARREDMSGS: usize = 41; const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42; const DC_STR_AC_SETUP_MSG_BODY: usize = 43; -const DC_STR_SELFTALK_SUBTITLE: usize = 50; const DC_STR_CANNOT_LOGIN: usize = 60; const DC_STR_SERVER_RESPONSE: usize = 61; const DC_STR_MSGACTIONBYUSER: usize = 62; diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index e5ef249dd..80654c561 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -1192,9 +1192,9 @@ fn create_or_lookup_adhoc_group( return Ok((ChatId::new(0), Blocked::Not)); } // use subject as initial chat name - let grpname = mime_parser.get_subject().unwrap_or_else(|| { - context.stock_string_repl_int(StockMessage::Member, member_ids.len() as i32) - }); + let grpname = mime_parser + .get_subject() + .unwrap_or("Unnamed group".to_string()); // create group record let new_chat_id: ChatId = create_group_record( diff --git a/src/stock.rs b/src/stock.rs index d536d522e..c5ff765f0 100644 --- a/src/stock.rs +++ b/src/stock.rs @@ -35,12 +35,6 @@ pub enum StockMessage { #[strum(props(fallback = "Draft"))] Draft = 3, - #[strum(props(fallback = "%1$s member(s)"))] - Member = 4, - - #[strum(props(fallback = "%1$s contact(s)"))] - Contact = 6, - #[strum(props(fallback = "Voice message"))] VoiceMessage = 7, @@ -136,9 +130,6 @@ pub enum StockMessage { ))] AcSetupMsgBody = 43, - #[strum(props(fallback = "Messages I sent to myself"))] - SelfTalkSubTitle = 50, - #[strum(props(fallback = "Cannot login as %1$s."))] CannotLogin = 60, @@ -430,8 +421,9 @@ mod tests { let t = dummy_context(); // uses %1$s substitution assert_eq!( - t.ctx.stock_string_repl_str(StockMessage::Member, "42"), - "42 member(s)" + t.ctx + .stock_string_repl_str(StockMessage::MsgAddMember, "Foo"), + "Member Foo added." ); // We have no string using %1$d to test... } @@ -440,8 +432,8 @@ mod tests { fn test_stock_string_repl_int() { let t = dummy_context(); assert_eq!( - t.ctx.stock_string_repl_int(StockMessage::Member, 42), - "42 member(s)" + t.ctx.stock_string_repl_int(StockMessage::MsgAddMember, 42), + "Member 42 added." ); } From e1903edd04e1f4293b4812f3f5b28763630fbe6c Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Thu, 16 Apr 2020 22:11:54 +0200 Subject: [PATCH 083/156] remove unused dc_array_add_id() api --- CHANGELOG.md | 2 +- deltachat-ffi/deltachat.h | 1 - deltachat-ffi/src/lib.rs | 10 ---------- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0468cd6fc..b06133e37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## UNRELEASED -- removed api: dc_chat_get_subtitle(), dc_get_version_str() +- removed api: dc_chat_get_subtitle(), dc_get_version_str(), dc_array_add_id() ## 1.28.0 diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index afe2bc213..a6f517b9b 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4524,7 +4524,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); #define DC_EVENT_DATA2_IS_STRING(e) ((e)>=100 && (e)<=499) #define DC_EVENT_RETURNS_INT(e) ((e)==DC_EVENT_IS_OFFLINE) // not used anymore #define DC_EVENT_RETURNS_STRING(e) ((e)==DC_EVENT_GET_STRING) // not used anymore -void dc_array_add_id (dc_array_t*, uint32_t); // deprecated #define dc_archive_chat(a,b,c) dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore #define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0) // not used anymore diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 8dd8e11f0..f32f95e26 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -2099,16 +2099,6 @@ pub unsafe extern "C" fn dc_array_unref(a: *mut dc_array::dc_array_t) { Box::from_raw(a); } -#[no_mangle] -pub unsafe extern "C" fn dc_array_add_id(array: *mut dc_array_t, item: libc::c_uint) { - if array.is_null() { - eprintln!("ignoring careless call to dc_array_add_id()"); - return; - } - - (*array).add_id(item); -} - #[no_mangle] pub unsafe extern "C" fn dc_array_get_cnt(array: *const dc_array_t) -> libc::size_t { if array.is_null() { From 1eaef94c7157fc6b0e9848973e8326658cb5a53a Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Thu, 16 Apr 2020 22:49:34 +0200 Subject: [PATCH 084/156] make clippy happy --- src/dc_receive_imf.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 80654c561..48b8df02b 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -1194,7 +1194,7 @@ fn create_or_lookup_adhoc_group( // use subject as initial chat name let grpname = mime_parser .get_subject() - .unwrap_or("Unnamed group".to_string()); + .unwrap_or_else(|| "Unnamed group".to_string()); // create group record let new_chat_id: ChatId = create_group_record( From b075a73222851f7ff1d7351f517c113d6f5278be Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Fri, 17 Apr 2020 16:45:18 +0200 Subject: [PATCH 085/156] adapt doc wrt events not longer having a return value --- deltachat-ffi/deltachat.h | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index a6f517b9b..155c7a049 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -41,7 +41,7 @@ typedef struct _dc_provider dc_provider_t; * uintptr_t event_handler_func(dc_context_t* context, int event, * uintptr_t data1, uintptr_t data2) * { - * return 0; // for unhandled events, it is always safe to return 0 + * return 0; * } * * dc_context_t* context = dc_context_new(event_handler_func, NULL, NULL); @@ -208,7 +208,7 @@ typedef struct _dc_provider dc_provider_t; * @param event one of the @ref DC_EVENT constants * @param data1 depends on the event parameter * @param data2 depends on the event parameter - * @return return 0 unless stated otherwise in the event parameter documentation + * @return events do not expect a return value, just always return 0 */ typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t data1, uintptr_t data2); @@ -229,7 +229,7 @@ typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t * otherwise! * - The callback SHOULD return _fast_, for GUI updates etc. you should * post yourself an asynchronous message to your GUI thread, if needed. - * - If not mentioned otherweise, the callback should return 0. + * - events do not expect a return value, just always return 0. * @param userdata can be used by the client for any purpuse. He finds it * later in dc_get_userdata(). * @param os_name is only for decorative use @@ -4147,8 +4147,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * These constants are used as events * reported to the callback given to dc_context_new(). - * If you do not want to handle an event, it is always safe to return 0, - * so there is no need to add a "case" for every event. * * @addtogroup DC_EVENT * @{ @@ -4163,7 +4161,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_INFO 100 @@ -4174,7 +4171,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_SMTP_CONNECTED 101 @@ -4185,7 +4181,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_IMAP_CONNECTED 102 @@ -4195,7 +4190,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_SMTP_MESSAGE_SENT 103 @@ -4205,7 +4199,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_IMAP_MESSAGE_DELETED 104 @@ -4215,7 +4208,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_IMAP_MESSAGE_MOVED 105 @@ -4225,7 +4217,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) folder name. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_IMAP_FOLDER_EMPTIED 106 @@ -4235,7 +4226,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) path name * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_NEW_BLOB_FILE 150 @@ -4245,7 +4235,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) path name * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_DELETED_BLOB_FILE 151 @@ -4258,7 +4247,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 0 * @param data2 (const char*) Warning string in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_WARNING 300 @@ -4281,7 +4269,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * Some error strings are taken from dc_set_stock_translation(), * however, most error strings will be in english language. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_ERROR 400 @@ -4305,7 +4292,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * 0=subsequent network error, should be logged only * @param data2 (const char*) Error string, always set, never NULL. * Must not be unref'd or modified and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_ERROR_NETWORK 401 @@ -4321,7 +4307,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data2 (const char*) Info string in english language. * Must not be unref'd or modified * and is valid only until the callback returns. - * @return 0 */ #define DC_EVENT_ERROR_SELF_NOT_IN_GROUP 410 @@ -4335,7 +4320,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id for single added messages * @param data2 (int) msg_id for single added messages - * @return 0 */ #define DC_EVENT_MSGS_CHANGED 2000 @@ -4348,7 +4332,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 (int) msg_id - * @return 0 */ #define DC_EVENT_INCOMING_MSG 2005 @@ -4359,7 +4342,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 (int) msg_id - * @return 0 */ #define DC_EVENT_MSG_DELIVERED 2010 @@ -4370,7 +4352,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 (int) msg_id - * @return 0 */ #define DC_EVENT_MSG_FAILED 2012 @@ -4381,7 +4362,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 (int) msg_id - * @return 0 */ #define DC_EVENT_MSG_READ 2015 @@ -4394,7 +4374,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 0 - * @return 0 */ #define DC_EVENT_CHAT_MODIFIED 2020 @@ -4404,7 +4383,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected. * @param data2 0 - * @return 0 */ #define DC_EVENT_CONTACTS_CHANGED 2030 @@ -4417,7 +4395,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * If the locations of several contacts have been changed, * eg. after calling dc_delete_all_locations(), this parameter is set to 0. * @param data2 0 - * @return 0 */ #define DC_EVENT_LOCATION_CHANGED 2035 @@ -4427,7 +4404,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done * @param data2 0 - * @return 0 */ #define DC_EVENT_CONFIGURE_PROGRESS 2041 @@ -4437,7 +4413,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done * @param data2 0 - * @return 0 */ #define DC_EVENT_IMEX_PROGRESS 2051 @@ -4452,7 +4427,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data1 (const char*) Path and file name. * Must not be unref'd or modified and is valid only until the callback returns. * @param data2 0 - * @return 0 */ #define DC_EVENT_IMEX_FILE_WRITTEN 2052 @@ -4470,7 +4444,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * 600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified". * 800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol. * 1000=Protocol finished for this contact. - * @return 0 */ #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS 2060 @@ -4486,7 +4459,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * @param data2 (int) Progress as: * 400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself." * (Bob has verified alice and waits until Alice does the same for him) - * @return 0 */ #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS 2061 @@ -4496,7 +4468,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 (int) contact_id - * @return 0 */ #define DC_EVENT_MEMBER_ADDED 2062 @@ -4505,7 +4476,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); * * @param data1 (int) chat_id * @param data2 (int) contact_id - * @return 0 */ #define DC_EVENT_MEMBER_REMOVED 2063 From 8d2f526ee721c0aaf9459ee79dc6ca05265aaa72 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Fri, 17 Apr 2020 22:29:58 +0300 Subject: [PATCH 086/156] Fix a typo --- src/sql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql.rs b/src/sql.rs index e43360a6b..2e5ef48f2 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -1173,7 +1173,7 @@ pub fn housekeeping(context: &Context) { if let Err(err) = prune_tombstones(context) { warn!( context, - "Houskeeping: Cannot prune message tombstones: {}", err + "Housekeeping: Cannot prune message tombstones: {}", err ); } From dbd37054418002a70874ae4e7d65eabaa6df515a Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 18 Apr 2020 01:46:08 +0200 Subject: [PATCH 087/156] fix doc for return value of dc_lookup_contact_id_by_addr() --- deltachat-ffi/deltachat.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 155c7a049..d42b4bc48 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1786,7 +1786,7 @@ int dc_may_be_valid_addr (const char* addr); /** * Check if an e-mail address belongs to a known and unblocked contact. - * Known and unblocked contacts will be returned by dc_get_contacts(). + * To get a list of all known and unblocked contacts, use dc_get_contacts(). * * To validate an e-mail address independently of the contact database * use dc_may_be_valid_addr(). @@ -1794,7 +1794,8 @@ int dc_may_be_valid_addr (const char* addr); * @memberof dc_context_t * @param context The context object as created by dc_context_new(). * @param addr The e-mail-address to check. - * @return 1=address is a contact in use, 0=address is not a contact in use. + * @return Contact ID of the contact belonging to the e-mail-address + * or 0 if there is no such contact. */ uint32_t dc_lookup_contact_id_by_addr (dc_context_t* context, const char* addr); From da88d8f17f5c9c45b7a6e21fd8551131b88ae448 Mon Sep 17 00:00:00 2001 From: bjoern Date: Sat, 18 Apr 2020 13:17:33 +0200 Subject: [PATCH 088/156] Update deltachat-ffi/deltachat.h Co-Authored-By: holger krekel --- deltachat-ffi/deltachat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index d42b4bc48..85580d388 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1795,7 +1795,7 @@ int dc_may_be_valid_addr (const char* addr); * @param context The context object as created by dc_context_new(). * @param addr The e-mail-address to check. * @return Contact ID of the contact belonging to the e-mail-address - * or 0 if there is no such contact. + * or 0 if there is no contact that is or was introduced by an accepted contact. */ uint32_t dc_lookup_contact_id_by_addr (dc_context_t* context, const char* addr); From 511727fdfa4ac7d94443926d7de62c5f2891024a Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 15 Apr 2020 20:18:21 +0200 Subject: [PATCH 089/156] Do not ellipsize everything in a message --- src/simplify.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index 2fb803907..ddf580b1c 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -41,19 +41,28 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) { let lines = split_lines(&input); let (lines, is_forwarded) = skip_forward_header(&lines); + let original_lines = &lines; + let lines = remove_message_footer(lines); - let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines); - let (lines, has_bottom_quote) = if !is_chat_message { + let (lines, mut has_nonstandard_footer) = remove_nonstandard_footer(lines); + let (lines, mut has_bottom_quote) = if !is_chat_message { remove_bottom_quote(lines) } else { (lines, false) }; - let (lines, has_top_quote) = if !is_chat_message { + let (mut lines, mut has_top_quote) = if !is_chat_message { remove_top_quote(lines) } else { (lines, false) }; + if lines.iter().all(|it| it.trim().is_empty()) { + lines = original_lines; + has_top_quote = false; + has_bottom_quote = false; + has_nonstandard_footer = false; + } + // re-create buffer from the remaining lines let text = render_message( lines, @@ -203,6 +212,17 @@ mod tests { } } + #[test] + fn test_dont_remove_whole_message() { + let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string(); + let (plain, is_forwarded) = simplify(input, false); + assert_eq!( + plain, + "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text" + ); + assert!(!is_forwarded); + } + #[test] fn test_simplify_trim() { let input = "line1\n\r\r\rline2".to_string(); From 6e2f4d85a348cf1d59377b2620e30fa0e914ab6d Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 19 Apr 2020 13:15:38 +0200 Subject: [PATCH 090/156] Do not ellipsize non-standard footers in chat messages --- src/simplify.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/simplify.rs b/src/simplify.rs index ddf580b1c..de0a19b05 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -44,7 +44,11 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) { let original_lines = &lines; let lines = remove_message_footer(lines); - let (lines, mut has_nonstandard_footer) = remove_nonstandard_footer(lines); + let (lines, mut has_nonstandard_footer) = if !is_chat_message { + remove_nonstandard_footer(lines) + } else { + (lines, false) + }; let (lines, mut has_bottom_quote) = if !is_chat_message { remove_bottom_quote(lines) } else { @@ -223,6 +227,14 @@ mod tests { assert!(!is_forwarded); } + #[test] + fn test_chat_message() { + let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string(); + let (plain, is_forwarded) = simplify(input, true); + assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good."); + assert!(!is_forwarded); + } + #[test] fn test_simplify_trim() { let input = "line1\n\r\r\rline2".to_string(); From a87a2d0b71363acb49fba4f9e4430d32370655fb Mon Sep 17 00:00:00 2001 From: Hocuri Date: Sun, 19 Apr 2020 16:30:24 +0200 Subject: [PATCH 091/156] Change to functional style --- src/simplify.rs | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index de0a19b05..cf61dc8ae 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -44,35 +44,24 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) { let original_lines = &lines; let lines = remove_message_footer(lines); - let (lines, mut has_nonstandard_footer) = if !is_chat_message { - remove_nonstandard_footer(lines) - } else { - (lines, false) - }; - let (lines, mut has_bottom_quote) = if !is_chat_message { - remove_bottom_quote(lines) - } else { - (lines, false) - }; - let (mut lines, mut has_top_quote) = if !is_chat_message { - remove_top_quote(lines) - } else { - (lines, false) - }; - if lines.iter().all(|it| it.trim().is_empty()) { - lines = original_lines; - has_top_quote = false; - has_bottom_quote = false; - has_nonstandard_footer = false; - } + let text = if is_chat_message { + render_message(lines, false, false) + } else { + let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines); + let (lines, has_bottom_quote) = remove_bottom_quote(lines); + let (lines, has_top_quote) = remove_top_quote(lines); - // re-create buffer from the remaining lines - let text = render_message( - lines, - has_top_quote, - has_nonstandard_footer || has_bottom_quote, - ); + if lines.iter().all(|it| it.trim().is_empty()) { + render_message(original_lines, false, false) + } else { + render_message( + lines, + has_top_quote, + has_nonstandard_footer || has_bottom_quote, + ) + } + }; (text, is_forwarded) } From 9eda71053810aac26544006eb537d85e0869c967 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Fri, 17 Apr 2020 21:32:27 +0200 Subject: [PATCH 092/156] refine member-added and member-removed plugin hooks to signal the sender (who added/removed a contact ) add ac_chat_modified hook event add account.get_contact_by_addr (thanks @r10s) --- python/examples/group_tracking.py | 10 ++- python/examples/test_examples.py | 4 +- python/src/deltachat/account.py | 47 +++++++++--- python/src/deltachat/hookspec.py | 10 ++- python/src/deltachat/message.py | 11 +++ python/tests/test_account.py | 123 +++++++++++++++++------------- src/chat.rs | 4 - 7 files changed, 133 insertions(+), 76 deletions(-) diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index baed3e951..dc66c1f2e 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -22,14 +22,16 @@ class GroupTrackingPlugin: print("*** ac_configure_completed:", success) @account_hookimpl - def ac_member_added(self, chat, contact): - print("*** ac_member_added", contact.addr, "from", chat) + def ac_member_added(self, chat, contact, sender): + print("*** ac_member_added {} to chat {} from {}".format( + contact.addr, chat.id, sender.addr)) for member in chat.get_contacts(): print("chat member: {}".format(member.addr)) @account_hookimpl - def ac_member_removed(self, chat, contact): - print("*** ac_member_removed", contact.addr, "from", chat) + def ac_member_removed(self, chat, contact, sender): + print("*** ac_member_removed {} from chat {} by {}".format( + contact.addr, chat.id, sender.addr)) def main(argv=None): diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index f3feb0e58..d3a16fe2d 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -43,14 +43,14 @@ def test_group_tracking_plugin(acfactory, lp): ac1.add_account_plugin(FFIEventLogger(ac1, "ac1")) ac2.add_account_plugin(FFIEventLogger(ac2, "ac2")) - lp.sec("creating bot test group with all three") + lp.sec("creating bot test group with bot") bot_contact = ac1.create_contact(botproc.addr) ch = ac1.create_group_chat("bot test group") ch.add_contact(bot_contact) ch.send_text("hello") botproc.fnmatch_lines(""" - *ac_member_added {}* + *ac_chat_modified* """.format(ac1.get_config("addr"))) lp.sec("adding third member {}".format(ac2.get_config("addr"))) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 7f881ebb7..da3588c9d 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -245,6 +245,14 @@ class Account(object): assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL return bool(lib.dc_delete_contact(self._dc_context, contact_id)) + def get_contact_by_addr(self, email): + """ get a contact for the email address or None if it's blocked or doesn't exist. """ + _, addr = parseaddr(email) + addr = as_dc_charpointer(addr) + contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr) + if contact_id: + return self.get_contact_by_id(contact_id) + def get_contacts(self, query=None, with_self=False, only_verified=False): """ get a (filtered) list of contacts. @@ -617,25 +625,46 @@ class Account(object): return "ac_configure_completed", dict(success=success) elif name == "DC_EVENT_INCOMING_MSG": msg = self.get_message_by_id(ffi_event.data2) - return "ac_incoming_message", dict(message=msg) + return self._map_incoming(msg) elif name == "DC_EVENT_MSGS_CHANGED": if ffi_event.data2 != 0: msg = self.get_message_by_id(ffi_event.data2) + if msg.is_outgoing(): + evname, kwargs = self._map_incoming(msg) + if evname.startswith("ac_member"): + return evname, kwargs if msg.is_in_fresh(): - return "ac_incoming_message", dict(message=msg) + return self._map_incoming(msg) elif name == "DC_EVENT_MSG_DELIVERED": msg = self.get_message_by_id(ffi_event.data2) return "ac_message_delivered", dict(message=msg) - elif name == "DC_EVENT_MEMBER_ADDED": + elif name == "DC_EVENT_CHAT_MODIFIED": chat = self.get_chat_by_id(ffi_event.data1) - contact = self.get_contact_by_id(ffi_event.data2) - return "ac_member_added", dict(chat=chat, contact=contact) - elif name == "DC_EVENT_MEMBER_REMOVED": - chat = self.get_chat_by_id(ffi_event.data1) - contact = self.get_contact_by_id(ffi_event.data2) - return "ac_member_removed", dict(chat=chat, contact=contact) + return "ac_chat_modified", dict(chat=chat) return None, {} + def _map_incoming(self, msg): + if msg.is_system_message(): + res = parse_system_add_remove(msg.text) + if res: + contact = msg.account.get_contact_by_addr(res[1]) + if contact: + d = dict(chat=msg.chat, contact=contact, sender=msg.get_sender_contact()) + return "ac_member_" + res[0], d + return "ac_incoming_message", dict(message=msg) + + +def parse_system_add_remove(text): + # Member Me (x@y) removed by a@b. + # Member x@y removed by a@b + text = text.lower() + parts = text.split() + if parts[0] == "member": + if parts[2] in ("removed", "added"): + return parts[2], parts[1] + if parts[3] in ("removed", "added"): + return parts[3], parts[2].strip("()") + def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): # destructor for dc_context diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 1512d031e..cd1859dad 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -53,11 +53,15 @@ class PerAccount: """ Called when an outgoing message has been delivered to SMTP. """ @account_hookspec - def ac_member_added(self, chat, contact): - """ Called for each contact added to a chat. """ + def ac_chat_modified(self, chat): + """ Chat was created or modified regarding membership, avatar, title. """ @account_hookspec - def ac_member_removed(self, chat, contact): + def ac_member_added(self, chat, contact, sender): + """ Called for each contact added to an accepted chat. """ + + @account_hookspec + def ac_member_removed(self, chat, contact, sender): """ Called for each contact removed from a chat. """ diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index fc63636de..d79ca2523 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -91,6 +91,10 @@ class Message(object): """mime type of the file (if it exists)""" return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg)) + def is_system_message(self): + """ return True if this message is a system/info message. """ + return lib.dc_msg_is_info(self._dc_msg) + def is_setup_message(self): """ return True if this message is a setup message. """ return lib.dc_msg_is_setupmessage(self._dc_msg) @@ -224,6 +228,13 @@ class Message(object): """ return self._msgstate == const.DC_STATE_IN_SEEN + def is_outgoing(self): + """Return True if Message is outgoing. """ + return self._msgstate in ( + const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING, + const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD, + const.DC_STATE_OUT_DELIVERED) + def is_out_preparing(self): """Return True if Message is outgoing, but its file is being prepared. """ diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 4cbbab238..bcf1ad20a 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -11,6 +11,17 @@ from conftest import (wait_configuration_progress, wait_securejoin_inviter_progress) +@pytest.mark.parametrize("msgtext,res", [ + ("Member Me (tmp1@x.org) removed by tmp2@x.org.", ("removed", "tmp1@x.org")), + ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")), +]) +def test_parse_system_add_remove(msgtext, res): + from deltachat.account import parse_system_add_remove + + out = parse_system_add_remove(msgtext) + assert out == res + + class TestOfflineAccountBasic: def test_wrong_db(self, tmpdir): p = tmpdir.join("hello.db") @@ -170,35 +181,6 @@ class TestOfflineChat: else: pytest.fail("could not find chat") - def test_add_member_event(self, ac1): - chat = ac1.create_group_chat(name="title1") - assert chat.is_group() - contact1 = ac1.create_contact("some1@hello.com", name="some1") - - chat.add_contact(contact1) - for ev in ac1.iter_events(timeout=1): - if ev.name == "ac_member_added": - assert ev.kwargs["chat"] == chat - if ev.kwargs["contact"] == ac1.get_self_contact(): - continue - assert ev.kwargs["contact"] == contact1 - break - - def test_remove_member_event(self, ac1): - chat = ac1.create_group_chat(name="title1") - assert chat.is_group() - contact1 = ac1.create_contact("some1@hello.com", name="some1") - chat.add_contact(contact1) - ac1._handle_current_events() - chat.remove_contact(contact1) - for ev in ac1.iter_events(timeout=1): - if ev.name == "ac_member_removed": - assert ev.kwargs["chat"] == chat - if ev.kwargs["contact"] == ac1.get_self_contact(): - continue - assert ev.kwargs["contact"] == contact1 - break - def test_group_chat_creation(self, ac1): contact1 = ac1.create_contact("some1@hello.com", name="some1") contact2 = ac1.create_contact("some2@hello.com", name="some2") @@ -496,7 +478,7 @@ class TestOfflineChat: # perform plugin hooks ac1._handle_current_events() - assert len(in_list) == 11 + assert len(in_list) == 10 chat_contacts = chat.get_contacts() for in_cmd, in_chat, in_contact in in_list: assert in_cmd == "added" @@ -504,19 +486,23 @@ class TestOfflineChat: assert in_contact in chat_contacts chat_contacts.remove(in_contact) + assert chat_contacts[0].id == 1 # self contact + + in_list[:] = [] + lp.sec("ac1: removing two contacts and checking things are right") chat.remove_contact(contacts[9]) chat.remove_contact(contacts[3]) assert len(chat.get_contacts()) == 9 ac1._handle_current_events() - assert len(in_list) == 13 - assert in_list[-2][0] == "removed" - assert in_list[-2][1] == chat - assert in_list[-2][2] == contacts[9] - assert in_list[-1][0] == "removed" - assert in_list[-1][1] == chat - assert in_list[-1][2] == contacts[3] + assert len(in_list) == 2 + assert in_list[0][0] == "removed" + assert in_list[0][1] == chat + assert in_list[0][2] == contacts[9] + assert in_list[1][0] == "removed" + assert in_list[1][1] == chat + assert in_list[1][2] == contacts[3] class TestOnlineAccount: @@ -1297,48 +1283,77 @@ class TestOnlineAccount: def test_add_remove_member_remote_events(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() + ac1_addr = ac1.get_config("addr") + ac2_addr = ac2.get_config("addr") # activate local plugin for ac2 in_list = queue.Queue() + class EventHolder: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + class InPlugin: @account_hookimpl - def ac_member_added(self, chat, contact): - in_list.put(("added", chat, contact)) + def ac_incoming_message(self, message): + # we immediately accept the sender because + # otherwise we won't see member_added contacts + message.accept_sender_contact() @account_hookimpl - def ac_member_removed(self, chat, contact): - in_list.put(("removed", chat, contact)) + def ac_chat_modified(self, chat): + in_list.put(EventHolder(action="chat-modified", chat=chat)) + + @account_hookimpl + def ac_member_added(self, chat, contact, sender): + in_list.put(EventHolder(action="added", chat=chat, contact=contact, sender=sender)) + + @account_hookimpl + def ac_member_removed(self, chat, contact, sender): + in_list.put(EventHolder(action="removed", chat=chat, contact=contact, sender=sender)) ac2.add_account_plugin(InPlugin()) lp.sec("ac1: create group chat with ac2") chat = ac1.create_group_chat("hello") - contact = ac1.create_contact(email=ac2.get_config("addr")) + contact = ac1.create_contact(email=ac2_addr) chat.add_contact(contact) lp.sec("ac1: send a message to group chat to promote the group") chat.send_text("afterwards promoted") - ev1 = in_list.get() - ev2 = in_list.get() - assert ev1[2] == ac2.get_self_contact() - assert ev2[2].addr == ac1.get_config("addr") + ev = in_list.get(timeout=10) + assert ev.action == "chat-modified" + assert chat.is_promoted() + assert sorted(x.addr for x in chat.get_contacts()) == \ + sorted(x.addr for x in ev.chat.get_contacts()) lp.sec("ac1: add address2") - contact2 = ac1.create_contact(email="not@example.org") + # note that if the above accept_sender_contact() would not + # happen we would not receive a proper member_added event + contact2 = ac1.create_contact(email="notexistingaccountihope@testrun.org") chat.add_contact(contact2) - ev1 = in_list.get() - assert ev1[2].addr == contact2.addr + ev = in_list.get(timeout=10) + assert ev.action == "chat-modified" + ev = in_list.get(timeout=10) + assert ev.action == "added" + assert ev.sender.addr == ac1_addr + assert ev.contact.addr == "notexistingaccountihope@testrun.org" lp.sec("ac1: remove address2") chat.remove_contact(contact2) - ev1 = in_list.get() - assert ev1[0] == "removed" - assert ev1[2].addr == contact2.addr + ev = in_list.get(timeout=10) + assert ev.action == "chat-modified" + ev = in_list.get(timeout=10) + assert ev.action == "removed" + assert ev.contact.addr == contact2.addr + assert ev.sender.addr == ac1_addr lp.sec("ac1: remove ac2 contact from chat") chat.remove_contact(contact) - ev1 = in_list.get() - assert ev1[2] == ac2.get_self_contact() + ev = in_list.get(timeout=10) + assert ev.action == "chat-modified" + ev = in_list.get(timeout=10) + assert ev.action == "removed" + assert ev.sender.addr == ac1_addr def test_set_get_group_image(self, acfactory, data, lp): ac1, ac2 = acfactory.get_two_online_accounts() diff --git a/src/chat.rs b/src/chat.rs index 8685e3743..df0b961a9 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2155,10 +2155,6 @@ pub fn remove_contact_from_chat( msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup); msg.param.set(Param::Arg, contact.get_addr()); msg.id = send_msg(context, chat_id, &mut msg)?; - context.call_cb(Event::MsgsChanged { - chat_id, - msg_id: msg.id, - }); } } // we remove the member from the chat after constructing the From a1c82eaea60a54965456f2c7848904d145b9b331 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 19 Apr 2020 11:08:08 +0200 Subject: [PATCH 093/156] refine bot testing and ac_member* handling --- python/examples/group_tracking.py | 4 +++ python/examples/test_examples.py | 6 ++-- python/src/deltachat/account.py | 47 ++++++++---------------------- python/src/deltachat/message.py | 23 +++++++++++++++ python/src/deltachat/testplugin.py | 8 +++-- python/tests/test_account.py | 2 +- 6 files changed, 48 insertions(+), 42 deletions(-) diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index dc66c1f2e..e3962c868 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -21,6 +21,10 @@ class GroupTrackingPlugin: def ac_configure_completed(self, success): print("*** ac_configure_completed:", success) + @account_hookimpl + def ac_chat_modified(self, chat): + print("*** ac_chat_modified:", chat.id, chat.get_name()) + @account_hookimpl def ac_member_added(self, chat, contact, sender): print("*** ac_member_added {} to chat {} from {}".format( diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index d3a16fe2d..5e6d363fe 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -33,12 +33,12 @@ def test_echo_quit_plugin(acfactory): def test_group_tracking_plugin(acfactory, lp): lp.sec("creating one group-tracking bot and two temp accounts") - botproc = acfactory.run_bot_process(group_tracking) + botproc = acfactory.run_bot_process(group_tracking, ffi=False) ac1, ac2 = acfactory.get_two_online_accounts(quiet=True) botproc.fnmatch_lines(""" - *ac_configure_completed: True* + *ac_configure_completed* """) ac1.add_account_plugin(FFIEventLogger(ac1, "ac1")) ac2.add_account_plugin(FFIEventLogger(ac2, "ac2")) @@ -50,7 +50,7 @@ def test_group_tracking_plugin(acfactory, lp): ch.send_text("hello") botproc.fnmatch_lines(""" - *ac_chat_modified* + *ac_chat_modified*bot test group* """.format(ac1.get_config("addr"))) lp.sec("adding third member {}".format(ac2.get_config("addr"))) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index da3588c9d..514884207 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -13,7 +13,7 @@ from . import const from .capi import ffi, lib from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot from .chat import Chat -from .message import Message +from .message import Message, map_system_message from .contact import Contact from .tracker import ImexTracker from . import hookspec, iothreads @@ -65,8 +65,7 @@ class Account(object): @hookspec.account_hookimpl def ac_process_ffi_event(self, ffi_event): - name, kwargs = self._map_ffi_event(ffi_event) - if name is not None: + for name, kwargs in self._map_ffi_event(ffi_event): ev = HookEvent(self, name=name, kwargs=kwargs) self._hook_event_queue.put(ev) @@ -622,48 +621,26 @@ class Account(object): data1 = ffi_event.data1 if data1 == 0 or data1 == 1000: success = data1 == 1000 - return "ac_configure_completed", dict(success=success) + yield "ac_configure_completed", dict(success=success) elif name == "DC_EVENT_INCOMING_MSG": msg = self.get_message_by_id(ffi_event.data2) - return self._map_incoming(msg) + yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg)) elif name == "DC_EVENT_MSGS_CHANGED": if ffi_event.data2 != 0: msg = self.get_message_by_id(ffi_event.data2) if msg.is_outgoing(): - evname, kwargs = self._map_incoming(msg) - if evname.startswith("ac_member"): - return evname, kwargs - if msg.is_in_fresh(): - return self._map_incoming(msg) + res = map_system_message(msg) + if res and res[0].startswith("ac_member"): + yield res + yield "ac_outgoing_message", dict(message=msg) + elif msg.is_in_fresh(): + yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg)) elif name == "DC_EVENT_MSG_DELIVERED": msg = self.get_message_by_id(ffi_event.data2) - return "ac_message_delivered", dict(message=msg) + yield "ac_message_delivered", dict(message=msg) elif name == "DC_EVENT_CHAT_MODIFIED": chat = self.get_chat_by_id(ffi_event.data1) - return "ac_chat_modified", dict(chat=chat) - return None, {} - - def _map_incoming(self, msg): - if msg.is_system_message(): - res = parse_system_add_remove(msg.text) - if res: - contact = msg.account.get_contact_by_addr(res[1]) - if contact: - d = dict(chat=msg.chat, contact=contact, sender=msg.get_sender_contact()) - return "ac_member_" + res[0], d - return "ac_incoming_message", dict(message=msg) - - -def parse_system_add_remove(text): - # Member Me (x@y) removed by a@b. - # Member x@y removed by a@b - text = text.lower() - parts = text.split() - if parts[0] == "member": - if parts[2] in ("removed", "added"): - return parts[2], parts[1] - if parts[3] in ("removed", "added"): - return parts[3], parts[2].strip("()") + yield "ac_chat_modified", dict(chat=chat) def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref): diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index d79ca2523..907ffe4c2 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -320,3 +320,26 @@ def get_viewtype_code_from_name(view_type_name): return code raise ValueError("message typecode not found for {!r}, " "available {!r}".format(view_type_name, list(_view_type_mapping.values()))) + + +# some helper code for turning system messages into hook events +def map_system_message(msg): + if msg.is_system_message(): + res = parse_system_add_remove(msg.text) + if res: + contact = msg.account.get_contact_by_addr(res[1]) + if contact: + d = dict(chat=msg.chat, contact=contact, sender=msg.get_sender_contact()) + return "ac_member_" + res[0], d + + +def parse_system_add_remove(text): + # Member Me (x@y) removed by a@b. + # Member x@y removed by a@b + text = text.lower() + parts = text.split() + if parts[0] == "member": + if parts[2] in ("removed", "added"): + return parts[2], parts[1] + if parts[3] in ("removed", "added"): + return parts[3], parts[2].strip("()") diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index c3e92f0eb..7719ff157 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -280,26 +280,28 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data): ac.start() return ac - def run_bot_process(self, module): + def run_bot_process(self, module, ffi=True): fn = module.__file__ bot_ac, bot_cfg = self.get_online_config() args = [ sys.executable, + "-u", fn, - "--show-ffi", "--email", bot_cfg["addr"], "--password", bot_cfg["mail_pw"], bot_ac.db_path, ] + if ffi: + args.insert(-1, "--show-ffi") print("$", " ".join(args)) popen = subprocess.Popen( args=args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, # combine stdout/stderr in one stream - bufsize=1, # line buffering + bufsize=0, # line buffering close_fds=True, # close all FDs other than 0/1/2 universal_newlines=True # give back text ) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index bcf1ad20a..7563f5bf5 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -16,7 +16,7 @@ from conftest import (wait_configuration_progress, ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org")), ]) def test_parse_system_add_remove(msgtext, res): - from deltachat.account import parse_system_add_remove + from deltachat.message import parse_system_add_remove out = parse_system_add_remove(msgtext) assert out == res From 02cda1e611f98a23350adb221d8fa842ef7073bb Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 19 Apr 2020 12:31:15 +0200 Subject: [PATCH 094/156] refine member/add remove further, and introduce ac_outgoing_message --- python/examples/group_tracking.py | 26 ++++++++++++++++---------- python/src/deltachat/chat.py | 2 +- python/src/deltachat/hookspec.py | 10 +++++++--- python/src/deltachat/message.py | 9 +++++++-- python/src/deltachat/testplugin.py | 6 ++++++ python/tests/test_account.py | 21 ++++++++++++++------- 6 files changed, 51 insertions(+), 23 deletions(-) diff --git a/python/examples/group_tracking.py b/python/examples/group_tracking.py index e3962c868..d7f180985 100644 --- a/python/examples/group_tracking.py +++ b/python/examples/group_tracking.py @@ -17,25 +17,31 @@ class GroupTrackingPlugin: text = message.text message.chat.send_text("echoing from {}:\n{}".format(addr, text)) + @account_hookimpl + def ac_outgoing_message(self, message): + print("ac_outgoing_message:", message) + @account_hookimpl def ac_configure_completed(self, success): - print("*** ac_configure_completed:", success) + print("ac_configure_completed:", success) @account_hookimpl def ac_chat_modified(self, chat): - print("*** ac_chat_modified:", chat.id, chat.get_name()) - - @account_hookimpl - def ac_member_added(self, chat, contact, sender): - print("*** ac_member_added {} to chat {} from {}".format( - contact.addr, chat.id, sender.addr)) + print("ac_chat_modified:", chat.id, chat.get_name()) for member in chat.get_contacts(): print("chat member: {}".format(member.addr)) @account_hookimpl - def ac_member_removed(self, chat, contact, sender): - print("*** ac_member_removed {} from chat {} by {}".format( - contact.addr, chat.id, sender.addr)) + def ac_member_added(self, chat, contact, message): + print("ac_member_added {} to chat {} from {}".format( + contact.addr, chat.id, message.get_sender_contact().addr)) + for member in chat.get_contacts(): + print("chat member: {}".format(member.addr)) + + @account_hookimpl + def ac_member_removed(self, chat, contact, message): + print("ac_member_removed {} from chat {} by {}".format( + contact.addr, chat.id, message.get_sender_contact().addr)) def main(argv=None): diff --git a/python/src/deltachat/chat.py b/python/src/deltachat/chat.py index 0c07dae11..f2d6cd09a 100644 --- a/python/src/deltachat/chat.py +++ b/python/src/deltachat/chat.py @@ -30,7 +30,7 @@ class Chat(object): return not (self == other) def __repr__(self): - return "".format(self.id, self.get_name(), self._dc_context) + return "".format(self.id, self.get_name()) @property def _dc_chat(self): diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index cd1859dad..00ec36d98 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -48,6 +48,10 @@ class PerAccount: def ac_incoming_message(self, message): """ Called on any incoming message (to deaddrop or chat). """ + @account_hookspec + def ac_outgoing_message(self, message): + """ Called on each outgoing message (both system and "normal").""" + @account_hookspec def ac_message_delivered(self, message): """ Called when an outgoing message has been delivered to SMTP. """ @@ -57,12 +61,12 @@ class PerAccount: """ Chat was created or modified regarding membership, avatar, title. """ @account_hookspec - def ac_member_added(self, chat, contact, sender): + def ac_member_added(self, chat, contact, message): """ Called for each contact added to an accepted chat. """ @account_hookspec - def ac_member_removed(self, chat, contact, sender): - """ Called for each contact removed from a chat. """ + def ac_member_removed(self, chat, contact, message): + """ Called for each contact removed from a chat. """ class Global: diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 907ffe4c2..91cc738c0 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -28,7 +28,9 @@ class Message(object): return self.account == other.account and self.id == other.id def __repr__(self): - return "".format(self.id, self._dc_context) + c = self.get_sender_contact() + return "".format( + self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name()) @classmethod def from_db(cls, account, id): @@ -322,14 +324,17 @@ def get_viewtype_code_from_name(view_type_name): "available {!r}".format(view_type_name, list(_view_type_mapping.values()))) +# # some helper code for turning system messages into hook events +# + def map_system_message(msg): if msg.is_system_message(): res = parse_system_add_remove(msg.text) if res: contact = msg.account.get_contact_by_addr(res[1]) if contact: - d = dict(chat=msg.chat, contact=contact, sender=msg.get_sender_contact()) + d = dict(chat=msg.chat, contact=contact, message=msg) return "ac_member_" + res[0], d diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 7719ff157..f336ac058 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -347,15 +347,21 @@ class BotProcess: patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()] for next_pattern in patterns: print("+++FNMATCH:", next_pattern) + ignored = [] while 1: line = self.stdout_queue.get(timeout=15) if line is None: + if ignored: + print("BOT stdout terminated after these lines") + for line in ignored: + print(line) raise IOError("BOT stdout-thread terminated") if fnmatch.fnmatch(line, next_pattern): print("+++MATCHED:", line) break else: print("+++IGN:", line) + ignored.append(line) @pytest.fixture diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 7563f5bf5..a411927b1 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -1052,12 +1052,17 @@ class TestOnlineAccount: message_queue.put(message) delivered = queue.Queue() + out = queue.Queue() class OutPlugin: @account_hookimpl def ac_message_delivered(self, message): delivered.put(message) + @account_hookimpl + def ac_outgoing_message(self, message): + out.put(message) + ac1.add_account_plugin(OutPlugin()) ac2.add_account_plugin(InPlugin()) @@ -1068,6 +1073,8 @@ class TestOnlineAccount: assert ev.data1 == chat.id assert ev.data2 == msg_out.id assert msg_out.is_out_delivered() + m = out.get() + assert m == msg_out m = delivered.get() assert m == msg_out @@ -1304,12 +1311,12 @@ class TestOnlineAccount: in_list.put(EventHolder(action="chat-modified", chat=chat)) @account_hookimpl - def ac_member_added(self, chat, contact, sender): - in_list.put(EventHolder(action="added", chat=chat, contact=contact, sender=sender)) + def ac_member_added(self, chat, contact, message): + in_list.put(EventHolder(action="added", chat=chat, contact=contact, message=message)) @account_hookimpl - def ac_member_removed(self, chat, contact, sender): - in_list.put(EventHolder(action="removed", chat=chat, contact=contact, sender=sender)) + def ac_member_removed(self, chat, contact, message): + in_list.put(EventHolder(action="removed", chat=chat, contact=contact, message=message)) ac2.add_account_plugin(InPlugin()) @@ -1335,7 +1342,7 @@ class TestOnlineAccount: assert ev.action == "chat-modified" ev = in_list.get(timeout=10) assert ev.action == "added" - assert ev.sender.addr == ac1_addr + assert ev.message.get_sender_contact().addr == ac1_addr assert ev.contact.addr == "notexistingaccountihope@testrun.org" lp.sec("ac1: remove address2") @@ -1345,7 +1352,7 @@ class TestOnlineAccount: ev = in_list.get(timeout=10) assert ev.action == "removed" assert ev.contact.addr == contact2.addr - assert ev.sender.addr == ac1_addr + assert ev.message.get_sender_contact().addr == ac1_addr lp.sec("ac1: remove ac2 contact from chat") chat.remove_contact(contact) @@ -1353,7 +1360,7 @@ class TestOnlineAccount: assert ev.action == "chat-modified" ev = in_list.get(timeout=10) assert ev.action == "removed" - assert ev.sender.addr == ac1_addr + assert ev.message.get_sender_contact().addr == ac1_addr def test_set_get_group_image(self, acfactory, data, lp): ac1, ac2 = acfactory.get_two_online_accounts() From 70c082a1b193b700b2e0f7ce387438bfa89603c4 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Sun, 19 Apr 2020 21:13:08 +0200 Subject: [PATCH 095/156] remove all member_added/remove_events --- deltachat-ffi/deltachat.h | 17 ----------------- deltachat-ffi/src/lib.rs | 19 ------------------- python/src/deltachat/const.py | 4 +--- python/tests/test_account.py | 7 ------- src/chat.rs | 20 ++------------------ src/events.rs | 18 ------------------ src/securejoin.rs | 13 ++++--------- 7 files changed, 7 insertions(+), 91 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 85580d388..9bfdff4ca 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -4463,23 +4463,6 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); */ #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS 2061 - -/** - * This event is sent for each member that gets added to a (verified or unverified) chat. - * - * @param data1 (int) chat_id - * @param data2 (int) contact_id - */ -#define DC_EVENT_MEMBER_ADDED 2062 - -/** - * This event is sent for each member that gets removed from a (verified or unverified) chat. - * - * @param data1 (int) chat_id - * @param data2 (int) contact_id - */ -#define DC_EVENT_MEMBER_REMOVED 2063 - /** * @} */ diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index f32f95e26..47ebba5c6 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -196,25 +196,6 @@ impl ContextWrapper { progress as uintptr_t, ); } - Event::SecurejoinMemberAdded { - chat_id, - contact_id, - } - | Event::MemberAdded { - chat_id, - contact_id, - } - | Event::MemberRemoved { - chat_id, - contact_id, - } => { - ffi_cb( - self, - event_id, - chat_id.to_u32() as uintptr_t, - contact_id as uintptr_t, - ); - } } } } diff --git a/python/src/deltachat/const.py b/python/src/deltachat/const.py index 13b638bb8..20139ef44 100644 --- a/python/src/deltachat/const.py +++ b/python/src/deltachat/const.py @@ -11,6 +11,7 @@ from os.path import join as joinpath DC_GCL_ARCHIVED_ONLY = 0x01 DC_GCL_NO_SPECIALS = 0x02 DC_GCL_ADD_ALLDONE_HINT = 0x04 +DC_GCL_FOR_FORWARDING = 0x08 DC_GCL_VERIFIED_ONLY = 0x01 DC_GCL_ADD_SELF = 0x02 DC_QR_ASK_VERIFYCONTACT = 200 @@ -98,9 +99,6 @@ DC_EVENT_IMEX_PROGRESS = 2051 DC_EVENT_IMEX_FILE_WRITTEN = 2052 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061 -DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062 -DC_EVENT_MEMBER_ADDED = 2063 -DC_EVENT_MEMBER_REMOVED = 2064 DC_EVENT_FILE_COPIED = 2055 DC_EVENT_IS_OFFLINE = 2081 DC_EVENT_GET_STRING = 2091 diff --git a/python/tests/test_account.py b/python/tests/test_account.py index a411927b1..f2fdb39e3 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -1202,11 +1202,6 @@ class TestOnlineAccount: ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") wait_securejoin_inviter_progress(ac1, 1000) - ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED") - - ch.remove_contact(ac1.get_self_contact()) - ac2._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED") - ac1._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED") def test_qr_verified_group_and_chatting(self, acfactory, lp): ac1, ac2 = acfactory.get_two_online_accounts() @@ -1218,7 +1213,6 @@ class TestOnlineAccount: chat2 = ac2.qr_join_chat(qr) assert chat2.id >= 10 wait_securejoin_inviter_progress(ac1, 1000) - ac1._evtracker.get_matching("DC_EVENT_MEMBER_ADDED") lp.sec("ac2: read member added message") msg = ac2._evtracker.wait_next_incoming_message() @@ -1540,7 +1534,6 @@ class TestGroupStressTests: to_remove = contacts[-1] msg.chat.remove_contact(to_remove) - ac2._evtracker.get_matching("DC_EVENT_MEMBER_REMOVED") lp.sec("ac1: receiving system message about contact removal") sysmsg = ac1._evtracker.wait_next_incoming_message() diff --git a/src/chat.rs b/src/chat.rs index df0b961a9..ab487be1e 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1795,7 +1795,6 @@ pub fn create_group_chat( } /// add a contact to the chats_contact table -/// on success emit MemberAdded event and return true pub(crate) fn add_to_chat_contacts_table( context: &Context, chat_id: ChatId, @@ -1807,14 +1806,7 @@ pub(crate) fn add_to_chat_contacts_table( "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)", params![chat_id, contact_id as i32], ) { - Ok(()) => { - context.call_cb(Event::MemberAdded { - chat_id, - contact_id, - }); - - true - } + Ok(()) => true, Err(err) => { error!( context, @@ -1827,7 +1819,6 @@ pub(crate) fn add_to_chat_contacts_table( } /// remove a contact from the chats_contact table -/// on success emit MemberRemoved event and return true pub(crate) fn remove_from_chat_contacts_table( context: &Context, chat_id: ChatId, @@ -1839,14 +1830,7 @@ pub(crate) fn remove_from_chat_contacts_table( "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?", params![chat_id, contact_id as i32], ) { - Ok(()) => { - context.call_cb(Event::MemberRemoved { - chat_id, - contact_id, - }); - - true - } + Ok(()) => true, Err(_) => { warn!( context, diff --git a/src/events.rs b/src/events.rs index 540fe3364..dc8a84f67 100644 --- a/src/events.rs +++ b/src/events.rs @@ -201,22 +201,4 @@ pub enum Event { /// (Bob has verified alice and waits until Alice does the same for him) #[strum(props(id = "2061"))] SecurejoinJoinerProgress { contact_id: u32, progress: usize }, - - /// This event is sent out to the inviter when a joiner successfully joined a group. - /// @param data1 (int) chat_id - /// @param data2 (int) contact_id - #[strum(props(id = "2062"))] - SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 }, - - /// This event is sent for each contact added to a chat. - /// @param data1 (int) chat_id - /// @param data2 (int) contact_id - #[strum(props(id = "2063"))] - MemberAdded { chat_id: ChatId, contact_id: u32 }, - - /// This event is sent for each contact removed from a chat. - /// @param data1 (int) chat_id - /// @param data2 (int) contact_id - #[strum(props(id = "2064"))] - MemberRemoved { chat_id: ChatId, contact_id: u32 }, } diff --git a/src/securejoin.rs b/src/securejoin.rs index bb77e8335..bcae24ba9 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -753,17 +753,12 @@ pub(crate) fn handle_securejoin_handshake( .get(HeaderDef::SecureJoinGroup) .map(|s| s.as_str()) .unwrap_or_else(|| ""); - let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid) - .map_err(|err| { + if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid) { warn!(context, "Failed to lookup chat_id from grpid: {}", err); - HandshakeError::ChatNotFound { + return Err(HandshakeError::ChatNotFound { group: field_grpid.to_string(), - } - })?; - context.call_cb(Event::MemberAdded { - chat_id: group_chat_id, - contact_id, - }); + }); + } } Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device } else { From 6c0dd8543dba57e8d49c3747dd3a21990b91dd51 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Mon, 20 Apr 2020 02:26:19 +0300 Subject: [PATCH 096/156] Delete expired messages instead of hiding them For hidden messages, blobs are not deleted during housekeeping. To actually free the space used by media files, messages should be moved to trash instead of being hidden. --- src/chat.rs | 15 ++++++++------- src/chatlist.rs | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 8685e3743..7821b35ec 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1441,7 +1441,7 @@ pub fn get_chat_msgs( flags: u32, marker1before: Option, ) -> Vec { - match hide_device_expired_messages(context) { + match delete_device_expired_messages(context) { Err(err) => warn!(context, "Failed to delete expired messages: {}", err), Ok(messages_deleted) => { if messages_deleted { @@ -1587,11 +1587,11 @@ pub fn marknoticed_all_chats(context: &Context) -> Result<(), Error> { Ok(()) } -/// Hides messages which are expired according to "delete_device_after" setting. +/// Deletes messages which are expired according to "delete_device_after" setting. /// -/// Returns true if any message is hidden, so event can be emitted. If nothing -/// has been hidden, returns false. -pub fn hide_device_expired_messages(context: &Context) -> Result { +/// Returns true if any message is deleted, so event can be emitted. If nothing +/// has been deleted, returns false. +pub fn delete_device_expired_messages(context: &Context) -> Result { if let Some(delete_device_after) = context.get_config_delete_device_after() { let threshold_timestamp = time() - delete_device_after; @@ -1602,19 +1602,20 @@ pub fn hide_device_expired_messages(context: &Context) -> Result { .unwrap_or_default() .0; - // Hide expired messages + // Delete expired messages // // Only update the rows that have to be updated, to avoid emitting // unnecessary "chat modified" events. let rows_modified = context.sql.execute( "UPDATE msgs \ - SET txt = 'DELETED', hidden = 1 \ + SET txt = 'DELETED', chat_id = ? \ WHERE timestamp < ? \ AND chat_id > ? \ AND chat_id != ? \ AND chat_id != ? \ AND NOT hidden", params![ + DC_CHAT_ID_TRASH, threshold_timestamp, DC_CHAT_ID_LAST_SPECIAL, self_chat_id, diff --git a/src/chatlist.rs b/src/chatlist.rs index f6ed9a003..ce3569f7b 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -93,8 +93,8 @@ impl Chatlist { query_contact_id: Option, ) -> Result { // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some - // messages get hidden to avoid reloading the same chatlist. - if let Err(err) = hide_device_expired_messages(context) { + // messages get deleted to avoid reloading the same chatlist. + if let Err(err) = delete_device_expired_messages(context) { warn!(context, "Failed to hide expired messages: {}", err); } From 7c33c7f7da337cc157b62c0b1946b91b1517bd2d Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 21 Apr 2020 13:52:05 +0200 Subject: [PATCH 097/156] expose obtaining list of fresh messages to python --- python/src/deltachat/account.py | 8 ++++++++ python/tests/test_account.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 514884207..ef2b70c83 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -273,6 +273,14 @@ class Account(object): ) return list(iter_array(dc_array, lambda x: Contact(self, x))) + def get_fresh_messages(self): + """ yield all fresh messages from all chats. """ + dc_array = ffi.gc( + lib.dc_get_fresh_msgs(self._dc_context), + lib.dc_array_unref + ) + yield from iter_array(dc_array, lambda x: Message.from_db(self, x)) + def create_chat_by_contact(self, contact): """ create or get an existing 1:1 chat object for the specified contact or contact id. diff --git a/python/tests/test_account.py b/python/tests/test_account.py index a411927b1..4b1948464 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -928,6 +928,13 @@ class TestOnlineAccount: assert msg_back.text == "message-back" assert msg_back.is_encrypted() + # test get_fresh_messages + fresh_msgs = list(ac1.get_fresh_messages()) + assert len(fresh_msgs) == 1 + assert fresh_msgs[0] == msg_back + msg_back.mark_seen() + assert not list(ac1.get_fresh_messages()) + # Test that we do not gossip peer keys in 1-to-1 chat, # as it makes no sense to gossip to peers their own keys. # Gossip is only sent in encrypted messages, From 61c84c8e014b937618e08e3aed8a5faaa205144d Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 22 Apr 2020 14:03:59 +0200 Subject: [PATCH 098/156] Log more in tests --- src/test_utils.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index 4b0e30b7c..815b50b7a 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -42,14 +42,14 @@ pub(crate) fn test_context(callback: Option>) -> TestContex /// specified in [test_context] but there is no callback hooked up, /// i.e. [Context::call_cb] will always return `0`. pub(crate) fn dummy_context() -> TestContext { - test_context(None) + test_context(Some(Box::new(logging_cb))) } pub(crate) fn logging_cb(_ctx: &Context, evt: Event) { match evt { Event::Info(msg) => println!("I: {}", msg), - Event::Warning(msg) => println!("W: {}", msg), - Event::Error(msg) => println!("E: {}", msg), + Event::Warning(msg) => eprintln!("=== WARNING ===\n{}\n===============", msg), + Event::Error(msg) => eprintln!("\n===================== ERROR =====================\n{}\n=================================================\n", msg), _ => (), } } From 4e828199c864155e15c2c0363732c713bf42a05a Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Thu, 23 Apr 2020 01:22:17 +0200 Subject: [PATCH 099/156] test, that also unrecoded avatars are copied to the blob-directory --- src/config.rs | 23 +++++++++++++++++++++++ test-data/image/avatar64x64.png | Bin 0 -> 2115 bytes 2 files changed, 23 insertions(+) create mode 100644 test-data/image/avatar64x64.png diff --git a/src/config.rs b/src/config.rs index 61f58ee90..203bdab0a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -323,4 +323,27 @@ mod tests { assert_eq!(img.width(), AVATAR_SIZE); assert_eq!(img.height(), AVATAR_SIZE); } + + #[test] + fn test_selfavatar_copy_without_recode() { + let t = dummy_context(); + let avatar_src = t.dir.path().join("avatar.png"); + let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png"); + File::create(&avatar_src) + .unwrap() + .write_all(avatar_bytes) + .unwrap(); + let avatar_blob = t.ctx.get_blobdir().join("avatar.png"); + assert!(!avatar_blob.exists()); + t.ctx + .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap())) + .unwrap(); + assert!(avatar_blob.exists()); + assert_eq!( + std::fs::metadata(&avatar_blob).unwrap().len(), + avatar_bytes.len() as u64 + ); + let avatar_cfg = t.ctx.get_config(Config::Selfavatar); + assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string())); + } } diff --git a/test-data/image/avatar64x64.png b/test-data/image/avatar64x64.png new file mode 100644 index 0000000000000000000000000000000000000000..4aa8f180401c2ddfc2bc04bea87c0dc338c76131 GIT binary patch literal 2115 zcmV-J2)y@+P)0004mX+uL$Nkc;* zaB^>EX>4Tx04R}tkv&MmKpe$i(~3nZ9V{Z^kfAzR5Eao)t5AdrrB-Ow!Q|2}Xws0R zxHt-~1qVMCs}3&Cx;nTDg5U>;lcSTOiV+KKqnfhERm4xT`x`&UicQKyjeeTcEtK>}v_(bAarW+RV2JzIU zrE}gVjTkx9=JM~KB@2g@DIN`^{2NgP#Fjq-)8 z%L?Z$&T6H`TKD8H4CS?zWvPFL`jhl} zTZpV2qvfWBLxXU*-cwU5&WAVXaxZ-9eC zV5CUd>mKh8cJ}t~nO1*4b2Dr77scts@a%^0000PbVXQnLvL+uWo~o;Lvm$dbY)~9cWHEJAV*0}P*;Ht7XSbT zElET{RA}DqnOjd9TNKCF-g9SUKrRQ0oZ3d4YPEgP*yn!R^F5lrHEGkNT8RX~DpV@B zxxvidv-f!zu+~No!==R&_A?>tx7PZv+n|58wjk<52rx#90s!3efHB}4{?HRAupS$R zR46F98~}Kim!w{2runBBq61(ot*xaF4pLiN0MI@@Zohx;*){~h!vH|48e3n_?eFJz zb^t)q^+B`g*J|8x9teOas=1s|D8!OU05A##HJ3x8ID-V5NLtYWWLZfjwM0Tx6adf? z2_>0CvOK-@lSyk55eteU$TC6*00dbU6a^5P7GP3oO#*;0Mk9+6LI~sO0VbA~2#fJ1 zme%|M#*^m6(h}!w{kYPcR9bTe7+;!`N-H`5=Zp|GvVa8vBjm1P_ZzxibO1`dUeB{E zmP!GDXIWma2b4}L%_x4E3V=C|RW5(~{F!qO0GEe{R=LdX@Mr{SM)Av30NnT8Qt2F_ z&dCV?IJKHvDskVRvg<|i%d{GxjfT}~*&jaw0C!#P`*+5AY-vXG%d}O9bMATE^JZDE zBKc(jJMqMR!SKM}65u~4S5dqcE}DMo{@oD}MM>91NxH5`Flz=pSN}`dY%HA?73JFZ zeAYLZIRYSA)`~@KbrlK1u-p2+Z<^!u*M$JUx*jVOv_t}9%sCHy-)uDOYL(6Oeu3vI z0NF5v(d6Uz``xoM_w;m}{<FIki_O2=Qa=DZWnXZc)2fQgr^XtjEhPfU@ z0|;db{mG2%juh&T0Sqw z<5&~{fDz*LdUu8Wz7z_7dh3*cFxJ!Q;{JaAKhxz#zyk#)5O}EY8b*O0xSqT%lh{D^EpCRt97fZ+zG&4*F8VKDwSkiSL1Oc z2;q@O)-+L3^i(QrQ!ANFW@Dqbzi&62UbpLYyLP40zdt4jfgoUAmy1Pxb2C;bXxVK1 z(W7`iucyB_Uu_=cQ>=MlYIPGOC%&slTj?Tr-r%$1t5JJk6C+?d!^xz;-C}2U5R8^29;5@Vd0CdLmR4R;#Rj=EP zhEuCK<#N#L0RR$3Wo>PBZ!f#Mo7vo4$z&u=LkNX`{$u(8V1)Snen1H4JWQd>!$YT5 zT*H6KR4ndgluQ<~YoCyMz$aY#krV@i@X52}0uUzw<9% z+>8mrSW;CUy9uPtX13DO(mzUkMGxz&Q*YW*}Scs*^kql(|X;! zy0Slf7%D`P5(fv=^R8op5E2wcQdJzD7Q?6j0OzFNZ`W$yOC|f;H?v%JD;3&qM=QZI z*avPY1fd@1)Ut*#5d=|D^jt1kENaQ*H2@$K_h0U0(;45{^WVsRo&vC{ zs_X0V&COUYhb4&z0cVT{f$uoh$%*sj3%R%e#uj*`L8Cb9qk002ovPDHLkV1g2p@NNJA literal 0 HcmV?d00001 From 979d7c562515da2a30983993048cd5184889059c Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Thu, 23 Apr 2020 00:11:32 +0300 Subject: [PATCH 100/156] Do not ignore database read errors in precheck_imf If precheck_imf fails to check if message with the same rfc724_mid already exists, the same message may be downloaded twice. Instead, abort the whole operation and retry later. --- src/dc_receive_imf.rs | 4 ++-- src/imap/mod.rs | 17 +++++++++++------ src/message.rs | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 48b8df02b..e3d589360 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -298,8 +298,8 @@ fn add_parts( // check, if the mail is already in our database - if so, just update the folder/uid // (if the mail was moved around) and finish. (we may get a mail twice eg. if it is // moved between folders. make sure, this check is done eg. before securejoin-processing) */ - if let Ok((old_server_folder, old_server_uid, _)) = - message::rfc724_mid_exists(context, &rfc724_mid) + if let Some((old_server_folder, old_server_uid, _)) = + message::rfc724_mid_exists(context, &rfc724_mid)? { if old_server_folder != server_folder.as_ref() || old_server_uid != server_uid { message::update_server_uid(context, &rfc724_mid, server_folder.as_ref(), server_uid); diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 24937faf3..b86b8c39e 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -604,7 +604,7 @@ impl Imap { let headers = get_fetch_headers(fetch)?; let message_id = prefetch_get_message_id(&headers).unwrap_or_default(); - if precheck_imf(context, &message_id, folder.as_ref(), cur_uid) { + if precheck_imf(context, &message_id, folder.as_ref(), cur_uid)? { // we know the message-id already or don't want the message otherwise. info!( context, @@ -1266,9 +1266,14 @@ fn get_folder_meaning(folder_name: &Name) -> FolderMeaning { } } -fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server_uid: u32) -> bool { - if let Ok((old_server_folder, old_server_uid, msg_id)) = - message::rfc724_mid_exists(context, &rfc724_mid) +fn precheck_imf( + context: &Context, + rfc724_mid: &str, + server_folder: &str, + server_uid: u32, +) -> Result { + if let Some((old_server_folder, old_server_uid, msg_id)) = + message::rfc724_mid_exists(context, &rfc724_mid)? { if old_server_folder.is_empty() && old_server_uid == 0 { info!( @@ -1322,9 +1327,9 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server if old_server_folder != server_folder || old_server_uid != server_uid { update_server_uid(context, &rfc724_mid, server_folder, server_uid); } - true + Ok(true) } else { - false + Ok(false) } } diff --git a/src/message.rs b/src/message.rs index 3d54aff8a..732ad4c26 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1431,12 +1431,12 @@ pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> i32 { pub(crate) fn rfc724_mid_exists( context: &Context, rfc724_mid: &str, -) -> Result<(String, u32, MsgId), Error> { +) -> Result, Error> { ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid"); context .sql - .query_row( + .query_row_optional( "SELECT server_folder, server_uid, id FROM msgs WHERE rfc724_mid=?", &[rfc724_mid], |row| { From d29c5eabbb3389b3d85a617bf161fc566513ff05 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 23 Apr 2020 20:25:36 +0200 Subject: [PATCH 101/156] Improve descriptions for the profile image files --- deltachat-ffi/deltachat.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 9bfdff4ca..10154b71a 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -342,7 +342,8 @@ char* dc_get_blobdir (const dc_context_t* context); * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0) * - `displayname` = Own name to use when sending messages. MUAs are allowed to spread this way eg. using CC, defaults to empty * - `selfstatus` = Own status to display eg. in email footers, defaults to a standard text - * - `selfavatar` = File containing avatar. Will be copied to blob directory. + * - `selfavatar` = File containing avatar. Will immediately be copied to the + * `blobdir`; the original image will not be needed anymore. * NULL to remove the avatar. * It is planned for future versions * to send this image together with the next messages. @@ -1612,8 +1613,10 @@ int dc_set_chat_name (dc_context_t* context, uint32_t ch * @memberof dc_context_t * @param context The context as created by dc_context_new(). * @param chat_id The chat ID to set the image for. - * @param image Full path of the image to use as the group image. If you pass NULL here, - * the group image is deleted (for promoted groups, all members are informed about this change anyway). + * @param image Full path of the image to use as the group image. The image will immediately be copied to the + * `blobdir`; the original image will not be needed anymore. + * If you pass NULL here, the group image is deleted (for promoted groups, all members are informed about + * this change anyway). * @return 1=success, 0=error */ int dc_set_chat_profile_image (dc_context_t* context, uint32_t chat_id, const char* image); From 220500efbbccae2bbf98fa72473949fa284cb7fa Mon Sep 17 00:00:00 2001 From: Floris Bruynooghe Date: Sat, 15 Feb 2020 16:56:39 +0100 Subject: [PATCH 102/156] Move key loading from deprecated Key struct to DcKey trait This moves the loading of the keys from the database to the trait and thus with types differing between public and secret keys. This fetches the Config::ConfiguredAddr (configured_addr) directly from the database in the SQL to simplify the API and consistency instead of making this the responsiblity of all callers to get this right. Since anyone invoking these methods also wants to be sure the keys exist, move key generation here as well. This already simplifies some code in contact.rs and will eventually replace all manual checks for existing keys. To make errors more manageable this gives EmailAddress it's own error type and adds some conversions for it. Otherwise the general error type leaks to far. The EmailAddress type also gets its ToSql trait impl to be able to save it to the database directly. --- src/contact.rs | 14 +--- src/context.rs | 9 +-- src/dc_tools.rs | 70 +++++++++++++---- src/e2ee.rs | 89 ++-------------------- src/imex.rs | 6 +- src/key.rs | 186 ++++++++++++++++++++++++++++++++++++++-------- src/pgp.rs | 2 +- src/securejoin.rs | 11 +-- 8 files changed, 230 insertions(+), 157 deletions(-) diff --git a/src/contact.rs b/src/contact.rs index b7ad24015..34e3d051e 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -11,10 +11,9 @@ use crate::config::Config; use crate::constants::*; use crate::context::Context; use crate::dc_tools::*; -use crate::e2ee; use crate::error::{bail, ensure, format_err, Result}; use crate::events::Event; -use crate::key::*; +use crate::key::{DcKey, Key, SignedPublicKey}; use crate::login_param::LoginParam; use crate::message::{MessageState, MsgId}; use crate::mimeparser::AvatarAction; @@ -647,8 +646,6 @@ impl Contact { let peerstate = Peerstate::from_addr(context, &context.sql, &contact.addr); let loginparam = LoginParam::from_database(context, "configured_"); - let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql); - if peerstate.is_some() && peerstate .as_ref() @@ -663,16 +660,11 @@ impl Contact { StockMessage::E2eAvailable }); ret += &p; - if self_key.is_none() { - e2ee::ensure_secret_key_exists(context)?; - self_key = Key::from_self_public(context, &loginparam.addr, &context.sql); - } + let self_key = Key::from(SignedPublicKey::load_self(context)?); let p = context.stock_str(StockMessage::FingerPrints); ret += &format!(" {}:", p); - let fingerprint_self = self_key - .map(|k| k.formatted_fingerprint()) - .unwrap_or_default(); + let fingerprint_self = self_key.formatted_fingerprint(); let fingerprint_other_verified = peerstate .peek_key(PeerstateVerifiedStatus::BidirectVerified) .map(|k| k.formatted_fingerprint()) diff --git a/src/context.rs b/src/context.rs index a037e7a7f..113860431 100644 --- a/src/context.rs +++ b/src/context.rs @@ -14,7 +14,7 @@ use crate::events::Event; use crate::imap::*; use crate::job::*; use crate::job_thread::JobThread; -use crate::key::Key; +use crate::key::{DcKey, Key, SignedPublicKey}; use crate::login_param::LoginParam; use crate::lot::Lot; use crate::message::{self, Message, MessengerMessage, MsgId}; @@ -251,10 +251,9 @@ impl Context { rusqlite::NO_PARAMS, ); - let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) { - key.fingerprint() - } else { - "".into() + let fingerprint_str = match SignedPublicKey::load_self(self) { + Ok(key) => Key::from(key).fingerprint(), + Err(err) => format!("", err), }; let inbox_watch = self.get_config_int(Config::InboxWatch); diff --git a/src/dc_tools.rs b/src/dc_tools.rs index e510be1d1..b43696b03 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -12,7 +12,7 @@ use chrono::{Local, TimeZone}; use rand::{thread_rng, Rng}; use crate::context::Context; -use crate::error::{bail, ensure, Error}; +use crate::error::{bail, Error}; use crate::events::Event; pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool { @@ -452,6 +452,23 @@ pub(crate) fn time() -> i64 { .as_secs() as i64 } +/// An invalid email address was encountered +#[derive(Debug, thiserror::Error)] +#[error("Invalid email address: {message} ({addr})")] +pub struct InvalidEmailError { + message: String, + addr: String, +} + +impl InvalidEmailError { + fn new(msg: impl Into, addr: impl Into) -> InvalidEmailError { + InvalidEmailError { + message: msg.into(), + addr: addr.into(), + } + } +} + /// Very simple email address wrapper. /// /// Represents an email address, right now just the `name@domain` portion. @@ -475,7 +492,7 @@ pub struct EmailAddress { } impl EmailAddress { - pub fn new(input: &str) -> Result { + pub fn new(input: &str) -> Result { input.parse::() } } @@ -487,35 +504,58 @@ impl fmt::Display for EmailAddress { } impl FromStr for EmailAddress { - type Err = Error; + type Err = InvalidEmailError; /// Performs a dead-simple parse of an email address. - fn from_str(input: &str) -> Result { - ensure!(!input.is_empty(), "empty string is not valid"); + fn from_str(input: &str) -> Result { + if input.is_empty() { + return Err(InvalidEmailError::new("empty string is not valid", input)); + } let parts: Vec<&str> = input.rsplitn(2, '@').collect(); + let err = |msg: &str| { + Err(InvalidEmailError { + message: msg.to_string(), + addr: input.to_string(), + }) + }; match &parts[..] { [domain, local] => { - ensure!( - !local.is_empty(), - "empty string is not valid for local part" - ); - ensure!(domain.len() > 3, "domain is too short"); - + if local.is_empty() { + return err("empty string is not valid for local part"); + } + if domain.len() <= 3 { + return err("domain is too short"); + } let dot = domain.find('.'); - ensure!(dot.is_some(), "invalid domain"); - ensure!(dot.unwrap() < domain.len() - 2, "invalid domain"); - + match dot { + None => { + return err("invalid domain"); + } + Some(dot_idx) => { + if dot_idx >= domain.len() - 2 { + return err("invalid domain"); + } + } + } Ok(EmailAddress { local: (*local).to_string(), domain: (*domain).to_string(), }) } - _ => bail!("missing '@' character"), + _ => err("missing '@' character"), } } } +impl rusqlite::types::ToSql for EmailAddress { + fn to_sql(&self) -> rusqlite::Result { + let val = rusqlite::types::Value::Text(self.to_string()); + let out = rusqlite::types::ToSqlOutput::Owned(val); + Ok(out) + } +} + /// Utility to check if a in the binary represantion of listflags /// the bit at position bitindex is 1. pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool { diff --git a/src/e2ee.rs b/src/e2ee.rs index ec101fc91..b3e650d25 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -1,19 +1,16 @@ //! End-to-end encryption support. use std::collections::HashSet; -use std::convert::TryFrom; use mailparse::ParsedMail; use num_traits::FromPrimitive; use crate::aheader::*; use crate::config::Config; -use crate::constants::KeyGenType; use crate::context::Context; -use crate::dc_tools::EmailAddress; use crate::error::*; use crate::headerdef::{HeaderDef, HeaderDefMap}; -use crate::key::{self, Key, KeyPairUse, SignedPublicKey}; +use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey}; use crate::keyring::*; use crate::peerstate::*; use crate::pgp; @@ -38,7 +35,7 @@ impl EncryptHelper { Some(addr) => addr, }; - let public_key = load_or_generate_self_public_key(context, &addr)?; + let public_key = SignedPublicKey::load_self(context)?; Ok(EncryptHelper { prefer_encrypt, @@ -108,8 +105,7 @@ impl EncryptHelper { } let public_key = Key::from(self.public_key.clone()); keyring.add_ref(&public_key); - let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql) - .ok_or_else(|| format_err!("missing own private key"))?; + let sign_key = Key::from(SignedSecretKey::load_self(context)?); let raw_message = mail_to_encrypt.build().as_string().into_bytes(); @@ -189,41 +185,6 @@ pub fn try_decrypt( Ok((out_mail, signatures)) } -/// Load public key from database or generate a new one. -/// -/// This will load a public key from the database, generating and -/// storing a new one when one doesn't exist yet. Care is taken to -/// only generate one key per context even when multiple threads call -/// this function concurrently. -fn load_or_generate_self_public_key( - context: &Context, - self_addr: impl AsRef, -) -> Result { - if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) { - return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key")); - } - let _guard = context.generating_key_mutex.lock().unwrap(); - - // Check again in case the key was generated while we were waiting for the lock. - if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) { - return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key")); - } - - let start = std::time::Instant::now(); - - let keygen_type = - KeyGenType::from_i32(context.get_config_int(Config::KeyGenType)).unwrap_or_default(); - info!(context, "Generating keypair with type {}", keygen_type); - let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?, keygen_type)?; - key::store_self_keypair(context, &keypair, KeyPairUse::Default)?; - info!( - context, - "Keypair generated in {:.3}s.", - start.elapsed().as_secs() - ); - Ok(keypair.public) -} - /// Returns a reference to the encrypted payload and validates the autocrypt structure. fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> { ensure!( @@ -345,6 +306,7 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool { /// /// If this succeeds you are also guaranteed that the /// [Config::ConfiguredAddr] is configured, this address is returned. +// TODO, remove this once deltachat::key::Key no longer exists. pub fn ensure_secret_key_exists(context: &Context) -> Result { let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| { format_err!(concat!( @@ -352,7 +314,7 @@ pub fn ensure_secret_key_exists(context: &Context) -> Result { "cannot ensure secret key if not configured." )) })?; - load_or_generate_self_public_key(context, &self_addr)?; + SignedPublicKey::load_self(context)?; Ok(self_addr) } @@ -403,47 +365,6 @@ Sent with my Delta Chat Messenger: https://delta.chat"; ); } - mod load_or_generate_self_public_key { - use super::*; - - #[test] - fn test_existing() { - let t = dummy_context(); - let addr = configure_alice_keypair(&t.ctx); - let key = load_or_generate_self_public_key(&t.ctx, addr); - assert!(key.is_ok()); - } - - #[test] - fn test_generate() { - let t = dummy_context(); - let addr = "alice@example.org"; - let key0 = load_or_generate_self_public_key(&t.ctx, addr); - assert!(key0.is_ok()); - let key1 = load_or_generate_self_public_key(&t.ctx, addr); - assert!(key1.is_ok()); - assert_eq!(key0.unwrap(), key1.unwrap()); - } - - #[test] - fn test_generate_concurrent() { - use std::sync::Arc; - use std::thread; - - let t = dummy_context(); - let ctx = Arc::new(t.ctx); - let ctx0 = Arc::clone(&ctx); - let thr0 = - thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org")); - let ctx1 = Arc::clone(&ctx); - let thr1 = - thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org")); - let res0 = thr0.join().unwrap(); - let res1 = thr1.join().unwrap(); - assert_eq!(res0.unwrap(), res1.unwrap()); - } - } - #[test] fn test_has_decrypted_pgp_armor() { let data = b" -----BEGIN PGP MESSAGE-----"; diff --git a/src/imex.rs b/src/imex.rs index f27219e94..849705d04 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -17,7 +17,7 @@ use crate::e2ee; use crate::error::*; use crate::events::Event; use crate::job::*; -use crate::key::{self, Key}; +use crate::key::{self, DcKey, Key, SignedSecretKey}; use crate::message::{Message, MsgId}; use crate::mimeparser::SystemMessage; use crate::param::*; @@ -175,9 +175,7 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result passphrase.len() >= 2, "Passphrase must be at least 2 chars long." ); - let self_addr = e2ee::ensure_secret_key_exists(context)?; - let private_key = Key::from_self_private(context, self_addr, &context.sql) - .ok_or_else(|| format_err!("Failed to get private key."))?; + let private_key = Key::from(SignedSecretKey::load_self(context)?); let ac_headers = match context.get_config_bool(Config::E2eeEnabled) { false => None, true => Some(("Autocrypt-Prefer-Encrypt", "mutual")), diff --git a/src/key.rs b/src/key.rs index fb658e92c..15609c27d 100644 --- a/src/key.rs +++ b/src/key.rs @@ -4,14 +4,16 @@ use std::collections::BTreeMap; use std::io::Cursor; use std::path::Path; +use num_traits::FromPrimitive; use pgp::composed::Deserializable; use pgp::ser::Serialize; use pgp::types::{KeyTrait, SecretKeyTrait}; +use crate::config::Config; use crate::constants::*; use crate::context::Context; -use crate::dc_tools::*; -use crate::sql::Sql; +use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError}; +use crate::sql; // Re-export key types pub use crate::pgp::KeyPair; @@ -19,11 +21,22 @@ pub use pgp::composed::{SignedPublicKey, SignedSecretKey}; /// Error type for deltachat key handling. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum Error { #[error("Could not decode base64")] Base64Decode(#[from] base64::DecodeError), - #[error("rPGP error: {0}")] - PgpError(#[from] pgp::errors::Error), + #[error("rPGP error: {}", _0)] + Pgp(#[from] pgp::errors::Error), + #[error("Failed to generate PGP key: {}", _0)] + Keygen(#[from] crate::pgp::PgpKeygenError), + #[error("Failed to load key: {}", _0)] + LoadKey(#[from] sql::Error), + #[error("Failed to save generated key: {}", _0)] + StoreKey(#[from] SaveKeyError), + #[error("No address configured")] + NoConfiguredAddr, + #[error("Configured address is invalid: {}", _0)] + InvalidConfiguredAddr(#[from] InvalidEmailError), } pub type Result = std::result::Result; @@ -51,6 +64,9 @@ pub trait DcKey: Serialize + Deserializable { Self::from_slice(&bytes) } + /// Load the users' default key from the database. + fn load_self(context: &Context) -> Result; + /// Serialise the key to a base64 string. fn to_base64(&self) -> String { // Not using Serialize::to_bytes() to make clear *why* it is @@ -65,10 +81,91 @@ pub trait DcKey: Serialize + Deserializable { impl DcKey for SignedPublicKey { type KeyType = SignedPublicKey; + + fn load_self(context: &Context) -> Result { + match context.sql.query_row( + r#" + SELECT public_key + FROM keypairs + WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr") + AND is_default=1; + "#, + params![], + |row| row.get::<_, Vec>(0), + ) { + Ok(bytes) => Self::from_slice(&bytes), + Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => { + let keypair = generate_keypair(context)?; + Ok(keypair.public) + } + Err(err) => Err(err.into()), + } + } } impl DcKey for SignedSecretKey { type KeyType = SignedSecretKey; + + fn load_self(context: &Context) -> Result { + match context.sql.query_row( + r#" + SELECT private_key + FROM keypairs + WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr") + AND is_default=1; + "#, + params![], + |row| row.get::<_, Vec>(0), + ) { + Ok(bytes) => Self::from_slice(&bytes), + Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => { + let keypair = generate_keypair(context)?; + Ok(keypair.secret) + } + Err(err) => Err(err.into()), + } + } +} + +fn generate_keypair(context: &Context) -> Result { + let addr = context + .get_config(Config::ConfiguredAddr) + .ok_or_else(|| Error::NoConfiguredAddr)?; + let addr = EmailAddress::new(&addr)?; + let _guard = context.generating_key_mutex.lock().unwrap(); + + // Check if the key appeared while we were waiting on the lock. + match context.sql.query_row( + r#" + SELECT public_key, private_key + FROM keypairs + WHERE addr=?1 + AND is_default=1; + "#, + params![addr], + |row| Ok((row.get::<_, Vec>(0)?, row.get::<_, Vec>(1)?)), + ) { + Ok((pub_bytes, sec_bytes)) => Ok(KeyPair { + addr, + public: SignedPublicKey::from_slice(&pub_bytes)?, + secret: SignedSecretKey::from_slice(&sec_bytes)?, + }), + Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => { + let start = std::time::Instant::now(); + let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType)) + .unwrap_or_default(); + info!(context, "Generating keypair with type {}", keytype); + let keypair = crate::pgp::create_keypair(addr, keytype)?; + store_self_keypair(context, &keypair, KeyPairUse::Default)?; + info!( + context, + "Keypair generated in {:.3}s.", + start.elapsed().as_secs() + ); + Ok(keypair) + } + Err(err) => Err(err.into()), + } } /// Cryptographic key @@ -185,34 +282,6 @@ impl Key { } } - pub fn from_self_public( - context: &Context, - self_addr: impl AsRef, - sql: &Sql, - ) -> Option { - let addr = self_addr.as_ref(); - - sql.query_get_value( - context, - "SELECT public_key FROM keypairs WHERE addr=? AND is_default=1;", - &[addr], - ) - .and_then(|blob: Vec| Self::from_slice(&blob, KeyType::Public)) - } - - pub fn from_self_private( - context: &Context, - self_addr: impl AsRef, - sql: &Sql, - ) -> Option { - sql.query_get_value( - context, - "SELECT private_key FROM keypairs WHERE addr=? AND is_default=1;", - &[self_addr.as_ref()], - ) - .and_then(|blob: Vec| Self::from_slice(&blob, KeyType::Private)) - } - pub fn to_bytes(&self) -> Vec { match self { Key::Public(k) => k.to_bytes().unwrap_or_default(), @@ -539,6 +608,59 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD } } + #[test] + fn test_load_self_existing() { + let alice = alice_keypair(); + let t = dummy_context(); + configure_alice_keypair(&t.ctx); + let pubkey = SignedPublicKey::load_self(&t.ctx).unwrap(); + assert_eq!(alice.public, pubkey); + let seckey = SignedSecretKey::load_self(&t.ctx).unwrap(); + assert_eq!(alice.secret, seckey); + } + + #[test] + #[ignore] // generating keys is expensive + fn test_load_self_generate_public() { + let t = dummy_context(); + t.ctx + .set_config(Config::ConfiguredAddr, Some("alice@example.com")) + .unwrap(); + let key = SignedPublicKey::load_self(&t.ctx); + assert!(key.is_ok()); + } + + #[test] + #[ignore] // generating keys is expensive + fn test_load_self_generate_secret() { + let t = dummy_context(); + t.ctx + .set_config(Config::ConfiguredAddr, Some("alice@example.com")) + .unwrap(); + let key = SignedSecretKey::load_self(&t.ctx); + assert!(key.is_ok()); + } + + #[test] + #[ignore] // generating keys is expensive + fn test_load_self_generate_concurrent() { + use std::sync::Arc; + use std::thread; + + let t = dummy_context(); + t.ctx + .set_config(Config::ConfiguredAddr, Some("alice@example.com")) + .unwrap(); + let ctx = Arc::new(t.ctx); + let ctx0 = Arc::clone(&ctx); + let thr0 = thread::spawn(move || SignedPublicKey::load_self(&ctx0)); + let ctx1 = Arc::clone(&ctx); + let thr1 = thread::spawn(move || SignedPublicKey::load_self(&ctx1)); + let res0 = thr0.join().unwrap(); + let res1 = thr1.join().unwrap(); + assert_eq!(res0.unwrap(), res1.unwrap()); + } + #[test] fn test_ascii_roundtrip() { let public_key = Key::from(KEYPAIR.public.clone()); diff --git a/src/pgp.rs b/src/pgp.rs index 61fad558f..287c7b0bf 100644 --- a/src/pgp.rs +++ b/src/pgp.rs @@ -119,7 +119,7 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap Option< } fn get_self_fingerprint(context: &Context) -> Option { - if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) { - if let Some(key) = Key::from_self_public(context, self_addr, &context.sql) { - return Some(key.fingerprint()); + match SignedPublicKey::load_self(context) { + Ok(key) => Some(Key::from(key).fingerprint()), + Err(_) => { + warn!(context, "get_self_fingerprint(): failed to load key"); + None } } - None } /// Take a scanned QR-code and do the setup-contact/join-group handshake. From 66d5c3f62052f12e1f894b91ce967230bdfbcb0b Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Wed, 22 Apr 2020 22:32:47 +0300 Subject: [PATCH 103/156] job: derive PartialEq for Action This makes it possible to compare action priorities in Rust code. --- src/job.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/job.rs b/src/job.rs index e12f32cb1..7a36b269e 100644 --- a/src/job.rs +++ b/src/job.rs @@ -75,7 +75,19 @@ impl Default for Thread { } } -#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)] +#[derive( + Debug, + Display, + Copy, + Clone, + PartialEq, + Eq, + PartialOrd, + FromPrimitive, + ToPrimitive, + FromSql, + ToSql, +)] #[repr(i32)] pub enum Action { Unknown = 0, From 502ec2a56fe6262801e8ab5a1985c397d23917c5 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Thu, 23 Apr 2020 22:23:16 +0300 Subject: [PATCH 104/156] job: new API for dynamic job creation Job::new() can be used to create jobs in-memory Job.update() is replaced with Job.save() which can create new database entries and consumes Job to avoid the need to update job ID after saving it to the database. --- src/job.rs | 111 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 42 deletions(-) diff --git a/src/job.rs b/src/job.rs index 7a36b269e..c73fee71b 100644 --- a/src/job.rs +++ b/src/job.rs @@ -162,30 +162,68 @@ impl fmt::Display for Job { } impl Job { - /// Deletes the job from the database. - fn delete(&self, context: &Context) -> bool { - context - .sql - .execute("DELETE FROM jobs WHERE id=?;", params![self.job_id as i32]) - .is_ok() + fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self { + let timestamp = time(); + + Self { + job_id: 0, + action, + foreign_id, + desired_timestamp: timestamp + delay_seconds, + added_timestamp: timestamp, + tries: 0, + param, + pending_error: None, + } } - /// Updates the job already stored in the database. + /// Deletes the job from the database. + fn delete(&self, context: &Context) -> bool { + if self.job_id != 0 { + context + .sql + .execute("DELETE FROM jobs WHERE id=?;", params![self.job_id as i32]) + .is_ok() + } else { + // Already deleted. + true + } + } + + /// Saves the job to the database, creating a new entry if necessary. /// - /// To add a new job, use [job_add]. - fn update(&self, context: &Context) -> bool { - sql::execute( - context, - &context.sql, - "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;", - params![ - self.desired_timestamp, - self.tries as i64, - self.param.to_string(), - self.job_id as i32, - ], - ) - .is_ok() + /// The Job is consumed by this method. + fn save(self, context: &Context) -> bool { + let thread: Thread = self.action.into(); + + if self.job_id != 0 { + sql::execute( + context, + &context.sql, + "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;", + params![ + self.desired_timestamp, + self.tries as i64, + self.param.to_string(), + self.job_id as i32, + ], + ) + .is_ok() + } else { + sql::execute( + context, + &context.sql, + "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);", + params![ + self.added_timestamp, + thread, + self.action, + self.foreign_id, + self.param.to_string(), + self.desired_timestamp + ] + ).is_ok() + } } fn smtp_send( @@ -1071,7 +1109,6 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) { job.tries = tries; let time_offset = get_backoff_time_offset(tries); job.desired_timestamp = time() + time_offset; - job.update(context); info!( context, "{}-job #{} not succeeded on try #{}, retry in {} seconds.", @@ -1080,6 +1117,7 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) { tries, time_offset ); + job.save(context); if thread == Thread::Smtp && tries < JOB_RETRIES - 1 { context .smtp_state @@ -1237,27 +1275,16 @@ pub fn job_add( return; } - let timestamp = time(); - let thread: Thread = action.into(); + let job = Job::new(action, foreign_id as u32, param, delay_seconds); + job.save(context); - sql::execute( - context, - &context.sql, - "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);", - params![ - timestamp, - thread, - action, - foreign_id, - param.to_string(), - (timestamp + delay_seconds as i64) - ] - ).ok(); - - match thread { - Thread::Imap => interrupt_inbox_idle(context), - Thread::Smtp => interrupt_smtp_idle(context), - Thread::Unknown => {} + if delay_seconds == 0 { + let thread: Thread = action.into(); + match thread { + Thread::Imap => interrupt_inbox_idle(context), + Thread::Smtp => interrupt_smtp_idle(context), + Thread::Unknown => {} + } } } From fc03f4c87a9905d5841b43832de814ffdf83454d Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Wed, 22 Apr 2020 22:33:47 +0300 Subject: [PATCH 105/156] job: generate IMAP deletion jobs dynamically This prevents generation of a large number of jobs when IMAP deletion setting is enabled for the first time. Writing jobs to the database locks it for readers and may cause UI freezing, because chatlist and messages can't be read until all jobs are written. Note that on failure job will be written to the database, to make sure it is postponed instead of being retried immediately. --- src/job.rs | 68 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/src/job.rs b/src/job.rs index c73fee71b..d4879b574 100644 --- a/src/job.rs +++ b/src/job.rs @@ -993,39 +993,34 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<()> { Ok(()) } -fn add_imap_deletion_jobs(context: &Context) -> sql::Result<()> { +fn load_imap_deletion_msgid(context: &Context) -> sql::Result> { if let Some(delete_server_after) = context.get_config_delete_server_after() { let threshold_timestamp = time() - delete_server_after; - // Select all expired messages which don't have a - // corresponding message deletion job yet. - let msg_ids = context.sql.query_map( + context.sql.query_row_optional( "SELECT id FROM msgs \ WHERE timestamp < ? \ - AND server_uid != 0 \ - AND NOT EXISTS (SELECT 1 FROM jobs WHERE foreign_id = msgs.id \ - AND action = ?)", - params![threshold_timestamp, Action::DeleteMsgOnImap], + AND server_uid != 0", + params![threshold_timestamp], |row| row.get::<_, MsgId>(0), - |ids| { - ids.collect::, _>>() - .map_err(Into::into) - }, - )?; - - // Schedule IMAP deletion for expired messages. - for msg_id in msg_ids { - job_add( - context, - Action::DeleteMsgOnImap, - msg_id.to_u32() as i32, - Params::new(), - 0, - ) - } + ) + } else { + Ok(None) } +} - Ok(()) +fn load_imap_deletion_job(context: &Context) -> sql::Result> { + let res = if let Some(msg_id) = load_imap_deletion_msgid(context)? { + Some(Job::new( + Action::DeleteMsgOnImap, + msg_id.to_u32(), + Params::new(), + 0, + )) + } else { + None + }; + Ok(res) } pub fn perform_inbox_jobs(context: &Context) { @@ -1035,9 +1030,6 @@ pub fn perform_inbox_jobs(context: &Context) { *context.probe_imap_network.write().unwrap() = false; *context.perform_inbox_jobs_needed.write().unwrap() = false; - if let Err(err) = add_imap_deletion_jobs(context) { - warn!(context, "Can't add IMAP message deletion jobs: {}", err); - } job_perform(context, Thread::Imap, probe_imap_network); info!(context, "dc_perform_inbox_jobs ended.",); } @@ -1328,7 +1320,7 @@ fn load_next_job(context: &Context, thread: Thread, probe_network: bool) -> Opti params_probe }; - context + let job = context .sql .query_map( query, @@ -1357,7 +1349,23 @@ fn load_next_job(context: &Context, thread: Thread, probe_network: bool) -> Opti Ok(None) }, ) - .unwrap_or_default() + .unwrap_or_default(); + + if thread == Thread::Imap { + if let Some(job) = job { + if job.action < Action::DeleteMsgOnImap { + load_imap_deletion_job(context) + .unwrap_or_default() + .or(Some(job)) + } else { + Some(job) + } + } else { + load_imap_deletion_job(context).unwrap_or_default() + } + } else { + job + } } #[cfg(test)] From 46fb6a21ee7efdc608b5ee9e0f459ac79e8e9840 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sat, 25 Apr 2020 01:43:26 +0300 Subject: [PATCH 106/156] job: perform no more than 20 IMAP jobs in a row Let the thread load new messages. This may happen when user switches the setting to delete messages on the server on and there are a lot of messages to delete. --- src/job.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/job.rs b/src/job.rs index d4879b574..7e101b9f6 100644 --- a/src/job.rs +++ b/src/job.rs @@ -1043,7 +1043,17 @@ pub fn perform_sentbox_jobs(context: &Context) { } fn job_perform(context: &Context, thread: Thread, probe_network: bool) { + let mut jobs_loaded = 0; + while let Some(mut job) = load_next_job(context, thread, probe_network) { + jobs_loaded += 1; + if thread == Thread::Imap && jobs_loaded > 20 { + // Let the fetch run, but return back to the job afterwards. + info!(context, "postponing {}-job {} to run fetch...", thread, job); + *context.perform_inbox_jobs_needed.write().unwrap() = true; + break; + } + info!(context, "{}-job {} started...", thread, job); // some configuration jobs are "exclusive": From 95cac4dfb9a1edf31052892497e2140ebc5a1622 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 25 Apr 2020 01:37:56 +0200 Subject: [PATCH 107/156] add context-uptime to dc_get_info() --- src/context.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/context.rs b/src/context.rs index 113860431..0be974ae2 100644 --- a/src/context.rs +++ b/src/context.rs @@ -21,6 +21,7 @@ use crate::message::{self, Message, MessengerMessage, MsgId}; use crate::param::Params; use crate::smtp::Smtp; use crate::sql::Sql; +use std::time::Instant; /// Callback function type for [Context] /// @@ -57,6 +58,7 @@ pub struct Context { /// Mutex to avoid generating the key for the user more than once. pub generating_key_mutex: Mutex<()>, pub translated_stockstrings: RwLock>, + creation_time: Instant, } #[derive(Debug, PartialEq, Eq)] @@ -138,6 +140,7 @@ impl Context { perform_inbox_jobs_needed: Arc::new(RwLock::new(false)), generating_key_mutex: Mutex::new(()), translated_stockstrings: RwLock::new(HashMap::new()), + creation_time: std::time::Instant::now(), }; ensure!( @@ -311,6 +314,12 @@ impl Context { ); res.insert("fingerprint", fingerprint_str); + let elapsed = self.creation_time.elapsed().as_secs(); + let hours = elapsed / 3600; + let minutes = elapsed % 3600 / 60; + let seconds = elapsed % 3600 % 60; + res.insert("uptime", format!("{}h {}m {}s", hours, minutes, seconds)); + res } From dff1ae0fb4db1a1e228d9fdb1928a73b119e7ff7 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 26 Apr 2020 11:16:59 +0200 Subject: [PATCH 108/156] move duration-formatting to a separate function and add tests for that --- src/context.rs | 8 +++----- src/dc_tools.rs | 43 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/context.rs b/src/context.rs index 0be974ae2..4bc2c3996 100644 --- a/src/context.rs +++ b/src/context.rs @@ -9,6 +9,7 @@ use crate::chat::*; use crate::config::Config; use crate::constants::*; use crate::contact::*; +use crate::dc_tools::duration_to_str; use crate::error::*; use crate::events::Event; use crate::imap::*; @@ -314,11 +315,8 @@ impl Context { ); res.insert("fingerprint", fingerprint_str); - let elapsed = self.creation_time.elapsed().as_secs(); - let hours = elapsed / 3600; - let minutes = elapsed % 3600 / 60; - let seconds = elapsed % 3600 % 60; - res.insert("uptime", format!("{}h {}m {}s", hours, minutes, seconds)); + let elapsed = self.creation_time.elapsed(); + res.insert("uptime", duration_to_str(elapsed.unwrap_or_default())); res } diff --git a/src/dc_tools.rs b/src/dc_tools.rs index b43696b03..0745dca9b 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -5,7 +5,7 @@ use core::cmp::{max, min}; use std::borrow::Cow; use std::path::{Path, PathBuf}; use std::str::FromStr; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use std::{fmt, fs}; use chrono::{Local, TimeZone}; @@ -75,6 +75,14 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String { ts.format("%Y.%m.%d %H:%M:%S").to_string() } +pub fn duration_to_str(duration: Duration) -> String { + let secs = duration.as_secs(); + let h = secs / 3600; + let m = secs % 3600 / 60; + let s = secs % 3600 % 60; + format!("{}h {}m {}s", h, m, s) +} + pub(crate) fn dc_gm2local_offset() -> i64 { /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime. the function may return negative values. */ @@ -854,4 +862,37 @@ mod tests { let next = dc_smeared_time(&t.ctx); assert!((start + count - 1) < next); } + + #[test] + fn test_duration_to_str() { + assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s"); + assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s"); + assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s"); + assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s"); + assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s"); + assert_eq!( + duration_to_str(Duration::from_secs(59 * 60 + 59)), + "0h 59m 59s" + ); + assert_eq!( + duration_to_str(Duration::from_secs(59 * 60 + 60)), + "1h 0m 0s" + ); + assert_eq!( + duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)), + "2h 59m 59s" + ); + assert_eq!( + duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)), + "3h 0m 0s" + ); + assert_eq!( + duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)), + "3h 0m 59s" + ); + assert_eq!( + duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)), + "3h 1m 0s" + ); + } } From 39cb9c425cb0a2147f68f5b147bee717c57dacee Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 26 Apr 2020 11:44:12 +0200 Subject: [PATCH 109/156] fixup --- src/dc_tools.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dc_tools.rs b/src/dc_tools.rs index 0745dca9b..1adaa5ed8 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -78,8 +78,8 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String { pub fn duration_to_str(duration: Duration) -> String { let secs = duration.as_secs(); let h = secs / 3600; - let m = secs % 3600 / 60; - let s = secs % 3600 % 60; + let m = (secs % 3600) / 60; + let s = (secs % 3600) % 60; format!("{}h {}m {}s", h, m, s) } From 432e4b7f0a961d7e6601bbefba2917246b117f98 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 26 Apr 2020 11:19:37 +0200 Subject: [PATCH 110/156] use std::time::SystemTime for uptime calculation std::time::Instant does not count eg. doze-time on android --- src/context.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/context.rs b/src/context.rs index 4bc2c3996..19e47f41c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -22,7 +22,7 @@ use crate::message::{self, Message, MessengerMessage, MsgId}; use crate::param::Params; use crate::smtp::Smtp; use crate::sql::Sql; -use std::time::Instant; +use std::time::SystemTime; /// Callback function type for [Context] /// @@ -59,7 +59,7 @@ pub struct Context { /// Mutex to avoid generating the key for the user more than once. pub generating_key_mutex: Mutex<()>, pub translated_stockstrings: RwLock>, - creation_time: Instant, + creation_time: SystemTime, } #[derive(Debug, PartialEq, Eq)] @@ -141,7 +141,7 @@ impl Context { perform_inbox_jobs_needed: Arc::new(RwLock::new(false)), generating_key_mutex: Mutex::new(()), translated_stockstrings: RwLock::new(HashMap::new()), - creation_time: std::time::Instant::now(), + creation_time: std::time::SystemTime::now(), }; ensure!( From bfdd6f36e213a271ec09afc8624921f1be68d4d5 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 27 Apr 2020 12:24:12 +0200 Subject: [PATCH 111/156] regard line with ony '--' as footer mark partly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the footer mark normally used in email-conversations is `-- `, note the trailing space, see RFC 3676, §4.3 unfortunately, the final space is removed by some providers, which lead to footers showing up on delta-to-delta-conversations (on nondc-to-delta, this is not an issue as we cannot be sure anyway and show a [...] therefore) this change accepts lines with only `--` as a footer separator if there is no other footer separator and if the line before is empty and the line after is not. as there is still some chance to remove text accidentally, see tests, some protection against that is needed in another commit. --- src/simplify.rs | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index cf61dc8ae..985639cf1 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -1,13 +1,25 @@ /// Remove standard (RFC 3676, §4.3) footer if it is found. fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] { + let mut nearly_standard_footer = None; for (ix, &line) in lines.iter().enumerate() { - // quoted-printable may encode `-- ` to `-- =20` which is converted - // back to `-- ` match line { + // some providers encode `-- ` to `-- =20` which results in `-- ` "-- " | "-- " => return &lines[..ix], + // some providers encode `-- ` to `=2D-` which results in only `--`; + // use that only when no other footer is found + // and if the line before is empty and the line after is not empty + "--" => { + if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != "" + { + nearly_standard_footer = Some(ix); + } + } _ => (), } } + if let Some(ix) = nearly_standard_footer { + return &lines[..ix]; + } lines } @@ -268,4 +280,31 @@ mod tests { assert_eq!(lines, &["not a quote", "> first", "> second"]); assert!(!has_top_quote); } + + #[test] + fn test_remove_message_footer() { + let input = "text\n--\nno footer".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n--\nno footer"); + + let input = "text\n\n--\n\nno footer".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n\n--\n\nno footer"); + + let input = "text\n\n-- no footer\n\n".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n\n-- no footer"); + + let input = "text\n\n--\nno footer\n-- \nfooter".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text\n\n--\nno footer"); + + let input = "text\n\n--\ntreated as footer when unescaped".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, "text"); // see remove_message_footer() for some explanations + + let input = "--\ntreated as footer when unescaped".to_string(); + let (plain, _) = simplify(input, true); + assert_eq!(plain, ""); + } } From 459fec56db7b851025427640ad5f016ca3fc9424 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 27 Apr 2020 16:02:45 +0200 Subject: [PATCH 112/156] protect '--' in message from being treated as a footer-beginning --- src/mimefactory.rs | 3 ++- src/simplify.rs | 40 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 127a23111..078e608e5 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -15,6 +15,7 @@ use crate::message::{self, Message}; use crate::mimeparser::SystemMessage; use crate::param::*; use crate::peerstate::{Peerstate, PeerstateVerifiedStatus}; +use crate::simplify::escape_message_footer_marks; use crate::stock::StockMessage; // attachments of 25 mb brutto should work on the majority of providers @@ -807,7 +808,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> { let message_text = format!( "{}{}{}{}{}", fwdhint.unwrap_or_default(), - &final_text, + escape_message_footer_marks(final_text), if !final_text.is_empty() && !footer.is_empty() { "\r\n\r\n" } else { diff --git a/src/simplify.rs b/src/simplify.rs index 985639cf1..eb8bc93c6 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -1,3 +1,19 @@ +// protect lines starting with `--` against being treated as a footer. +// for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B); +// this should be invisible on most systems and there is no need to unescape it again +// (which won't be done by non-deltas anyway) +// +// this escapes a bit more than actually needed by delta (eg. also lines as "-- footer"), +// but for non-delta-compatibility, that seems to be better. +// (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced) +pub fn escape_message_footer_marks(text: &str) -> String { + if text.starts_with("--") { + "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-") + } else { + text.replace("\n--", "\n-\u{200B}-") + } +} + /// Remove standard (RFC 3676, §4.3) footer if it is found. fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] { let mut nearly_standard_footer = None; @@ -281,6 +297,15 @@ mod tests { assert!(!has_top_quote); } + #[test] + fn test_escape_message_footer_marks() { + let esc = escape_message_footer_marks("--\n--text --in line"); + assert_eq!(esc, "-\u{200B}-\n-\u{200B}-text --in line"); + + let esc = escape_message_footer_marks("--\r\n--text"); + assert_eq!(esc, "-\u{200B}-\r\n-\u{200B}-text"); + } + #[test] fn test_remove_message_footer() { let input = "text\n--\nno footer".to_string(); @@ -300,11 +325,20 @@ mod tests { assert_eq!(plain, "text\n\n--\nno footer"); let input = "text\n\n--\ntreated as footer when unescaped".to_string(); - let (plain, _) = simplify(input, true); + let (plain, _) = simplify(input.clone(), true); assert_eq!(plain, "text"); // see remove_message_footer() for some explanations + let escaped = escape_message_footer_marks(&input); + let (plain, _) = simplify(escaped, true); + assert_eq!( + plain, + "text\n\n-\u{200B}-\ntreated as footer when unescaped" + ); let input = "--\ntreated as footer when unescaped".to_string(); - let (plain, _) = simplify(input, true); - assert_eq!(plain, ""); + let (plain, _) = simplify(input.clone(), true); + assert_eq!(plain, ""); // see remove_message_footer() for some explanations + let escaped = escape_message_footer_marks(&input); + let (plain, _) = simplify(escaped, true); + assert_eq!(plain, "-\u{200B}-\ntreated as footer when unescaped"); } } From 2a9b967d2d52a6367970c83022588ffb77498cfc Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 27 Apr 2020 16:32:16 +0200 Subject: [PATCH 113/156] remove footer-escape-character from message texts --- src/simplify.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/simplify.rs b/src/simplify.rs index eb8bc93c6..79d6f6605 100644 --- a/src/simplify.rs +++ b/src/simplify.rs @@ -184,7 +184,8 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> if is_cut_at_end && (!is_cut_at_begin || !empty_body) { ret += " [...]"; } - ret + // redo escaping done by escape_message_footer_marks() + ret.replace("\u{200B}", "") } /** @@ -329,16 +330,13 @@ mod tests { assert_eq!(plain, "text"); // see remove_message_footer() for some explanations let escaped = escape_message_footer_marks(&input); let (plain, _) = simplify(escaped, true); - assert_eq!( - plain, - "text\n\n-\u{200B}-\ntreated as footer when unescaped" - ); + assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped"); let input = "--\ntreated as footer when unescaped".to_string(); let (plain, _) = simplify(input.clone(), true); assert_eq!(plain, ""); // see remove_message_footer() for some explanations let escaped = escape_message_footer_marks(&input); let (plain, _) = simplify(escaped, true); - assert_eq!(plain, "-\u{200B}-\ntreated as footer when unescaped"); + assert_eq!(plain, "--\ntreated as footer when unescaped"); } } From b70e92effbcebf0a7bc5ef201a3fe89a66daff39 Mon Sep 17 00:00:00 2001 From: Hocuri <18012815+Hocuri@users.noreply.github.com> Date: Mon, 27 Apr 2020 19:18:19 +0200 Subject: [PATCH 114/156] Update Readme.md: Explain database path, fix #1431 Also set the database path to ~/deltachat-db so that nothing has to be changed in the command (it can just be copy-pasted). --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6b4613b56..298cc5ee7 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,9 @@ curl https://sh.rustup.rs -sSf | sh Compile and run Delta Chat Core command line utility, using `cargo`: ``` -cargo run --example repl -- /path/to/db +cargo run --example repl -- ~/deltachat-db ``` +where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist. Configure your account (if not already configured): From 2f5b6a115d094fb9dd1d4f48bb5af6bdbc1905fa Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 25 Apr 2020 00:38:49 +0200 Subject: [PATCH 115/156] update provider database --- src/provider/data.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/provider/data.rs b/src/provider/data.rs index 297c3977d..a33c77ca5 100644 --- a/src/provider/data.rs +++ b/src/provider/data.rs @@ -268,7 +268,14 @@ lazy_static::lazy_static! { }; // yandex.ru.md: yandex.ru, yandex.com - // - skipping provider with status OK and no special things to do + static ref P_YANDEX_RU: Provider = Provider { + status: Status::PREPARATION, + before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.", + after_login_hint: "", + overview_page: "https://providers.delta.chat/yandex-ru", + server: vec![ + ], + }; // ziggo.nl.md: ziggo.nl static ref P_ZIGGO_NL: Provider = Provider { @@ -360,6 +367,8 @@ lazy_static::lazy_static! { ("ymail.com", &*P_YAHOO), ("rocketmail.com", &*P_YAHOO), ("yahoodns.net", &*P_YAHOO), + ("yandex.ru", &*P_YANDEX_RU), + ("yandex.com", &*P_YANDEX_RU), ("ziggo.nl", &*P_ZIGGO_NL), ].iter().copied().collect(); } From 46253039df51958b079c1f7e65783283fff2302d Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Tue, 28 Apr 2020 21:25:59 +0300 Subject: [PATCH 116/156] Return BTreeMap instead of HashMap from get_info() BTreeMap is sorted by keys when iterated, making it easier to find the required line. --- deltachat-ffi/src/lib.rs | 4 ++-- src/context.rs | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 47ebba5c6..1e75bd359 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -13,7 +13,7 @@ extern crate human_panic; extern crate num_traits; extern crate serde_json; -use std::collections::HashMap; +use std::collections::BTreeMap; use std::convert::TryInto; use std::ffi::CString; use std::fmt::Write; @@ -434,7 +434,7 @@ pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c } fn render_info( - info: HashMap<&'static str, String>, + info: BTreeMap<&'static str, String>, ) -> std::result::Result { let mut res = String::new(); for (key, value) in &info { diff --git a/src/context.rs b/src/context.rs index 0be974ae2..a69795da3 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,6 +1,6 @@ //! Context module -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::sync::{Arc, Condvar, Mutex, RwLock}; @@ -73,8 +73,8 @@ pub struct RunningState { /// actual keys and their values which will be present are not /// guaranteed. Calling [Context::get_info] also includes information /// about the context on top of the information here. -pub fn get_info() -> HashMap<&'static str, String> { - let mut res = HashMap::new(); +pub fn get_info() -> BTreeMap<&'static str, String> { + let mut res = BTreeMap::new(); res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR)); res.insert("sqlite_version", rusqlite::version().to_string()); res.insert("arch", (std::mem::size_of::() * 8).to_string()); @@ -225,7 +225,7 @@ impl Context { * UI chat/message related API ******************************************************************************/ - pub fn get_info(&self) -> HashMap<&'static str, String> { + pub fn get_info(&self) -> BTreeMap<&'static str, String> { let unset = "0"; let l = LoginParam::from_database(self, ""); let l2 = LoginParam::from_database(self, "configured_"); From 737a741a544c406acc46f8fad532f10b1f564e58 Mon Sep 17 00:00:00 2001 From: jikstra Date: Tue, 28 Apr 2020 16:42:44 +0200 Subject: [PATCH 117/156] Support lowercased openpgp4fpr uri scheme --- src/qr.rs | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/qr.rs b/src/qr.rs index 890c1df07..61480a94a 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -37,6 +37,10 @@ impl Into for Error { } } +pub fn starts_with_ignore_case(string: &str, pattern: &str) -> bool { + string.starts_with(&pattern.to_uppercase()) || string.starts_with(&pattern.to_lowercase()) +} + /// Check a scanned QR code. /// The function should be called after a QR code is scanned. /// The function takes the raw text scanned and checks what can be done with it. @@ -45,7 +49,7 @@ pub fn check_qr(context: &Context, qr: impl AsRef) -> Lot { info!(context, "Scanned QR code: {}", qr); - if qr.starts_with(OPENPGP4FPR_SCHEME) { + if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) { decode_openpgp(context, qr) } else if qr.starts_with(DCACCOUNT_SCHEME) { decode_account(context, qr) @@ -517,6 +521,16 @@ mod tests { assert_ne!(res.get_id(), 0); assert_eq!(res.get_text1().unwrap(), "test ? test !"); + let res = check_qr( + &ctx.ctx, + "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL" + ); + + println!("{:?}", res); + assert_eq!(res.get_state(), LotState::QrAskVerifyGroup); + assert_ne!(res.get_id(), 0); + assert_eq!(res.get_text1().unwrap(), "test ? test !"); + let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap(); assert_eq!(contact.get_addr(), "cli@deltachat.de"); } @@ -534,6 +548,16 @@ mod tests { assert_eq!(res.get_state(), LotState::QrAskVerifyContact); assert_ne!(res.get_id(), 0); + // Test it again with lowercase + let res = check_qr( + &ctx.ctx, + "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB" + ); + + println!("{:?}", res); + assert_eq!(res.get_state(), LotState::QrAskVerifyContact); + assert_ne!(res.get_id(), 0); + let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap(); assert_eq!(contact.get_addr(), "cli@deltachat.de"); assert_eq!(contact.get_name(), "Jörn P. P."); @@ -554,6 +578,20 @@ mod tests { ); assert_eq!(res.get_id(), 0); + // Test it again with lowercased openpgp4fpr uri scheme + + let res = check_qr( + &ctx.ctx, + "openpgp4fpr:1234567890123456789012345678901234567890", + ); + assert_eq!(res.get_state(), LotState::QrFprWithoutAddr); + assert_eq!( + res.get_text1().unwrap(), + "1234 5678 9012 3456 7890\n1234 5678 9012 3456 7890" + ); + assert_eq!(res.get_id(), 0); + + let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890"); assert_eq!(res.get_state(), LotState::QrError); assert_eq!(res.get_id(), 0); From 9eab96090d1e39f4ca0709ff21183f57e943032c Mon Sep 17 00:00:00 2001 From: jikstra Date: Tue, 28 Apr 2020 16:44:31 +0200 Subject: [PATCH 118/156] Cargo fmt --- src/qr.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/qr.rs b/src/qr.rs index 61480a94a..ad543223f 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -38,7 +38,7 @@ impl Into for Error { } pub fn starts_with_ignore_case(string: &str, pattern: &str) -> bool { - string.starts_with(&pattern.to_uppercase()) || string.starts_with(&pattern.to_lowercase()) + string.starts_with(&pattern.to_uppercase()) || string.starts_with(&pattern.to_lowercase()) } /// Check a scanned QR code. @@ -591,7 +591,6 @@ mod tests { ); assert_eq!(res.get_id(), 0); - let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890"); assert_eq!(res.get_state(), LotState::QrError); assert_eq!(res.get_id(), 0); From 50e18f84c2a765b70ff6120a8f969d24ab510f4f Mon Sep 17 00:00:00 2001 From: jikstra Date: Tue, 28 Apr 2020 17:00:44 +0200 Subject: [PATCH 119/156] Also support lowercased dcaccount: uris --- src/qr.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/qr.rs b/src/qr.rs index ad543223f..8e97c544b 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -38,7 +38,7 @@ impl Into for Error { } pub fn starts_with_ignore_case(string: &str, pattern: &str) -> bool { - string.starts_with(&pattern.to_uppercase()) || string.starts_with(&pattern.to_lowercase()) + string.to_lowercase().starts_with(&pattern.to_lowercase()) } /// Check a scanned QR code. @@ -51,7 +51,7 @@ pub fn check_qr(context: &Context, qr: impl AsRef) -> Lot { if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) { decode_openpgp(context, qr) - } else if qr.starts_with(DCACCOUNT_SCHEME) { + } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) { decode_account(context, qr) } else if qr.starts_with(MAILTO_SCHEME) { decode_mailto(context, qr) @@ -521,6 +521,8 @@ mod tests { assert_ne!(res.get_id(), 0); assert_eq!(res.get_text1().unwrap(), "test ? test !"); + + // Test it again with lowercased "openpgp4fpr:" uri scheme let res = check_qr( &ctx.ctx, "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL" @@ -548,7 +550,7 @@ mod tests { assert_eq!(res.get_state(), LotState::QrAskVerifyContact); assert_ne!(res.get_id(), 0); - // Test it again with lowercase + // Test it again with lowercased "openpgp4fpr:" uri scheme let res = check_qr( &ctx.ctx, "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB" @@ -578,7 +580,7 @@ mod tests { ); assert_eq!(res.get_id(), 0); - // Test it again with lowercased openpgp4fpr uri scheme + // Test it again with lowercased "openpgp4fpr:" uri scheme let res = check_qr( &ctx.ctx, @@ -606,6 +608,14 @@ mod tests { ); assert_eq!(res.get_state(), LotState::QrAccount); assert_eq!(res.get_text1().unwrap(), "example.org"); + + // Test it again with lowercased "dcaccount:" uri scheme + let res = check_qr( + &ctx.ctx, + "dcaccount:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3", + ); + assert_eq!(res.get_state(), LotState::QrAccount); + assert_eq!(res.get_text1().unwrap(), "example.org"); } #[test] @@ -618,5 +628,13 @@ mod tests { ); assert_eq!(res.get_state(), LotState::QrError); assert!(res.get_text1().is_some()); + + // Test it again with lowercased "dcaccount:" uri scheme + let res = check_qr( + &ctx.ctx, + "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3", + ); + assert_eq!(res.get_state(), LotState::QrError); + assert!(res.get_text1().is_some()); } } From 1b485770b6d7b92605c208ddc1bb593771104e3f Mon Sep 17 00:00:00 2001 From: jikstra Date: Tue, 28 Apr 2020 17:11:46 +0200 Subject: [PATCH 120/156] cargo fmt --- src/qr.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/qr.rs b/src/qr.rs index 8e97c544b..9f9ac77d0 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -521,7 +521,6 @@ mod tests { assert_ne!(res.get_id(), 0); assert_eq!(res.get_text1().unwrap(), "test ? test !"); - // Test it again with lowercased "openpgp4fpr:" uri scheme let res = check_qr( &ctx.ctx, @@ -628,7 +627,7 @@ mod tests { ); assert_eq!(res.get_state(), LotState::QrError); assert!(res.get_text1().is_some()); - + // Test it again with lowercased "dcaccount:" uri scheme let res = check_qr( &ctx.ctx, From 551f7dc05a118c5cfc1ca9f58cd82afcd218aac2 Mon Sep 17 00:00:00 2001 From: jikstra Date: Tue, 28 Apr 2020 21:22:48 +0200 Subject: [PATCH 121/156] Make starts_with_ignore_case() private --- src/qr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qr.rs b/src/qr.rs index 9f9ac77d0..6c4f21ce5 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -37,7 +37,7 @@ impl Into for Error { } } -pub fn starts_with_ignore_case(string: &str, pattern: &str) -> bool { +fn starts_with_ignore_case(string: &str, pattern: &str) -> bool { string.to_lowercase().starts_with(&pattern.to_lowercase()) } From 0d0e7f774ea8231f5b0b3edf5925e5d42949ddfb Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Tue, 28 Apr 2020 19:09:49 +0200 Subject: [PATCH 122/156] update readme --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b06133e37..b92268884 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,40 @@ # Changelog -## UNRELEASED +## 1.29.0 -- removed api: dc_chat_get_subtitle(), dc_get_version_str(), dc_array_add_id() +- new config options `delete_device_after` and `delete_server_after`, + each taking an amount of seconds after which messages + are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423 + +- new api `dc_estimate_deletion_cnt()` to estimate the effect + of `delete_device_after` and `delete_server_after` + +- use Ed25519 keys by default, these keys are much shorter + than RSA keys, which results in saving traffic and speed improvements #1362 + +- improve message ellipsizing #1397 #1430 + +- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378 + +- do not show badly formatted non-delta-messages as empty #1384 + +- try over SMTP on potentially recoverable error 5.5.0 #1379 + +- remove device-chat from forward-to-chat-list #1367 + +- improve group-handling #1368 + +- `dc_get_info()` returns uptime (how long the context is in use) + +- python improvements and adaptions #1408 #1415 + +- log to the stdout and stderr in tests #1416 + +- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419 + +- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()` + +- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED` ## 1.28.0 From f444af825fe907c599aaafb427882e261df94a26 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Wed, 29 Apr 2020 14:49:40 +0200 Subject: [PATCH 123/156] bump version to 1.29.0 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80c9b4d9f..537beee8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,7 +629,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.28.0" +version = "1.29.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -695,10 +695,10 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.28.0" +version = "1.29.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", - "deltachat 1.28.0", + "deltachat 1.29.0", "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 7d85672b6..583fb0055 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.28.0" +version = "1.29.0" authors = ["Delta Chat Developers (ML) "] edition = "2018" license = "MPL-2.0" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 593dd9c26..a9a723465 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.28.0" +version = "1.29.0" description = "Deltachat FFI" authors = ["Delta Chat Developers (ML) "] edition = "2018" From 54395a72521f2fd3e5a7105a2c3baeb75c749a6f Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Thu, 30 Apr 2020 16:56:43 +0200 Subject: [PATCH 124/156] do not send DC_EVENT_MSGS_CHANGED or DC_EVENT_INCOMING_MSG for hidden messages these events take the message-id as parameter and might be used to update an existing list (although to recommended) if the event is issued for hidden messages, this might led to "empty" messages flashing up - the ui tries to get the message from the event, after a moment, on the next update, the message disappears again as hidden messages are of course not returned eg. by dc_get_chat_msgs(). the effect was probably always visible for secure-join-messages on ios, however, become much more visible recently when read-receipts are added as hidden messages as well (to make them auto-deletable). --- src/dc_receive_imf.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index e3d589360..4730d540a 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -674,7 +674,7 @@ fn add_parts( ); // check event to send - if chat_id.is_trash() { + if chat_id.is_trash() || *hidden { *create_event_to_send = None; } else if incoming && state == MessageState::InFresh { if from_id_blocked { From c41a6b87b8d7816a2da14a3a8634b8afb77b642d Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Thu, 30 Apr 2020 22:25:35 +0300 Subject: [PATCH 125/156] imap: always close folder before selecting if expunge is needed --- src/imap/select_folder.rs | 65 ++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index c794611fd..4d454c332 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -23,6 +23,34 @@ pub enum Error { } impl Imap { + /// Issues a CLOSE command to expunge selected folder. + /// + /// CLOSE is considerably faster than an EXPUNGE, see + /// https://tools.ietf.org/html/rfc3501#section-6.4.2 + async fn close_folder(&self, context: &Context) -> Result<()> { + if let Some(ref folder) = self.config.read().await.selected_folder { + info!(context, "Expunge messages in \"{}\".", folder); + + if let Some(ref mut session) = &mut *self.session.lock().await { + match session.close().await { + Ok(_) => { + info!(context, "close/expunge succeeded"); + } + Err(err) => { + self.trigger_reconnect(); + return Err(Error::CloseExpungeFailed(err)); + } + } + } else { + return Err(Error::NoSession); + } + } + let mut cfg = self.config.write().await; + cfg.selected_folder = None; + cfg.selected_folder_needs_expunge = false; + Ok(()) + } + /// select a folder, possibly update uid_validity and, if needed, /// expunge the folder to remove delete-marked messages. pub(super) async fn select_folder>( @@ -38,39 +66,14 @@ impl Imap { return Err(Error::NoSession); } - // if there is a new folder and the new folder is equal to the selected one, there's nothing to do. - // if there is _no_ new folder, we continue as we might want to expunge below. - if let Some(ref folder) = folder { - if let Some(ref selected_folder) = self.config.read().await.selected_folder { - if folder.as_ref() == selected_folder { - return Ok(()); - } - } + let needs_expunge = self.config.read().await.selected_folder_needs_expunge; + if needs_expunge { + self.close_folder(context).await?; } - // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then) - let needs_expunge = { self.config.read().await.selected_folder_needs_expunge }; - if needs_expunge { - if let Some(ref folder) = self.config.read().await.selected_folder { - info!(context, "Expunge messages in \"{}\".", folder); - - // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see - // https://tools.ietf.org/html/rfc3501#section-6.4.2 - if let Some(ref mut session) = &mut *self.session.lock().await { - match session.close().await { - Ok(_) => { - info!(context, "close/expunge succeeded"); - } - Err(err) => { - self.trigger_reconnect(); - return Err(Error::CloseExpungeFailed(err)); - } - } - } else { - return Err(Error::NoSession); - } - } - self.config.write().await.selected_folder_needs_expunge = false; + let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref()); + if self.config.read().await.selected_folder.as_deref() == folder_str { + return Ok(()); } // select new folder From e8763e936df6a2ed695af4f6c2acf75e361d7fdd Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Thu, 30 Apr 2020 22:33:55 +0300 Subject: [PATCH 126/156] imap: simplify select_folder() interface Accept AsRef instead of Option>. There is no need to pass None to force expunge anymore. --- src/imap/idle.rs | 4 +-- src/imap/mod.rs | 8 +++--- src/imap/select_folder.rs | 59 ++++++++++++++++++--------------------- src/job_thread.rs | 17 +++++------ 4 files changed, 42 insertions(+), 46 deletions(-) diff --git a/src/imap/idle.rs b/src/imap/idle.rs index 30e27b9ee..d437e1619 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -59,7 +59,7 @@ impl Imap { task::block_on(async move { self.config.read().await.can_idle }) } - pub fn idle(&self, context: &Context, watch_folder: Option) -> Result<()> { + pub fn idle(&self, context: &Context, watch_folder: String) -> Result<()> { task::block_on(async move { if !self.can_idle() { return Err(Error::IdleAbilityMissing); @@ -67,7 +67,7 @@ impl Imap { self.setup_handle_if_needed(context).await?; - self.select_folder(context, watch_folder.clone()).await?; + self.select_folder(context, watch_folder).await?; let session = self.session.lock().await.take(); let timeout = Duration::from_secs(23 * 60); diff --git a/src/imap/mod.rs b/src/imap/mod.rs index b86b8c39e..204a0157c 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -476,7 +476,7 @@ impl Imap { folder: &str, ) -> Result<(u32, u32)> { task::block_on(async move { - self.select_folder(context, Some(folder)).await?; + self.select_folder(context, folder).await?; // compare last seen UIDVALIDITY against the current one let (uid_validity, last_seen_uid) = self.get_config_last_seen_uid(context, &folder); @@ -912,7 +912,7 @@ impl Imap { return Some(ImapActionResult::RetryLater); } } - match self.select_folder(context, Some(&folder)).await { + match self.select_folder(context, &folder).await { Ok(()) => None, Err(select_folder::Error::ConnectionLost) => { warn!(context, "Lost imap connection"); @@ -1183,7 +1183,7 @@ impl Imap { error!(context, "could not setup imap connection: {}", err); return; } - if let Err(err) = self.select_folder(context, Some(&folder)).await { + if let Err(err) = self.select_folder(context, &folder).await { error!( context, "Could not select {} for expunging: {}", folder, err @@ -1201,7 +1201,7 @@ impl Imap { // we now trigger expunge to actually delete messages self.config.write().await.selected_folder_needs_expunge = true; - match self.select_folder::(context, None).await { + match self.select_folder(context, &folder).await { Ok(()) => { emit_event!(context, Event::ImapFolderEmptied(folder.to_string())); } diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index 4d454c332..e0fce0dff 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -56,7 +56,7 @@ impl Imap { pub(super) async fn select_folder>( &self, context: &Context, - folder: Option, + folder: S, ) -> Result<()> { if self.session.lock().await.is_none() { let mut cfg = self.config.write().await; @@ -71,46 +71,41 @@ impl Imap { self.close_folder(context).await?; } - let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref()); - if self.config.read().await.selected_folder.as_deref() == folder_str { + if self.config.read().await.selected_folder.as_deref() == Some(folder.as_ref()) { return Ok(()); } // select new folder - if let Some(ref folder) = folder { - if let Some(ref mut session) = &mut *self.session.lock().await { - let res = session.select(folder).await; + if let Some(ref mut session) = &mut *self.session.lock().await { + let res = session.select(&folder).await; - // https://tools.ietf.org/html/rfc3501#section-6.3.1 - // says that if the server reports select failure we are in - // authenticated (not-select) state. + // https://tools.ietf.org/html/rfc3501#section-6.3.1 + // says that if the server reports select failure we are in + // authenticated (not-select) state. - match res { - Ok(mailbox) => { - let mut config = self.config.write().await; - config.selected_folder = Some(folder.as_ref().to_string()); - config.selected_mailbox = Some(mailbox); - Ok(()) - } - Err(async_imap::error::Error::ConnectionLost) => { - self.trigger_reconnect(); - self.config.write().await.selected_folder = None; - Err(Error::ConnectionLost) - } - Err(async_imap::error::Error::Validate(_)) => { - Err(Error::BadFolderName(folder.as_ref().to_string())) - } - Err(err) => { - self.config.write().await.selected_folder = None; - self.trigger_reconnect(); - Err(Error::Other(err.to_string())) - } + match res { + Ok(mailbox) => { + let mut config = self.config.write().await; + config.selected_folder = Some(folder.as_ref().to_string()); + config.selected_mailbox = Some(mailbox); + Ok(()) + } + Err(async_imap::error::Error::ConnectionLost) => { + self.trigger_reconnect(); + self.config.write().await.selected_folder = None; + Err(Error::ConnectionLost) + } + Err(async_imap::error::Error::Validate(_)) => { + Err(Error::BadFolderName(folder.as_ref().to_string())) + } + Err(err) => { + self.config.write().await.selected_folder = None; + self.trigger_reconnect(); + Err(Error::Other(err.to_string())) } - } else { - Err(Error::NoSession) } } else { - Ok(()) + Err(Error::NoSession) } } } diff --git a/src/job_thread.rs b/src/job_thread.rs index 12512a3d6..0e524e161 100644 --- a/src/job_thread.rs +++ b/src/job_thread.rs @@ -173,14 +173,15 @@ impl JobThread { if !self.imap.can_idle() { true // we have to do fake_idle } else { - let watch_folder = self.get_watch_folder(context); - info!(context, "{} started...", prefix); - let res = self.imap.idle(context, watch_folder); - info!(context, "{} ended...", prefix); - if let Err(err) = res { - warn!(context, "{} failed: {} -> reconnecting", prefix, err); - // something is borked, let's start afresh on the next occassion - self.imap.disconnect(context); + if let Some(watch_folder) = self.get_watch_folder(context) { + info!(context, "{} started...", prefix); + let res = self.imap.idle(context, watch_folder); + info!(context, "{} ended...", prefix); + if let Err(err) = res { + warn!(context, "{} failed: {} -> reconnecting", prefix, err); + // something is Label { Label }orked, let's start afresh on the next occassion + self.imap.disconnect(context); + } } false } From 96436814f55d3b5d77f562749e1e136bb85f5c24 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 2 May 2020 17:16:14 +0200 Subject: [PATCH 127/156] changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b92268884..c0a33eee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 1.30.0 + +- expunge deleted messages #1440 + +- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439 + + ## 1.29.0 - new config options `delete_device_after` and `delete_server_after`, From bb59cf94e9dbdacab749fb2fda84afe6e263db99 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sat, 2 May 2020 17:17:14 +0200 Subject: [PATCH 128/156] bump version to 1.30.0 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 537beee8e..700d553e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,7 +629,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.29.0" +version = "1.30.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -695,10 +695,10 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.29.0" +version = "1.30.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", - "deltachat 1.29.0", + "deltachat 1.30.0", "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 583fb0055..811ae9359 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.29.0" +version = "1.30.0" authors = ["Delta Chat Developers (ML) "] edition = "2018" license = "MPL-2.0" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index a9a723465..5bf9607ee 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.29.0" +version = "1.30.0" description = "Deltachat FFI" authors = ["Delta Chat Developers (ML) "] edition = "2018" From 4efcbee7721c1cf9f2c20f7a75dd4fee5abf5926 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 3 May 2020 13:38:39 +0200 Subject: [PATCH 129/156] support dc_get|set_config("media_quality") --- deltachat-ffi/deltachat.h | 16 ++++++++++++++++ src/config.rs | 22 ++++++++++++++++++++++ src/constants.rs | 13 +++++++++++++ 3 files changed, 51 insertions(+) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 10154b71a..5bc9f3112 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -382,6 +382,14 @@ char* dc_get_blobdir (const dc_context_t* context); * >=1=seconds, after which messages are deleted automatically from the server. * "Saved messages" are deleted from the server as well as * emails matching the `show_emails` settings above, the UI should clearly point that out. + * - `media_quality` = DC_MEDIA_QUALITY_BALANCED (0) = + * good outgoing images/videos/voice quality at reasonable sizes (default) + * DC_MEDIA_QUALITY_WORSE (1) + * allow worse images/videos/voice quality to gain smaller sizes, + * suitable for providers or areas known to have a bad connection. + * In contrast to other options, the implementation of this option is currently up to the UIs; + * this may change in future, however, + * having the option in the core allows provider-specific-defaults already today. * * If you want to retrieve a value, use dc_get_config(). * @@ -4492,6 +4500,14 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot); #define DC_SHOW_EMAILS_ACCEPTED_CONTACTS 1 #define DC_SHOW_EMAILS_ALL 2 + +/* + * Values for dc_get|set_config("media_quality") + */ +#define DC_MEDIA_QUALITY_BALANCED 0 +#define DC_MEDIA_QUALITY_WORSE 1 + + /* * Values for dc_get|set_config("key_gen_type") */ diff --git a/src/config.rs b/src/config.rs index 203bdab0a..7abf10b2f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -65,6 +65,9 @@ pub enum Config { #[strum(props(default = "0"))] // also change ShowEmails.default() on changes ShowEmails, + #[strum(props(default = "0"))] // also change MediaQuality.default() on changes + MediaQuality, + #[strum(props(default = "0"))] KeyGenType, @@ -248,9 +251,11 @@ mod tests { use std::str::FromStr; use std::string::ToString; + use crate::constants; use crate::constants::AVATAR_SIZE; use crate::test_utils::*; use image::GenericImageView; + use num_traits::FromPrimitive; use std::fs::File; use std::io::Write; @@ -346,4 +351,21 @@ mod tests { let avatar_cfg = t.ctx.get_config(Config::Selfavatar); assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string())); } + + #[test] + fn test_media_quality_config_option() { + let t = dummy_context(); + let media_quality = t.ctx.get_config_int(Config::MediaQuality); + assert_eq!(media_quality, 0); + let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default(); + assert_eq!(media_quality, constants::MediaQuality::Balanced); + + t.ctx.set_config(Config::MediaQuality, Some("1")).unwrap(); + + let media_quality = t.ctx.get_config_int(Config::MediaQuality); + assert_eq!(media_quality, 1); + assert_eq!(constants::MediaQuality::Worse as i32, 1); + let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default(); + assert_eq!(media_quality, constants::MediaQuality::Worse); + } } diff --git a/src/constants.rs b/src/constants.rs index 64ce2cd59..dbdad54db 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -57,6 +57,19 @@ impl Default for ShowEmails { } } +#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)] +#[repr(u8)] +pub enum MediaQuality { + Balanced = 0, + Worse = 1, +} + +impl Default for MediaQuality { + fn default() -> Self { + MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes + } +} + #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)] #[repr(u8)] pub enum KeyGenType { From 3035c8af30194f32bcafed065eb73fd8850acd20 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Mon, 4 May 2020 16:35:42 +0300 Subject: [PATCH 130/156] Always describe the context of the displayed error --- src/job.rs | 2 +- src/message.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/job.rs b/src/job.rs index 7e101b9f6..32eea25fd 100644 --- a/src/job.rs +++ b/src/job.rs @@ -1190,7 +1190,7 @@ fn perform_job_action(context: &Context, mut job: &mut Job, thread: Thread, trie Action::ImexImap => match JobImexImap(context, &job) { Ok(()) => Status::Finished(Ok(())), Err(err) => { - error!(context, "{}", err); + error!(context, "Import/export failed: {}", err); Status::Finished(Err(err)) } }, diff --git a/src/message.rs b/src/message.rs index 732ad4c26..8668adef4 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1215,7 +1215,7 @@ pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option Date: Sun, 3 May 2020 05:08:22 +0300 Subject: [PATCH 131/156] fetch_single_msg: do not ignore dc_receive_imf errors If error is ignored, the message will never be fetched again, even if there was a database write error. dc_receive_imf itself is modified to ignore unrecoverable errors, to prevent endless refetching of incorrect messages. --- src/dc_receive_imf.rs | 26 +++++++++++++++++++++----- src/imap/mod.rs | 8 +------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 4730d540a..de0246103 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -32,6 +32,10 @@ enum CreateEvent { } /// Receive a message and add it to the database. +/// +/// Returns an error on recoverable errors, e.g. database errors. In this case, +/// message parsing should be retried later. If message itself is wrong, logs +/// the error and returns success. pub fn dc_receive_imf( context: &Context, imf_raw: &[u8], @@ -55,10 +59,19 @@ pub fn dc_receive_imf( println!("{}", String::from_utf8_lossy(imf_raw)); } - let mut mime_parser = MimeMessage::from_bytes(context, imf_raw)?; + let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw) { + Err(err) => { + warn!(context, "dc_receive_imf: can't parse MIME: {}", err); + return Ok(()); + } + Ok(mime_parser) => mime_parser, + }; // we can not add even an empty record if we have no info whatsoever - ensure!(mime_parser.has_headers(), "No Headers Found"); + if !mime_parser.has_headers() { + warn!(context, "dc_receive_imf: no headers found"); + return Ok(()); + } // the function returns the number of created messages in the database let mut chat_id = ChatId::new(0); @@ -305,7 +318,8 @@ fn add_parts( message::update_server_uid(context, &rfc724_mid, server_folder.as_ref(), server_uid); } - bail!("Message already in DB"); + warn!(context, "Message already in DB"); + return Ok(()); } let mut msgrmsg = if mime_parser.has_chat_version() { @@ -369,7 +383,8 @@ fn add_parts( *hidden = true; context.bob.write().unwrap().status = 0; // secure-join failed context.stop_ongoing(); - error!(context, "Error in Secure-Join message handling: {}", err); + warn!(context, "Error in Secure-Join message handling: {}", err); + return Ok(()); } } } @@ -496,7 +511,8 @@ fn add_parts( } Err(err) => { *hidden = true; - error!(context, "Error in Secure-Join watching: {}", err); + warn!(context, "Error in Secure-Join watching: {}", err); + return Ok(()); } } } diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 204a0157c..dbfe3ff91 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -733,13 +733,7 @@ impl Imap { if let Err(err) = dc_receive_imf(context, &body, folder.as_ref(), server_uid, is_seen) { - warn!( - context, - "dc_receive_imf failed for imap-message {}/{}: {:?}", - folder.as_ref(), - server_uid, - err - ); + return Err(Error::Other(format!("dc_receive_imf error: {}", err))); } } } else { From 076cdae3fd584eebb8fa003b873013fccaff7993 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Tue, 5 May 2020 21:44:03 +0200 Subject: [PATCH 132/156] do not show errors during sending as a ephemeral popup or so, just set the message-state to failed, the error can be queried by the user at any time via 'Info' or so --- src/message.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index 8668adef4..d96833ee7 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1215,7 +1215,7 @@ pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option Date: Tue, 5 May 2020 19:18:45 +0300 Subject: [PATCH 133/156] Parse attachment filenames from Content-Type "name" attribute Outlook specifies filename there and omits Content-Disposition. --- src/mimeparser.rs | 77 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index cc3c4198b..684e295da 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -984,6 +984,11 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result +To: Anonymous +Subject: Delta Chat is great stuff! +Date: Tue, 5 May 2020 01:23:45 +0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + boundary="----=_NextPart_000_0003_01D622B3.CA753E60" +X-Mailer: Microsoft Outlook 15.0 + +This is a multipart message in MIME format. + +------=_NextPart_000_0003_01D622B3.CA753E60 +Content-Type: multipart/alternative; + boundary="----=_NextPart_001_0004_01D622B3.CA753E60" + + +------=_NextPart_001_0004_01D622B3.CA753E60 +Content-Type: text/plain; + charset="us-ascii" +Content-Transfer-Encoding: 7bit + + + + +------=_NextPart_001_0004_01D622B3.CA753E60 +Content-Type: text/html; + charset="us-ascii" +Content-Transfer-Encoding: quoted-printable + + + +

+Test +

+ + +------=_NextPart_001_0004_01D622B3.CA753E60-- + +------=_NextPart_000_0003_01D622B3.CA753E60 +Content-Type: image/jpeg; + name="image001.jpg" +Content-Transfer-Encoding: base64 +Content-ID: + +ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG +acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT +/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal +n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C +xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2 +rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU +8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI +K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4 +CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= + +------=_NextPart_000_0003_01D622B3.CA753E60-- +"##; + + let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap(); + assert_eq!( + message.get_subject(), + Some("Delta Chat is great stuff!".to_string()) + ); + + assert_eq!(message.parts.len(), 1); + assert_eq!(message.parts[0].typ, Viewtype::Image); + assert_eq!(message.parts[0].msg, "Test"); + } } From d78f75aa600e2d2ca891dbc2e5cd8f4f1c696200 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Tue, 5 May 2020 23:52:53 +0200 Subject: [PATCH 134/156] changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0a33eee1..62a463915 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 1.31.0 + +- always describe the context of the displayed error #1451 + +- do not emit `DC_EVENT_ERROR` when message sending fails; + `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451 + +- new config-option `media_quality` #1449 + +- try over if writing message to database fails #1447 + + ## 1.30.0 - expunge deleted messages #1440 From fcf3786fc5e74c0cf348a5864014300d0993db92 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Tue, 5 May 2020 23:53:54 +0200 Subject: [PATCH 135/156] bump version to 1.31.0 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 700d553e8..6b69b5987 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,7 +629,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.30.0" +version = "1.31.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -695,10 +695,10 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.30.0" +version = "1.31.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", - "deltachat 1.30.0", + "deltachat 1.31.0", "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 811ae9359..46a356eb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.30.0" +version = "1.31.0" authors = ["Delta Chat Developers (ML) "] edition = "2018" license = "MPL-2.0" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 5bf9607ee..c604906a8 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.30.0" +version = "1.31.0" description = "Deltachat FFI" authors = ["Delta Chat Developers (ML) "] edition = "2018" From 4724101e75b130722f4db71eac573c300811620f Mon Sep 17 00:00:00 2001 From: holger krekel Date: Wed, 6 May 2020 17:31:43 +0200 Subject: [PATCH 136/156] fix upload error?! (#1454) * use latest setuptools * clear indexes also if nothing was uploaded to dc/* branches --- .circleci/config.yml | 4 ++++ ci_scripts/ci_upload.sh | 2 +- ci_scripts/cleanup_devpi_indices.py | 9 ++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c40632fb..cd3d70517 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,6 +7,10 @@ executors: doxygen: docker: - image: hrektts/doxygen + python: + docker: + - image: 3.7.7-stretch + restore-workspace: &restore-workspace attach_workspace: diff --git a/ci_scripts/ci_upload.sh b/ci_scripts/ci_upload.sh index bb610aa53..e357b9617 100755 --- a/ci_scripts/ci_upload.sh +++ b/ci_scripts/ci_upload.sh @@ -37,7 +37,7 @@ echo ----------------------- # Bundle external shared libraries into the wheels pushd $WHEELHOUSEDIR -pip3 install -U pip +pip3 install -U pip setuptools pip3 install devpi-client devpi use https://m.devpi.net devpi login dc --password $DEVPI_LOGIN diff --git a/ci_scripts/cleanup_devpi_indices.py b/ci_scripts/cleanup_devpi_indices.py index 628c47d84..8e1eaa9ee 100644 --- a/ci_scripts/cleanup_devpi_indices.py +++ b/ci_scripts/cleanup_devpi_indices.py @@ -48,7 +48,7 @@ def run(): projectnames = get_projectnames(baseurl, username, indexname) if indexname == "master" or not indexname: continue - assert projectnames == ["deltachat"] + clear_index = not projectnames for projectname in projectnames: dates = get_release_dates(baseurl, username, indexname, projectname) if not dates: @@ -60,8 +60,11 @@ def run(): date = datetime.datetime(*max(dates)) if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS): assert username and indexname - url = baseurl + username + "/" + indexname - subprocess.check_call(["devpi", "index", "-y", "--delete", url]) + clear_index = True + break + if clear_index: + url = baseurl + username + "/" + indexname + subprocess.check_call(["devpi", "index", "-y", "--delete", url]) From a586a1d525c2082b557e2d114b8acc46608ad906 Mon Sep 17 00:00:00 2001 From: Hocuri <18012815+Hocuri@users.noreply.github.com> Date: Thu, 7 May 2020 13:55:09 +0200 Subject: [PATCH 137/156] Fix #1120 Contact requests are not shown when name of sender includes a comma character (#1438) * First try making get_recipients use MailHeader (nice and functional) * Get it to compile by using not-so-functional style * Add "empty-from" test, drop unnecessary check for error; continue using addrparse_header() instead of addrparse() * Try to use functional style, unfortunately, I can't get the compiler to accept it * Do it imperative-style: Do not overwrite To with Cc and vice versa * Use addrparse_header() once more * Still addrparse_header() * Clippy * Fix compile errors in tests * Fix typo * Fix tests again ;-) * Code style * Code style; try a HashMap as an address list but I am not convinced * Code style; Use Vec as address list * Clippy * Add tests * Add another test * Remove stale comments --- src/dc_receive_imf.rs | 231 ++++++++++++++++++++++++++++++++---------- src/e2ee.rs | 7 +- src/headerdef.rs | 4 + src/imap/mod.rs | 8 +- src/mimeparser.rs | 192 +++++++++++++++++++---------------- 5 files changed, 295 insertions(+), 147 deletions(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index de0246103..8a10e0ecd 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -3,6 +3,8 @@ use sha2::{Digest, Sha256}; use num_traits::FromPrimitive; +use mailparse::SingleInfo; + use crate::chat::{self, Chat, ChatId}; use crate::config::Config; use crate::constants::*; @@ -110,29 +112,23 @@ pub fn dc_receive_imf( // we do not check Return-Path any more as this is unreliable, see // https://github.com/deltachat/deltachat-core/issues/150) let (from_id, from_id_blocked, incoming_origin) = - if let Some(field_from) = mime_parser.get(HeaderDef::From_) { - from_field_to_contact_id(context, field_from)? - } else { - (0, false, Origin::Unknown) - }; + from_field_to_contact_id(context, &mime_parser.from)?; + let incoming = from_id != DC_CONTACT_ID_SELF; let mut to_ids = ContactIds::new(); - for header_def in &[HeaderDef::To, HeaderDef::Cc] { - if let Some(field) = mime_parser.get(header_def.clone()) { - to_ids.extend(&dc_add_or_lookup_contacts_by_address_list( - context, - &field, - if !incoming { - Origin::OutgoingTo - } else if incoming_origin.is_known() { - Origin::IncomingTo - } else { - Origin::IncomingUnknownTo - }, - )?); - } - } + + to_ids.extend(&dc_add_or_lookup_contacts_by_address_list( + context, + &mime_parser.recipients, + if !incoming { + Origin::OutgoingTo + } else if incoming_origin.is_known() { + Origin::IncomingTo + } else { + Origin::IncomingUnknownTo + }, + )?); // Add parts @@ -242,11 +238,11 @@ pub fn dc_receive_imf( /// Also returns whether it is blocked or not and its origin. pub fn from_field_to_contact_id( context: &Context, - field_from: &str, + from_address_list: &[SingleInfo], ) -> Result<(u32, bool, Origin)> { let from_ids = dc_add_or_lookup_contacts_by_address_list( context, - &field_from, + from_address_list, Origin::IncomingUnknownFrom, )?; @@ -256,7 +252,7 @@ pub fn from_field_to_contact_id( if from_ids.len() > 1 { warn!( context, - "mail has more than one From address, only using first: {:?}", field_from + "mail has more than one From address, only using first: {:?}", from_address_list ); } let from_id = from_ids.get_index(0).cloned().unwrap_or_default(); @@ -269,7 +265,10 @@ pub fn from_field_to_contact_id( } Ok((from_id, from_id_blocked, incoming_origin)) } else { - warn!(context, "mail has an empty From header: {:?}", field_from); + warn!( + context, + "mail has an empty From header: {:?}", from_address_list + ); // if there is no from given, from_id stays 0 which is just fine. These messages // are very rare, however, we have to add them to the database (they go to the // "deaddrop" chat) to avoid a re-download from the server. See also [**] @@ -1593,38 +1592,17 @@ fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool { fn dc_add_or_lookup_contacts_by_address_list( context: &Context, - addr_list_raw: &str, + address_list: &[SingleInfo], origin: Origin, ) -> Result { - let addrs = match mailparse::addrparse(addr_list_raw) { - Ok(addrs) => addrs, - Err(err) => { - bail!("could not parse {:?}: {:?}", addr_list_raw, err); - } - }; - let mut contact_ids = ContactIds::new(); - for addr in addrs.iter() { - match addr { - mailparse::MailAddr::Single(info) => { - contact_ids.insert(add_or_lookup_contact_by_addr( - context, - &info.display_name, - &info.addr, - origin, - )?); - } - mailparse::MailAddr::Group(infos) => { - for info in &infos.addrs { - contact_ids.insert(add_or_lookup_contact_by_addr( - context, - &info.display_name, - &info.addr, - origin, - )?); - } - } - } + for info in address_list.iter() { + contact_ids.insert(add_or_lookup_contact_by_addr( + context, + &info.display_name, + &info.addr, + origin, + )?); } Ok(contact_ids) @@ -2017,4 +1995,151 @@ mod tests { let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap(); assert!(one2one.get_visibility() == ChatVisibility::Archived); } + + #[test] + fn test_no_from() { + // if there is no from given, from_id stays 0 which is just fine. These messages + // are very rare, however, we have to add them to the database (they go to the + // "deaddrop" chat) to avoid a re-download from the server. See also [**] + + let t = configured_offline_context(); + let context = &t.ctx; + + let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap(); + assert!(chats.get_msg_id(0).is_err()); + + dc_receive_imf( + context, + b"To: bob@example.org\n\ + Subject: foo\n\ + Message-ID: <3924@example.org>\n\ + Chat-Version: 1.0\n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ + \n\ + hello\n", + "INBOX", + 1, + false, + ) + .unwrap(); + + let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap(); + // Check that the message was added to the database: + assert!(chats.get_msg_id(0).is_ok()); + } + + #[test] + fn test_escaped_from() { + let t = configured_offline_context(); + let contact_id = Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap(); + let chat_id = chat::create_by_contact_id(&t.ctx, contact_id).unwrap(); + dc_receive_imf( + &t.ctx, + b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= \n\ + To: alice@example.org\n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Chat-Disposition-Notification-To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= \n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ + \n\ + hello\n", + "INBOX", + 1, + false, + ).unwrap(); + assert_eq!( + Contact::load_from_db(&t.ctx, contact_id) + .unwrap() + .get_authname(), + "Фамилия Имя", // The name was "Имя, Фамилия" and ("lastname, firstname") and should be swapped to "firstname, lastname" + ); + let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None); + assert_eq!(msgs.len(), 1); + let msg_id = msgs.first().unwrap(); + let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap(); + assert_eq!(msg.is_dc_message, MessengerMessage::Yes); + assert_eq!(msg.text.unwrap(), "hello"); + assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1); + } + + #[test] + fn test_escaped_recipients() { + let t = configured_offline_context(); + Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap(); + + let carl_contact_id = + Contact::add_or_lookup(&t.ctx, "Carl", "carl@host.tld", Origin::IncomingUnknownFrom) + .unwrap() + .0; + + dc_receive_imf( + &t.ctx, + b"From: Foobar \n\ + To: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= alice@example.org\n\ + Cc: =?utf-8?q?=3Ch2=3E?= \n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Chat-Disposition-Notification-To: \n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ + \n\ + hello\n", + "INBOX", + 1, + false, + ) + .unwrap(); + assert_eq!( + Contact::load_from_db(&t.ctx, carl_contact_id) + .unwrap() + .get_name(), + "h2" + ); + + let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap(); + let msg = Message::load_from_db(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap(); + assert_eq!(msg.is_dc_message, MessengerMessage::Yes); + assert_eq!(msg.text.unwrap(), "hello"); + assert_eq!(msg.param.get_int(Param::WantsMdn).unwrap(), 1); + } + + #[test] + fn test_cc_to_contact() { + let t = configured_offline_context(); + Contact::create(&t.ctx, "foobar", "foobar@example.com").unwrap(); + + let carl_contact_id = Contact::add_or_lookup( + &t.ctx, + "garabage", + "carl@host.tld", + Origin::IncomingUnknownFrom, + ) + .unwrap() + .0; + + dc_receive_imf( + &t.ctx, + b"From: Foobar \n\ + To: alice@example.org\n\ + Cc: Carl \n\ + Subject: foo\n\ + Message-ID: \n\ + Chat-Version: 1.0\n\ + Chat-Disposition-Notification-To: \n\ + Date: Sun, 22 Mar 2020 22:37:57 +0000\n\ + \n\ + hello\n", + "INBOX", + 1, + false, + ) + .unwrap(); + assert_eq!( + Contact::load_from_db(&t.ctx, carl_contact_id) + .unwrap() + .get_name(), + "Carl" + ); + } } diff --git a/src/e2ee.rs b/src/e2ee.rs index b3e650d25..60ec042da 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -9,7 +9,8 @@ use crate::aheader::*; use crate::config::Config; use crate::context::Context; use crate::error::*; -use crate::headerdef::{HeaderDef, HeaderDefMap}; +use crate::headerdef::HeaderDef; +use crate::headerdef::HeaderDefMap; use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey}; use crate::keyring::*; use crate::peerstate::*; @@ -122,8 +123,8 @@ pub fn try_decrypt( ) -> Result<(Option>, HashSet)> { let from = mail .headers - .get_header_value(HeaderDef::From_) - .and_then(|from_addr| mailparse::addrparse(&from_addr).ok()) + .get_header(HeaderDef::From_) + .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok()) .and_then(|from| from.extract_single_info()) .map(|from| from.addr) .unwrap_or_default(); diff --git a/src/headerdef.rs b/src/headerdef.rs index af8343949..10beaf4c8 100644 --- a/src/headerdef.rs +++ b/src/headerdef.rs @@ -53,12 +53,16 @@ impl HeaderDef { pub trait HeaderDefMap { fn get_header_value(&self, headerdef: HeaderDef) -> Option; + fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>; } impl HeaderDefMap for [MailHeader<'_>] { fn get_header_value(&self, headerdef: HeaderDef) -> Option { self.get_first_value(headerdef.get_headername()) } + fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> { + self.get_first_header(headerdef.get_headername()) + } } #[cfg(test)] diff --git a/src/imap/mod.rs b/src/imap/mod.rs index dbfe3ff91..3c90f235a 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -25,6 +25,7 @@ use crate::headerdef::{HeaderDef, HeaderDefMap}; use crate::job::{job_add, Action}; use crate::login_param::{CertificateChecks, LoginParam}; use crate::message::{self, update_server_uid}; +use crate::mimeparser; use crate::oauth2::dc_get_oauth2_access_token; use crate::param::Params; use crate::stock::StockMessage; @@ -1373,11 +1374,8 @@ fn prefetch_should_download( .get_header_value(HeaderDef::AutocryptSetupMessage) .is_some(); - let from_field = headers - .get_header_value(HeaderDef::From_) - .unwrap_or_default(); - - let (_contact_id, blocked_contact, origin) = from_field_to_contact_id(context, &from_field)?; + let (_contact_id, blocked_contact, origin) = + from_field_to_contact_id(context, &mimeparser::get_from(headers))?; let accepted_contact = origin.is_known(); let show = is_autocrypt_setup_message diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 684e295da..c1bcb05d4 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet}; use anyhow::Context as _; use deltachat_derive::{FromSql, ToSql}; use lettre_email::mime::{self, Mime}; -use mailparse::{DispositionType, MailAddr, MailHeaderMap}; +use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo}; use crate::aheader::Aheader; use crate::blob::BlobObject; @@ -37,6 +37,11 @@ use crate::stock::StockMessage; pub struct MimeMessage { pub parts: Vec, header: HashMap, + + /// Addresses are normalized and lowercased: + pub recipients: Vec, + pub from: Vec, + pub chat_disposition_notification_to: Option, pub decrypting_failed: bool, pub signatures: HashSet, pub gossipped_addr: HashSet, @@ -88,9 +93,19 @@ impl MimeMessage { .unwrap_or_default(); let mut headers = Default::default(); + let mut recipients = Default::default(); + let mut from = Default::default(); + let mut chat_disposition_notification_to = None; // init known headers with what mailparse provided us - MimeMessage::merge_headers(&mut headers, &mail.headers); + MimeMessage::merge_headers( + context, + &mut headers, + &mut recipients, + &mut from, + &mut chat_disposition_notification_to, + &mail.headers, + ); // remove headers that are allowed _only_ in the encrypted part headers.remove("secure-join-fingerprint"); @@ -118,7 +133,14 @@ impl MimeMessage { // let known protected headers from the decrypted // part override the unencrypted top-level - MimeMessage::merge_headers(&mut headers, &decrypted_mail.headers); + MimeMessage::merge_headers( + context, + &mut headers, + &mut recipients, + &mut from, + &mut chat_disposition_notification_to, + &decrypted_mail.headers, + ); (decrypted_mail, signatures) } else { @@ -142,6 +164,9 @@ impl MimeMessage { let mut parser = MimeMessage { parts: Vec::new(), header: headers, + recipients, + from, + chat_disposition_notification_to, decrypting_failed: false, // only non-empty if it was a valid autocrypt message @@ -310,11 +335,9 @@ impl MimeMessage { // See if an MDN is requested from the other side if !self.decrypting_failed && !self.parts.is_empty() { - if let Some(ref dn_to_addr) = - self.parse_first_addr(context, HeaderDef::ChatDispositionNotificationTo) - { - if let Some(ref from_addr) = self.parse_first_addr(context, HeaderDef::From_) { - if compare_addrs(from_addr, dn_to_addr) { + if let Some(ref dn_to) = self.chat_disposition_notification_to { + if let Some(ref from) = self.from.get(0) { + if from.addr == dn_to.addr { if let Some(part) = self.parts.last_mut() { part.param.set_int(Param::WantsMdn, 1); } @@ -388,20 +411,6 @@ impl MimeMessage { self.header.get(headerdef.get_headername()) } - fn parse_first_addr(&self, context: &Context, headerdef: HeaderDef) -> Option { - if let Some(value) = self.get(headerdef.clone()) { - match mailparse::addrparse(&value) { - Ok(ref addrs) => { - return addrs.first().cloned(); - } - Err(err) => { - warn!(context, "header {} parse error: {:?}", headerdef, err); - } - } - } - None - } - fn parse_mime_recursive( &mut self, context: &Context, @@ -745,17 +754,41 @@ impl MimeMessage { .and_then(|msgid| parse_message_id(msgid).ok()) } - fn merge_headers(headers: &mut HashMap, fields: &[mailparse::MailHeader<'_>]) { + fn merge_headers( + context: &Context, + headers: &mut HashMap, + recipients: &mut Vec, + from: &mut Vec, + chat_disposition_notification_to: &mut Option, + fields: &[mailparse::MailHeader<'_>], + ) { for field in fields { // lowercasing all headers is technically not correct, but makes things work better let key = field.get_key().to_lowercase(); if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers) is_known(&key) || key.starts_with("chat-") { - let value = field.get_value(); - headers.insert(key.to_string(), value); + if key == HeaderDef::ChatDispositionNotificationTo.get_headername() { + match addrparse_header(field) { + Ok(addrlist) => { + *chat_disposition_notification_to = addrlist.extract_single_info(); + } + Err(e) => warn!(context, "Could not read {} address: {}", key, e), + } + } else { + let value = field.get_value(); + headers.insert(key.to_string(), value); + } } } + let recipients_new = get_recipients(fields); + if !recipients_new.is_empty() { + *recipients = recipients_new; + } + let from_new = get_from(fields); + if !from_new.is_empty() { + *from = from_new; + } } fn process_report( @@ -823,23 +856,15 @@ fn update_gossip_peerstates( gossip_headers: Vec, ) -> Result> { // XXX split the parsing from the modification part - let mut recipients: Option> = None; let mut gossipped_addr: HashSet = Default::default(); for value in &gossip_headers { let gossip_header = value.parse::(); if let Ok(ref header) = gossip_header { - if recipients.is_none() { - recipients = Some(get_recipients( - mail.headers.iter().map(|v| (v.get_key(), v.get_value())), - )); - } - - if recipients - .as_ref() - .unwrap() - .contains(&header.addr.to_lowercase()) + if get_recipients(&mail.headers) + .iter() + .any(|info| info.addr == header.addr.to_lowercase()) { let mut peerstate = Peerstate::from_addr(context, &context.sql, &header.addr); if let Some(ref mut peerstate) = peerstate { @@ -1004,50 +1029,50 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result, T: Iterator>(headers: T) -> HashSet { - let mut recipients: HashSet = Default::default(); +/// Returned addresses are normalized and lowercased. +fn get_recipients(headers: &[MailHeader]) -> Vec { + get_all_addresses_from_header(headers, |header_key| { + header_key == "to" || header_key == "cc" + }) +} - for (hkey, hvalue) in headers { - let hkey = hkey.as_ref().to_lowercase(); - let hvalue = hvalue.as_ref(); - if hkey == "to" || hkey == "cc" { - if let Ok(addrs) = mailparse::addrparse(hvalue) { - for addr in addrs.iter() { - match addr { - mailparse::MailAddr::Single(ref info) => { - recipients.insert(addr_normalize(&info.addr).to_lowercase()); - } - mailparse::MailAddr::Group(ref infos) => { - for info in &infos.addrs { - recipients.insert(addr_normalize(&info.addr).to_lowercase()); - } +/// Returned addresses are normalized and lowercased. +pub(crate) fn get_from(headers: &[MailHeader]) -> Vec { + get_all_addresses_from_header(headers, |header_key| header_key == "from") +} + +fn get_all_addresses_from_header(headers: &[MailHeader], pred: F) -> Vec +where + F: Fn(String) -> bool, +{ + let mut result: Vec = Default::default(); + + headers + .iter() + .filter(|header| pred(header.get_key().to_lowercase())) + .filter_map(|header| mailparse::addrparse_header(header).ok()) + .for_each(|addrs| { + for addr in addrs.iter() { + match addr { + mailparse::MailAddr::Single(ref info) => { + result.push(SingleInfo { + addr: addr_normalize(&info.addr).to_lowercase(), + display_name: info.display_name.clone(), + }); + } + mailparse::MailAddr::Group(ref infos) => { + for info in &infos.addrs { + result.push(SingleInfo { + addr: addr_normalize(&info.addr).to_lowercase(), + display_name: info.display_name.clone(), + }); } } } } - } - } + }); - recipients -} - -/// Check if the only addrs match, ignoring names. -fn compare_addrs(a: &mailparse::MailAddr, b: &mailparse::MailAddr) -> bool { - match a { - mailparse::MailAddr::Group(group_a) => match b { - mailparse::MailAddr::Group(group_b) => group_a - .addrs - .iter() - .zip(group_b.addrs.iter()) - .all(|(a, b)| a.addr == b.addr), - _ => false, - }, - mailparse::MailAddr::Single(single_a) => match b { - mailparse::MailAddr::Single(single_b) => single_a.addr == single_b.addr, - _ => false, - }, - } + result } #[cfg(test)] @@ -1096,12 +1121,11 @@ mod tests { #[test] fn test_get_recipients() { - let context = dummy_context(); let raw = include_bytes!("../test-data/message/mail_with_cc.txt"); - let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap(); - let recipients = get_recipients(mimeparser.header.iter()); - assert!(recipients.contains("abc@bcd.com")); - assert!(recipients.contains("def@def.de")); + let mail = mailparse::parse_mail(&raw[..]).unwrap(); + let recipients = get_recipients(&mail.headers); + assert!(recipients.iter().any(|info| info.addr == "abc@bcd.com")); + assert!(recipients.iter().any(|info| info.addr == "def@def.de")); assert_eq!(recipients.len(), 2); } @@ -1156,14 +1180,10 @@ mod tests { let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap(); - let of = mimeparser - .parse_first_addr(&context.ctx, HeaderDef::From_) - .unwrap(); - assert_eq!(of, mailparse::addrparse("hello@one.org").unwrap()[0]); + let of = &mimeparser.from[0]; + assert_eq!(of.addr, "hello@one.org"); - let of = - mimeparser.parse_first_addr(&context.ctx, HeaderDef::ChatDispositionNotificationTo); - assert!(of.is_none()); + assert!(mimeparser.chat_disposition_notification_to.is_none()); } #[test] From 2dbb1bbbeab42275e1f356f52f9af4f0fce14457 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Fri, 8 May 2020 00:48:20 +0300 Subject: [PATCH 138/156] Do not reply to hidden messages Especially with read receipts, it is wrong, because they are never encrypted and their Message-IDs are not known to other users in a group. --- src/chat.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chat.rs b/src/chat.rs index 1c766a0cd..45490e1ea 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -385,7 +385,7 @@ impl ChatId { let sql = &context.sql; let query = format!( "SELECT {} \ - FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) \ + FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) AND NOT hidden \ ORDER BY timestamp DESC, id DESC \ LIMIT 1;", fields From 0fefe11bfd8a7d221841e5f753524697a7d0f250 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sun, 10 May 2020 16:21:42 +0300 Subject: [PATCH 139/156] Do not return "empty rfc724_mid" errors from rfc724_mid_cnt This function should only return temporary errors, e.g. database errors, as precheck_imf() and dc_receive_imf::add_parts() treat them as such, retrying the fetch on failure. When permanent errors, like missing Message-ID, are bubbled up, they cause infinite fetch loop. --- src/message.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/message.rs b/src/message.rs index d96833ee7..bb067eb5e 100644 --- a/src/message.rs +++ b/src/message.rs @@ -1432,7 +1432,10 @@ pub(crate) fn rfc724_mid_exists( context: &Context, rfc724_mid: &str, ) -> Result, Error> { - ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid"); + if rfc724_mid.is_empty() { + warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists"); + return Ok(None); + } context .sql From 2f6bae4e2afc8a3f7a2c2b5f24083bb12ee553be Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sun, 10 May 2020 17:49:12 +0300 Subject: [PATCH 140/156] sql: do not send DC_EVENT_ERROR on database errors These errors are usually just "database busy" errors, it is enough to write them to the log instead of displaying to the user. --- src/sql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql.rs b/src/sql.rs index 2e5ef48f2..a09380a8a 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -242,7 +242,7 @@ impl Sql { match self.query_get_value_result(query, params) { Ok(res) => res, Err(err) => { - error!(context, "sql: Failed query_row: {}", err); + warn!(context, "sql: Failed query_row: {}", err); None } } From 000ed3175d58bbacf3a7d335f51114cf42c75276 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 14:55:46 +0200 Subject: [PATCH 141/156] add failing test --- src/mimeparser.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index c1bcb05d4..f2b565233 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -1713,4 +1713,48 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= assert_eq!(message.parts[0].typ, Viewtype::Image); assert_eq!(message.parts[0].msg, "Test"); } + + #[test] + fn test_parse_message_id() { + let test = parse_message_id(""); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foobar"); + + let test = parse_message_id(" "); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foo"); + + let test = parse_message_id(" < foo > "); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foo"); + + let test = parse_message_id("foo"); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foo"); + + let test = parse_message_id(" foo "); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foo"); + + let test = parse_message_id("foo bar"); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foo"); + + let test = parse_message_id(" foo bar "); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "foo"); + + let test = parse_message_id(""); + assert!(test.is_err()); + + let test = parse_message_id(" "); + assert!(test.is_err()); + + let test = parse_message_id("<>"); + assert!(test.is_err()); + + let test = parse_message_id("<> bar"); + assert!(test.is_ok()); + assert_eq!(test.unwrap(), "bar"); + } } From 0e72acee1084fd7155fc7a124472d0a8631c7fa4 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 15:22:54 +0200 Subject: [PATCH 142/156] more tolerant message-id parsing --- src/mimeparser.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index f2b565233..b7a343f3e 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -1,6 +1,5 @@ use std::collections::{HashMap, HashSet}; -use anyhow::Context as _; use deltachat_derive::{FromSql, ToSql}; use lettre_email::mime::{self, Mime}; use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo}; @@ -902,14 +901,21 @@ pub(crate) struct Report { additional_message_ids: Vec, } -pub(crate) fn parse_message_id(value: &str) -> crate::error::Result { - let ids = mailparse::msgidparse(value).context("failed to parse message id")?; - - if let Some(id) = ids.first() { - Ok(id.to_string()) - } else { - bail!("could not parse message_id: {}", value); +pub(crate) fn parse_message_id(ids: &str) -> crate::error::Result { + // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>` + for id in ids.split_whitespace() { + let mut id = id.to_string(); + if id.starts_with('<') { + id = id[1..].to_string(); + } + if id.ends_with('>') { + id = id[..id.len() - 1].to_string(); + } + if !id.is_empty() { + return Ok(id); + } } + bail!("could not parse message_id: {}", ids); } fn is_known(key: &str) -> bool { From 215cc5e71df8452e23bff28b09a5786218df56bd Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 18:56:42 +0200 Subject: [PATCH 143/156] add function for parsing multiple Message-Ids --- src/mimeparser.rs | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index b7a343f3e..1c4348eea 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -901,8 +901,9 @@ pub(crate) struct Report { additional_message_ids: Vec, } -pub(crate) fn parse_message_id(ids: &str) -> crate::error::Result { +pub(crate) fn parse_message_ids(ids: &str) -> Result> { // take care with mailparse::msgidparse() that is pretty untolerant eg. wrt missing `<` or `>` + let mut msgids = Vec::new(); for id in ids.split_whitespace() { let mut id = id.to_string(); if id.starts_with('<') { @@ -912,10 +913,18 @@ pub(crate) fn parse_message_id(ids: &str) -> crate::error::Result { id = id[..id.len() - 1].to_string(); } if !id.is_empty() { - return Ok(id); + msgids.push(id); } } - bail!("could not parse message_id: {}", ids); + Ok(msgids) +} + +pub(crate) fn parse_message_id(ids: &str) -> Result { + if let Some(id) = parse_message_ids(ids)?.first() { + Ok(id.to_string()) + } else { + bail!("could not parse message_id: {}", ids); + } } fn is_known(key: &str) -> bool { @@ -1763,4 +1772,26 @@ CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I= assert!(test.is_ok()); assert_eq!(test.unwrap(), "bar"); } + + #[test] + fn test_parse_message_ids() { + let test = parse_message_ids(" foo bar ").unwrap(); + assert_eq!(test.len(), 3); + assert_eq!(test[0], "foo"); + assert_eq!(test[1], "bar"); + assert_eq!(test[2], "foobar"); + + let test = parse_message_ids(" < foobar >").unwrap(); + assert_eq!(test.len(), 1); + assert_eq!(test[0], "foobar"); + + let test = parse_message_ids("").unwrap(); + assert!(test.is_empty()); + + let test = parse_message_ids(" ").unwrap(); + assert!(test.is_empty()); + + let test = parse_message_ids(" < ").unwrap(); + assert!(test.is_empty()); + } } From a406e0416fed6d4cee0319cf546b5ca476882c06 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 19:02:44 +0200 Subject: [PATCH 144/156] use new Message-ID parser --- src/dc_receive_imf.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dc_receive_imf.rs b/src/dc_receive_imf.rs index 8a10e0ecd..cdde8a2de 100644 --- a/src/dc_receive_imf.rs +++ b/src/dc_receive_imf.rs @@ -1520,7 +1520,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool { return false; } - if let Ok(ids) = mailparse::msgidparse(mid_list) { + if let Ok(ids) = parse_message_ids(mid_list) { for id in ids.iter() { if is_known_rfc724_mid(context, id) { return true; @@ -1568,7 +1568,7 @@ fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) - } pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool { - if let Ok(ids) = mailparse::msgidparse(mid_list) { + if let Ok(ids) = parse_message_ids(mid_list) { for id in ids.iter() { if is_msgrmsg_rfc724_mid(context, id) { return true; From c36227e2fcf606307f834816e707a78f4aa1f318 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Sun, 10 May 2020 23:16:38 +0300 Subject: [PATCH 145/156] Better SMTP ErrorNetwork message It uses stock string, just as for IMAP errors, and is distinguishable from IMAP errors: protocol is specified in the error message now. --- src/imap/mod.rs | 2 +- src/smtp/mod.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 3c90f235a..789ef02b2 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -269,7 +269,7 @@ impl Imap { let imap_port = config.imap_port; context.stock_string_repl_str2( StockMessage::ServerResponse, - format!("{}:{}", imap_server, imap_port), + format!("IMAP {}:{}", imap_server, imap_port), err.to_string(), ) }; diff --git a/src/smtp/mod.rs b/src/smtp/mod.rs index 83f9106e3..b8e698087 100644 --- a/src/smtp/mod.rs +++ b/src/smtp/mod.rs @@ -12,6 +12,7 @@ use crate::context::Context; use crate::events::Event; use crate::login_param::{dc_build_tls, LoginParam}; use crate::oauth2::*; +use crate::stock::StockMessage; /// SMTP write and read timeout in seconds. const SMTP_TIMEOUT: u64 = 30; @@ -171,7 +172,14 @@ impl Smtp { let mut trans = client.into_transport(); trans.connect().await.map_err(|err| { - emit_event!(context, Event::ErrorNetwork(err.to_string())); + let message = { + context.stock_string_repl_str2( + StockMessage::ServerResponse, + format!("SMTP {}:{}", domain, port), + err.to_string(), + ) + }; + emit_event!(context, Event::ErrorNetwork(message)); Error::ConnectionFailure(err) })?; From aa292ac6b88d949ce7adb13879cf7745fe0d466a Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 22:43:04 +0200 Subject: [PATCH 146/156] do normal receive_imf() if message-id is empty or if prefetch failed for other reasons. there are servers not sending a message ids, this and other cases is handled in receive_imf() - but not in prefetch (would be too much to maintain, also we need more information). this normal processing also prevents trying over the same message over and over as the server_uid is updated. --- src/imap/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 3c90f235a..f9bfcaf08 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -605,7 +605,7 @@ impl Imap { let headers = get_fetch_headers(fetch)?; let message_id = prefetch_get_message_id(&headers).unwrap_or_default(); - if precheck_imf(context, &message_id, folder.as_ref(), cur_uid)? { + if let Ok(true) = precheck_imf(context, &message_id, folder.as_ref(), cur_uid) { // we know the message-id already or don't want the message otherwise. info!( context, @@ -614,6 +614,9 @@ impl Imap { folder.as_ref(), ); } else { + // we do not know the message-id + // or the message-id is missing (in this case, we create one in the further process) + // or some other error happened let show = prefetch_should_download(context, &headers, show_emails) .map_err(|err| { warn!(context, "prefetch_should_download error: {}", err); From cc56edc91d9f952ad1839c2cdc1142819dec83ef Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Mon, 11 May 2020 00:19:14 +0300 Subject: [PATCH 147/156] Log precheck_imf errors --- src/imap/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 6a6e251cf..5cecc3488 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -605,7 +605,12 @@ impl Imap { let headers = get_fetch_headers(fetch)?; let message_id = prefetch_get_message_id(&headers).unwrap_or_default(); - if let Ok(true) = precheck_imf(context, &message_id, folder.as_ref(), cur_uid) { + if let Ok(true) = + precheck_imf(context, &message_id, folder.as_ref(), cur_uid).map_err(|err| { + warn!(context, "precheck_imf error: {}", err); + err + }) + { // we know the message-id already or don't want the message otherwise. info!( context, From 9f992409c7bda9266abca825f1d57956779ce616 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 23:57:21 +0200 Subject: [PATCH 148/156] changelog --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a463915..80560f305 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 1.32.0 + +- fix endless loop when trying to download messages with bad RFC Message-ID, + also be more reliable on similar errors #1463 #1466 #1462 + +- fix bug with comma in contact request #1438 + +- do not refer to hidden messages on replies #1459 + +- improve error handling #1468 #1465 #1464 + + ## 1.31.0 - always describe the context of the displayed error #1451 From 3f0136ae7c75fd56b2cf61dc7439e7b5dc164763 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Sun, 10 May 2020 23:58:22 +0200 Subject: [PATCH 149/156] bump version to 1.32.0 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b69b5987..4b1e1589d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,7 +629,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.31.0" +version = "1.32.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -695,10 +695,10 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.31.0" +version = "1.32.0" dependencies = [ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)", - "deltachat 1.31.0", + "deltachat 1.32.0", "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 46a356eb5..cffcfad77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.31.0" +version = "1.32.0" authors = ["Delta Chat Developers (ML) "] edition = "2018" license = "MPL-2.0" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index c604906a8..ad4bf5df0 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.31.0" +version = "1.32.0" description = "Deltachat FFI" authors = ["Delta Chat Developers (ML) "] edition = "2018" From fe23907eb3a5e27583f5b406aae204850a3f3023 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Mon, 11 May 2020 06:29:33 +0200 Subject: [PATCH 150/156] fix muting dm chats and rewrite the erroro message so that it makes more sense --- src/chat.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index 45490e1ea..b765766ce 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2072,8 +2072,7 @@ impl rusqlite::types::FromSql for MuteDuration { pub fn set_muted(context: &Context, chat_id: ChatId, duration: MuteDuration) -> Result<(), Error> { ensure!(!chat_id.is_special(), "Invalid chat ID"); - if real_group_exists(context, chat_id) - && sql::execute( + if sql::execute( context, &context.sql, "UPDATE chats SET muted_until=? WHERE id=?;", @@ -2083,7 +2082,7 @@ pub fn set_muted(context: &Context, chat_id: ChatId, duration: MuteDuration) -> { context.call_cb(Event::ChatModified(chat_id)); } else { - bail!("Failed to set name"); + bail!("Failed to set mute duration, chat might not exist -"); } Ok(()) } From c2c0c81f1ca33b79d76a5cf1e55c905763bf3b20 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Mon, 11 May 2020 06:31:48 +0200 Subject: [PATCH 151/156] cargo fmt --- src/chat.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/chat.rs b/src/chat.rs index b765766ce..c9a65538c 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2073,12 +2073,12 @@ impl rusqlite::types::FromSql for MuteDuration { pub fn set_muted(context: &Context, chat_id: ChatId, duration: MuteDuration) -> Result<(), Error> { ensure!(!chat_id.is_special(), "Invalid chat ID"); if sql::execute( - context, - &context.sql, - "UPDATE chats SET muted_until=? WHERE id=?;", - params![duration, chat_id], - ) - .is_ok() + context, + &context.sql, + "UPDATE chats SET muted_until=? WHERE id=?;", + params![duration, chat_id], + ) + .is_ok() { context.call_cb(Event::ChatModified(chat_id)); } else { From 682d52441df209479e9721e63584778b7e2e171b Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 11 May 2020 15:30:33 +0200 Subject: [PATCH 152/156] add dc_estimate_deletion_cnt() to docs, add some references --- deltachat-ffi/deltachat.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 5bc9f3112..4db0710d0 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -378,10 +378,12 @@ char* dc_get_blobdir (const dc_context_t* context); * >=1=seconds, after which messages are deleted automatically from the device. * Messages in the "saved messages" chat (see dc_chat_is_self_talk()) are skipped. * Messages are deleted whether they were seen or not, the UI should clearly point that out. + * See also dc_estimate_deletion_cnt(). * - `delete_server_after` = 0=do not delete messages from server automatically (default), * >=1=seconds, after which messages are deleted automatically from the server. * "Saved messages" are deleted from the server as well as * emails matching the `show_emails` settings above, the UI should clearly point that out. + * See also dc_estimate_deletion_cnt(). * - `media_quality` = DC_MEDIA_QUALITY_BALANCED (0) = * good outgoing images/videos/voice quality at reasonable sizes (default) * DC_MEDIA_QUALITY_WORSE (1) @@ -1318,6 +1320,7 @@ int dc_get_fresh_msg_cnt (dc_context_t* context, uint32_t ch * by the dc_set_config()-options `delete_device_after` or `delete_server_after`. * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages. * + * @memberof dc_context_t * @param context The context object as returned from dc_context_new(). * @param from_server 1=Estimate deletion count for server, 0=Estimate deletion count for device * @param seconds Count messages older than the given number of seconds. From c185d5b0b5fdf155eb0ea66c546ec93d7975d834 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Tue, 12 May 2020 00:44:57 +0300 Subject: [PATCH 153/156] Fix python lint error about unused format --- python/examples/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/test_examples.py b/python/examples/test_examples.py index 5e6d363fe..da32b127d 100644 --- a/python/examples/test_examples.py +++ b/python/examples/test_examples.py @@ -51,7 +51,7 @@ def test_group_tracking_plugin(acfactory, lp): botproc.fnmatch_lines(""" *ac_chat_modified*bot test group* - """.format(ac1.get_config("addr"))) + """) lp.sec("adding third member {}".format(ac2.get_config("addr"))) contact3 = ac1.create_contact(ac2.get_config("addr")) From f0d9bdd9015b6312edfe5ed0a5560d10c45251df Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Tue, 12 May 2020 17:48:03 +0200 Subject: [PATCH 154/156] clarify docs for DC_GCL_FOR_FORWARDING --- deltachat-ffi/deltachat.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 4db0710d0..53e202fa3 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -959,8 +959,10 @@ int dc_preconfigure_keypair (dc_context_t* context, const cha * if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and * the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived * chats - * - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist, + * - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist + * and hides the "Device chat" and the deaddrop. * typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS + * to also hide the archive link. * - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added * to the list (may be used eg. for selecting chats on forwarding, the flag is * not needed when DC_GCL_ARCHIVED_ONLY is already set) From fca9eae0fdae00b72b1629b09e4bc0ec7e9c4923 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 12 May 2020 20:23:03 +0200 Subject: [PATCH 155/156] Extract flags in try_load() to variables --- src/chatlist.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/chatlist.rs b/src/chatlist.rs index ce3569f7b..bf88c79ee 100644 --- a/src/chatlist.rs +++ b/src/chatlist.rs @@ -92,6 +92,11 @@ impl Chatlist { query: Option<&str>, query_contact_id: Option, ) -> Result { + let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY; + let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING; + let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS; + let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT; + // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some // messages get deleted to avoid reloading the same chatlist. if let Err(err) = delete_device_expired_messages(context) { @@ -111,7 +116,7 @@ impl Chatlist { .map_err(Into::into) }; - let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING { + let skip_id = if flag_for_forwarding { chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE) .unwrap_or_default() .0 @@ -155,7 +160,7 @@ impl Chatlist { process_row, process_rows, )? - } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY { + } else if flag_archived_only { // show archived chats // (this includes the archived device-chat; we could skip it, // however, then the number of archived chats do not match, which might be even more irritating. @@ -211,7 +216,7 @@ impl Chatlist { )? } else { // show normal chatlist - let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING { + let sort_id_up = if flag_for_forwarding { chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF) .unwrap_or_default() .0 @@ -237,9 +242,9 @@ impl Chatlist { process_row, process_rows, )?; - if 0 == listflags & DC_GCL_NO_SPECIALS { + if !flag_no_specials { if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) { - if 0 == listflags & DC_GCL_FOR_FORWARDING { + if !flag_for_forwarding { ids.insert( 0, (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id), @@ -252,7 +257,7 @@ impl Chatlist { }; if add_archived_link_item && dc_get_archived_cnt(context) > 0 { - if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT { + if ids.is_empty() && flag_add_alldone_hint { ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0))); } ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0))); From 3ee81cbee032a9487332b09997d7578825c95407 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Tue, 12 May 2020 00:32:18 +0300 Subject: [PATCH 156/156] Revert "imap: simplify select_folder() interface" This reverts commit b614de2f8062772bd9b042550b5d24a91aa5c8ad. --- src/imap/idle.rs | 4 +-- src/imap/mod.rs | 8 +++--- src/imap/select_folder.rs | 59 +++++++++++++++++++++------------------ src/job_thread.rs | 17 ++++++----- 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/src/imap/idle.rs b/src/imap/idle.rs index d437e1619..30e27b9ee 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -59,7 +59,7 @@ impl Imap { task::block_on(async move { self.config.read().await.can_idle }) } - pub fn idle(&self, context: &Context, watch_folder: String) -> Result<()> { + pub fn idle(&self, context: &Context, watch_folder: Option) -> Result<()> { task::block_on(async move { if !self.can_idle() { return Err(Error::IdleAbilityMissing); @@ -67,7 +67,7 @@ impl Imap { self.setup_handle_if_needed(context).await?; - self.select_folder(context, watch_folder).await?; + self.select_folder(context, watch_folder.clone()).await?; let session = self.session.lock().await.take(); let timeout = Duration::from_secs(23 * 60); diff --git a/src/imap/mod.rs b/src/imap/mod.rs index 5cecc3488..1e19c7844 100644 --- a/src/imap/mod.rs +++ b/src/imap/mod.rs @@ -477,7 +477,7 @@ impl Imap { folder: &str, ) -> Result<(u32, u32)> { task::block_on(async move { - self.select_folder(context, folder).await?; + self.select_folder(context, Some(folder)).await?; // compare last seen UIDVALIDITY against the current one let (uid_validity, last_seen_uid) = self.get_config_last_seen_uid(context, &folder); @@ -915,7 +915,7 @@ impl Imap { return Some(ImapActionResult::RetryLater); } } - match self.select_folder(context, &folder).await { + match self.select_folder(context, Some(&folder)).await { Ok(()) => None, Err(select_folder::Error::ConnectionLost) => { warn!(context, "Lost imap connection"); @@ -1186,7 +1186,7 @@ impl Imap { error!(context, "could not setup imap connection: {}", err); return; } - if let Err(err) = self.select_folder(context, &folder).await { + if let Err(err) = self.select_folder(context, Some(&folder)).await { error!( context, "Could not select {} for expunging: {}", folder, err @@ -1204,7 +1204,7 @@ impl Imap { // we now trigger expunge to actually delete messages self.config.write().await.selected_folder_needs_expunge = true; - match self.select_folder(context, &folder).await { + match self.select_folder::(context, None).await { Ok(()) => { emit_event!(context, Event::ImapFolderEmptied(folder.to_string())); } diff --git a/src/imap/select_folder.rs b/src/imap/select_folder.rs index e0fce0dff..4d454c332 100644 --- a/src/imap/select_folder.rs +++ b/src/imap/select_folder.rs @@ -56,7 +56,7 @@ impl Imap { pub(super) async fn select_folder>( &self, context: &Context, - folder: S, + folder: Option, ) -> Result<()> { if self.session.lock().await.is_none() { let mut cfg = self.config.write().await; @@ -71,41 +71,46 @@ impl Imap { self.close_folder(context).await?; } - if self.config.read().await.selected_folder.as_deref() == Some(folder.as_ref()) { + let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref()); + if self.config.read().await.selected_folder.as_deref() == folder_str { return Ok(()); } // select new folder - if let Some(ref mut session) = &mut *self.session.lock().await { - let res = session.select(&folder).await; + if let Some(ref folder) = folder { + if let Some(ref mut session) = &mut *self.session.lock().await { + let res = session.select(folder).await; - // https://tools.ietf.org/html/rfc3501#section-6.3.1 - // says that if the server reports select failure we are in - // authenticated (not-select) state. + // https://tools.ietf.org/html/rfc3501#section-6.3.1 + // says that if the server reports select failure we are in + // authenticated (not-select) state. - match res { - Ok(mailbox) => { - let mut config = self.config.write().await; - config.selected_folder = Some(folder.as_ref().to_string()); - config.selected_mailbox = Some(mailbox); - Ok(()) - } - Err(async_imap::error::Error::ConnectionLost) => { - self.trigger_reconnect(); - self.config.write().await.selected_folder = None; - Err(Error::ConnectionLost) - } - Err(async_imap::error::Error::Validate(_)) => { - Err(Error::BadFolderName(folder.as_ref().to_string())) - } - Err(err) => { - self.config.write().await.selected_folder = None; - self.trigger_reconnect(); - Err(Error::Other(err.to_string())) + match res { + Ok(mailbox) => { + let mut config = self.config.write().await; + config.selected_folder = Some(folder.as_ref().to_string()); + config.selected_mailbox = Some(mailbox); + Ok(()) + } + Err(async_imap::error::Error::ConnectionLost) => { + self.trigger_reconnect(); + self.config.write().await.selected_folder = None; + Err(Error::ConnectionLost) + } + Err(async_imap::error::Error::Validate(_)) => { + Err(Error::BadFolderName(folder.as_ref().to_string())) + } + Err(err) => { + self.config.write().await.selected_folder = None; + self.trigger_reconnect(); + Err(Error::Other(err.to_string())) + } } + } else { + Err(Error::NoSession) } } else { - Err(Error::NoSession) + Ok(()) } } } diff --git a/src/job_thread.rs b/src/job_thread.rs index 0e524e161..12512a3d6 100644 --- a/src/job_thread.rs +++ b/src/job_thread.rs @@ -173,15 +173,14 @@ impl JobThread { if !self.imap.can_idle() { true // we have to do fake_idle } else { - if let Some(watch_folder) = self.get_watch_folder(context) { - info!(context, "{} started...", prefix); - let res = self.imap.idle(context, watch_folder); - info!(context, "{} ended...", prefix); - if let Err(err) = res { - warn!(context, "{} failed: {} -> reconnecting", prefix, err); - // something is Label { Label }orked, let's start afresh on the next occassion - self.imap.disconnect(context); - } + let watch_folder = self.get_watch_folder(context); + info!(context, "{} started...", prefix); + let res = self.imap.idle(context, watch_folder); + info!(context, "{} ended...", prefix); + if let Err(err) = res { + warn!(context, "{} failed: {} -> reconnecting", prefix, err); + // something is borked, let's start afresh on the next occassion + self.imap.disconnect(context); } false }