Add some information about mailing list managers.

Remove additional chat type MailingList, make them GroupLists with a special param, make can_send() return false, code style and deduplication

.github/workflows/code-quality.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2019-11-06
+          toolchain: nightly-2020-03-12
           override: true
       - uses: actions-rs/cargo@v1
         with:
@@ -25,7 +25,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2019-11-06
+          toolchain: nightly-2020-03-12
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -39,10 +39,10 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: nightly-2019-11-06
+            toolchain: nightly-2020-03-12
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
+          args: --all-features

Cargo.lock

@@ -50,6 +50,11 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "arrayref"
 version = "0.3.6"
@@ -101,7 +106,7 @@ dependencies = [
  "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rental 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "stop-token 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -111,7 +116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "async-std 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -373,24 +378,6 @@ name = "bytes"
 version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "bzip2"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bzip2-sys 0.1.8+1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
-[[package]]
-name = "bzip2-sys"
-version = "0.1.8+1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)",
- "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "c2-chacha"
 version = "0.2.3"
@@ -644,6 +631,7 @@ dependencies = [
 name = "deltachat"
 version = "1.28.0"
 dependencies = [
+ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-native-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-smtp 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -659,11 +647,9 @@ dependencies = [
  "email 0.0.21 (git+https://github.com/deltachat/rust-email)",
  "encoded-words 0.1.0 (git+https://github.com/async-email/encoded-words)",
  "escaper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "image 0.22.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "image-meta 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "image-meta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -677,7 +663,7 @@ dependencies = [
  "pgp 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "proptest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proptest 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "quick-xml 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "r2d2 0.8.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "r2d2_sqlite 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -695,8 +681,8 @@ dependencies = [
  "strum 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "strum_macros 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "thread-local-object 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "zip 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -711,12 +697,13 @@ dependencies = [
 name = "deltachat_ffi"
 version = "1.28.0"
 dependencies = [
+ "anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)",
  "deltachat 1.28.0",
- "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -827,7 +814,7 @@ dependencies = [
  "hex 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1213,18 +1200,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "human-panic"
-version = "1.0.1"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "os_type 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
- "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1301,12 +1286,12 @@ dependencies = [
 
 [[package]]
 name = "image-meta"
-version = "0.1.0"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "skeptic 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1871,7 +1856,7 @@ dependencies = [
  "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "twofish 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "x25519-dalek 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1922,11 +1907,6 @@ dependencies = [
  "inflate 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "podio"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "ppv-lite86"
 version = "0.2.6"
@@ -1978,7 +1958,7 @@ dependencies = [
 
 [[package]]
 name = "proptest"
-version = "0.9.5"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2710,14 +2690,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "termcolor"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "termcolor"
 version = "1.1.0"
@@ -2728,15 +2700,15 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.11"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "thiserror-impl 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.11"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2810,7 +2782,7 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.4.10"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2921,15 +2893,6 @@ name = "utf8parse"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "uuid"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "uuid"
 version = "0.8.1"
@@ -3098,14 +3061,6 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "wincolor"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "winreg"
 version = "0.6.2"
@@ -3160,18 +3115,6 @@ dependencies = [
  "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "zip"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
- "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
- "podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [metadata]
 "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
 "checksum aes 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "54eb1d8fe354e5fc611daf4f2ea97dd45a765f4f1e4512306ec183ae2e8f20c9"
@@ -3179,6 +3122,7 @@ dependencies = [
 "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100"
 "checksum aho-corasick 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e63fd144e18ba274ae7095c0197a870a7b9468abc801dd62f190d80817d2ec"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
+"checksum anyhow 1.0.28 (registry+https://github.com/rust-lang/crates.io-index)" = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff"
 "checksum arrayref 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
 "checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
 "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
@@ -3216,8 +3160,6 @@ dependencies = [
 "checksum bytecount 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b92204551573580e078dc80017f36a213eb77a0450e4ddd8cfa0f3f2d1f0178f"
 "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
 "checksum bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1"
-"checksum bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b"
-"checksum bzip2-sys 0.1.8+1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "05305b41c5034ff0e93937ac64133d109b5a2660114ec45e9760bc6816d83038"
 "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
 "checksum cargo_metadata 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e5d1b4d380e1bab994591a24c2bdd1b054f64b60bef483a8c598c7c345bc3bbe"
 "checksum cast5 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ce5759b4c52ca74f9a98421817c882f1fd9b0071ae41cd61ab9f9d059c04fd6"
@@ -3306,14 +3248,14 @@ dependencies = [
 "checksum http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b"
 "checksum http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
-"checksum human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "21638c5955a6daf3ecc42cae702335fc37a72a4abcc6959ce457b31a7d43bbdd"
+"checksum human-panic 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "39f357a500abcbd7c5f967c1d45c8838585b36743823b9d43488f24850534e36"
 "checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
 "checksum hyper 0.13.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e7b15203263d1faa615f9337d79c1d37959439dc46c2b4faab33286fadc2a1c5"
 "checksum hyper-tls 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa"
 "checksum ident_case 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
 "checksum image 0.22.5 (registry+https://github.com/rust-lang/crates.io-index)" = "08ed2ada878397b045454ac7cfb011d73132c59f31a955d230bd1f1c2e68eb4a"
-"checksum image-meta 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b00861cbbb254a627d8acc0cec786b484297d896ab8f20fdc8e28536a3e918ef"
+"checksum image-meta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a2da7b4225d0954c9b8ba1a0dcec85be29f496cba4d85f9390426f810e3ab0d"
 "checksum imap-proto 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "16a6def1d5ac8975d70b3fd101d57953fe3278ef2ee5d7816cba54b1d1dfc22f"
 "checksum indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292"
 "checksum inflate 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
@@ -3381,14 +3323,13 @@ dependencies = [
 "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
 "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
 "checksum png 0.15.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ef859a23054bbfee7811284275ae522f0434a3c8e7f4b74bd4a35ae7e1c4a283"
-"checksum podio 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd"
 "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
 "checksum pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3f81e1644e1b54f5a68959a29aa86cde704219254669da328ecfdf6a1f09d427"
 "checksum pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "717ee476b1690853d222af4634056d830b5197ffd747726a9a1eee6da9f49074"
 "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5"
 "checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e"
 "checksum proc-macro2 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435"
-"checksum proptest 0.9.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bf6147d103a7c9d7598f4105cf049b15c99e2ecd93179bf024f0fd349be5ada4"
+"checksum proptest 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)" = "01c477819b845fe023d33583ebf10c9f62518c8d79a0960ba5c36d6ac8a55a5b"
 "checksum pulldown-cmark 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eef52fac62d0ea7b9b4dc7da092aa64ea7ec3d90af6679422d3d7e0e14b6ee15"
 "checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 "checksum quick-xml 0.17.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fe1e430bdcf30c9fdc25053b9c459bb1a4672af4617b6c783d7d91dc17c6bbb0"
@@ -3467,17 +3408,16 @@ dependencies = [
 "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
 "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
-"checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83"
 "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
-"checksum thiserror 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ee14bf8e6767ab4c687c9e8bc003879e042a96fd67a3ba5934eadb6536bef4db"
-"checksum thiserror-impl 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a7b51e1fbc44b5a0840be594fbc0f960be09050f2617e61e6aa43bef97cd3ef4"
+"checksum thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "f0570dc61221295909abdb95c739f2e74325e14293b2026b0a7e195091ec54ae"
+"checksum thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "227362df41d566be41a28f64401e07a043157c21c14b9785a0d8e256f940a8fd"
 "checksum thread-local-object 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7da3caa820d0308c84c8654f6cafd81cc3195d45433311cbe22fcf44fc8be071"
 "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
 "checksum tokio 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "0fa5e81d6bc4e67fe889d5783bd2a128ab2e0cfa487e0be16b6a8d177b101616"
 "checksum tokio-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828"
 "checksum tokio-util 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930"
-"checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f"
+"checksum toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a"
 "checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
 "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079"
 "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
@@ -3494,7 +3434,6 @@ dependencies = [
 "checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f"
 "checksum url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb"
 "checksum utf8parse 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d"
-"checksum uuid 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e1436e58182935dcd9ce0add9ea0b558e8a87befe01c1a301e6020aeb0876363"
 "checksum uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11"
 "checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
 "checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
@@ -3517,11 +3456,9 @@ dependencies = [
 "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 "checksum winapi-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4ccfbf554c6ad11084fb7517daca16cfdcaccbdadba4fc336f032a8b12c2ad80"
 "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-"checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767"
 "checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9"
 "checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum x25519-dalek 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "637ff90c9540fa3073bb577e65033069e4bae7c79d49d74aa3ffdf5342a53217"
 "checksum zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8"
 "checksum zeroize_derive 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2"
-"checksum zip 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6df134e83b8f0f8153a094c7b0fd79dfebe437f1d76e7715afa18ed95ebe2fd7"

Cargo.toml

@@ -32,8 +32,6 @@ percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
-failure = "0.1.5"
-failure_derive = "0.1.5"
 indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
@@ -58,9 +56,10 @@ encoded-words = { git = "https://github.com/async-email/encoded-words", branch="
 native-tls = "0.2.3"
 image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
 pretty_env_logger = "0.3.1"
-zip = "0.5"
 
 rustyline = { version = "4.1.0", optional = true }
+thiserror = "1.0.14"
+anyhow = "1.0.28"
 
 [dev-dependencies]
 tempfile = "3.0"

appveyor.yml

@@ -4,7 +4,7 @@ environment:
 
 install:
   - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2019-07-10
+  - rustup-init -yv --default-toolchain nightly-2020-03-12
   - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
   - rustc -vV
   - cargo -vV

assets/exported-chat.css

@@ -1,788 +0,0 @@
-/* TODO inlcude the referenced svgs as base64 data uris */
-
-.header {
-  background-color: #415e6b;
-  color: #fff;
-  position: absolute;
-  height: 52px;
-  width: 100%;
-  z-index: 5;
-  display:flex;
-}
-
-.header .avatar {
-  height: 36px;
-  width: 36px;
-  border-radius: 100%;
-  user-select: none;
-  margin: 8px;
-}
-
-.header .avatar.text-avatar {
-  background-color: #505050;
-  color: white;
-  font-size: 26px;
-  line-height: 36px;
-  text-align: center;
-}
-
-.header .name {
-  height: 52px;
-  line-height: 52px;
-  margin-left: 3px;
-  font-size: 20px;
-}
-
-.message.outgoing .author-avatar, .message.outgoing .author {
-  display: none!important;
-}
-
-:root {--colorPrimary: #42A5F5;--colorDanger: #f96856;--colorNone: #a0a0a0;--ovalButtonBg: #415e6b;--ovalButtonBgHover: rgb(120, 156, 173);--ovalButtonText: #fff;--ovalButtonTextHover: rgb(0, 0, 0);--navBarBackground: #415e6b;--navBarText: #fff;--navBarSearchPlaceholder: rgb(186, 186, 186);--navBarGroupSubtitle: rgb(186, 186, 186);--chatViewBg: #e6dcd4;--chatViewBgImgPath: url(../images/background_light.svg);--composerBg: #fff;--composerText: #010101;--composerPlaceholderText: rgba(1, 1, 1, 0.5);--composerBtnColor: rgba(1, 1, 1, 0.9);--composerSendButton: #415e6b;--emojiSelectorSelectionColor: #2090ea;--chatListItemSelectedBg: #4c6e7d;--chatListItemSelectedBgHover: #5E889B;--chatListItemSelectedText: #fff;--chatListItemBgHover: rgb(228, 228, 228);--chatListBorderColor: #b9b9b9;--chatListBorder: 1px solid undefined;--messageText: #010101;--messageTextLink: #010101;--setupMessageText: #ed824e;--infoMessageBubbleBg: #0000008c;--infoMessageBubbleText: white;--messageIncommingBg: #fff;--messageIncommingDate: #010101;--messageOutgoingBg: #efffde;--messageOutgoingStatusColor: #4caf50;--messageButtons: #8b8e91;--messageButtonsHover: #070c14;--messageStatusIcon: #4caf50;--messageStatusIconSending: #62656a;--messagePadlockOutgoing: #4caf50;--messagePadlockIncomming: #a4a6a9;--messageMetadataDate: #62656a;--messageMetadataIncomming: rgba(#ffffff, 0.7);--messageAuthor: #ffffff;--messageAttachmentIconExtentionColor: #070c14;--messageAttachmentIconBg: transparent;--messageAttachmentFileInfo: #010101;--loginInputFocusColor: #42A5F5;--loginButtonText: #42A5F5;--deltaChatPrimaryFg: #010101;--deltaChatPrimaryFgLight: #62656a;--contextMenuBg: #fff;--contextMenuBorder: rgb(221, 221, 221);--contextMenuText: #62656a;--contextMenuSelected: #f5f5f5;--contextMenuSelectedBg: #a4a6a9;--bp3DialogHeaderBg: #fff;--bp3DialogHeaderIcon: #666666;--bp3DialogBgSecondary: #ececec;--bp3DialogBgPrimary: #fff;--bp3Heading: #010101;--bp3ButtonText: #010101;--bp3ButtonBg: #fff;--bp3ButtonGradientTop: rgba(255,255,255,0.8);--bp3ButtonGradientBottom: rgba(255,255,255,0);--bp3ButtonHoverBg: #ebf1f5;--bp3InputText: #010101;--bp3InputBg: #fff;--bp3InputPlaceholder: lightgray;--bp3MenuText: #010101;--bp3MenuBg: #fff;--bp3Switch: #7a8084;--bp3SwitchShadow: unset;--bp3SwitchChecked: #acd4e8;--bp3SwitchShadowChecked: unset;--bp3SwitchKnob: #f5f5f5;--bp3SwitchKnobShadow: 0px 2px 0 0px #d2cfcfad;--bp3SwitchKnobChecked: #42A5F5;--bp3SwitchKnobShadowChecked: 0px 1px 0 0px #c9d4d2d1;--bp3SpinnerTrack: #acd4e8;--bp3SpinnerHead: #42a5f5;--bp3SelectorTop: rgba(255, 255, 255, 0.8);--bp3SelectorBottom: rgba(255, 255, 255, 0.0);--outlineProperties: 1px solid transparent;--outlineColor: b9b9b9;--emojiMartText: #010101;--emojiMartSearchBorder: lightgrey;--emojiMartBg: #fff;--emojiMartOutsideRadius: 5px;--emojiMartCategoryIcons: rgb(99, 99, 99);--emojiMartInputBg: #f5f5f5;--emojiMartInputText: #010101;--emojiMartInputPlaceholder: rgb(74, 74, 74);--emojiMartSelect: rgb(198, 198, 198);--galleryBg: #fff;--avatarLabelColor: #ffffff;--brokenMediaText: #070c14;--brokenMediaBg: #ffffff;--unreadCountBg: #2090ea;--unreadCountLabel: #ffffff;--contactListItemBg: #62656a;--contactListInitalColor: #62656a;--contactEmailColor: #62656a;--errorColor: #f44336;--globalLinkColor: #2090ea;--globalBackground: #fff;--globalText: #010101;--mapOverlayBg: #fff;--videoPlayBtnIcon: #2090ea;--videoPlayBtnBg: #ffffff;--scrollbarThumb: #666666;--scrollbarThumbHover: #606060;}
-* {
-  box-sizing: border-box;
-}
-html {
-  height: 100%;
-  --messageIncommingBg: rgb(232, 232, 232);
-}
-body {
-  position: relative;
-  height: 100%;
-  width: 100%;
-  margin: 0;
-  font-family: Roboto, "Apple Color Emoji", NotoEmoji, "Helvetica Neue", Arial,
-    Helvetica, NotoMono, sans-serif !important;
-  font-size: 14px;
-  color: black;
-  background-color: white;
-}
-ul {
-  list-style: none;
-  padding-left: 0;
-}
-
-input:focus {
-  outline: 0 !important;
-}
-button:focus {
-  outline: none;
-}
-button:focus {
-  outline: none;
-}
-::-webkit-scrollbar {
-  width: 6px;
-  height: 0;
-}
-::-webkit-scrollbar-track {
-  background: white;
-}
-::-webkit-scrollbar-thumb {
-  background: var(--scrollbarThumb);
-}
-::-webkit-scrollbar-thumb:hover {
-  background: var(--scrollbarThumbHover);
-}
-::-webkit-scrollbar-corner {
-  background: transparent;
-}
-span.module-contact-name {
-  font-weight: 200;
-  font-size: medium;
-}
-.module-contact-name__profile-name {
-  font-style: italic;
-}
-.AvatarBubble {
-  position: relative;
-  z-index: 2;
-  object-fit: cover;
-  height: 48px;
-  width: 48px;
-  margin-top: 8px;
-  margin-bottom: 8px;
-  border-radius: 100%;
-  background-color: #505050;
-  color: var(--avatarLabelColor);
-  font-size: 26px;
-  line-height: 48px;
-  text-align: center;
-  user-select: none;
-}
-.AvatarBubble.large {
-  height: 64px;
-  width: 64px;
-  line-height: 64px;
-  font-size: 39px;
-}
-.AvatarBubble--NoSearchResults {
-  transform: rotate(45deg);
-  line-height: 46px;
-  letter-spacing: 1px;
-}
-.AvatarBubble--NoSearchResults::after {
-  content: ":-(";
-}
-.AvatarImage {
-  position: relative;
-  z-index: 2;
-  object-fit: cover;
-  height: 48px;
-  width: 48px;
-  margin-top: 8px;
-  margin-bottom: 8px;
-  border-radius: 100%;
-  user-select: none;
-}
-.AvatarImage.large {
-  height: 64px;
-  width: 64px;
-}
-.attachment-overlay .attachment-view {
-  height: 100%;
-  padding: 0;
-  margin: 0;
-  display: flex;
-  align-items: center;
-  justify-content: center;
-  background-color: #313131;
-}
-.attachment-overlay .attachment-view img,
-.attachment-overlay .attachment-view video {
-  width: 100vw;
-  max-height: 100vh;
-  object-fit: contain;
-}
-.attachment-overlay .attachment-view video {
-  width: 95vw;
-}
-.attachment-overlay .render-media-wrapper {
-  width: 100vw;
-  height: 100vh;
-}
-.attachment-overlay .btn-wrapper {
-  float: right;
-  position: absolute;
-  z-index: 10;
-  cursor: pointer;
-}
-.attachment-overlay .download-btn {
-  height: 36px;
-  width: 36px;
-  display: inline-block;
-  -webkit-mask: url("../images/download.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageButtons);
-}
-.attachment-overlay .download-btn:hover {
-  background-color: var(--messageButtons);
-}
-.message-attachment-media {
-  text-align: center;
-  position: relative;
-  cursor: pointer;
-  margin-left: -12px;
-  margin-right: -12px;
-  margin-top: -10px;
-  margin-bottom: -10px;
-  border-radius: 16px;
-  overflow: hidden;
-  background-color: var(--messageAttachmentIconBg);
-}
-.message-attachment-media > .attachment-content {
-  object-fit: scale-down;
-  object-position: center;
-  min-height: 150px;
-  max-height: 300px;
-  max-width: 40vw;
-  margin-bottom: -4px;
-  cursor: pointer;
-}
-.message-attachment-media > .video-play-btn {
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  transform: translate(-50%, -50%);
-  width: 48px;
-  height: 48px;
-  background-color: var(--videoPlayBtnBg);
-  border-radius: 24px;
-}
-.message-attachment-media > .video-play-btn > .video-play-btn-icon {
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  transform: translate(-50%, -50%);
-  height: 36px;
-  width: 36px;
-  -webkit-mask: url("../images/play.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--videoPlayBtnIcon);
-}
-.message-attachment-media.content-below {
-  margin-bottom: 7px;
-  border-bottom-left-radius: 0px;
-  border-bottom-right-radius: 0px;
-}
-.message-attachment-media.content-above {
-  margin-top: 4px;
-  border-top-left-radius: 0px;
-  border-top-right-radius: 0px;
-}
-.message-attachment-broken-media {
-  font-size: 11px;
-  line-height: 16px;
-  letter-spacing: 0.3px;
-  padding: 10px;
-  text-align: center;
-  text-transform: uppercase;
-  color: var(--brokenMediaBg);
-}
-.message-attachment-broken-media.incoming {
-  color: var(--brokenMediaText);
-}
-.message-attachment-audio {
-  margin-top: 2px;
-  display: block;
-  margin-right: 30px;
-}
-.message-attachment-audio.content-below {
-  margin-bottom: 5px;
-}
-.message-attachment-audio.content-above {
-  margin-top: 6px;
-}
-.message-attachment-generic {
-  display: flex;
-  flex-direction: row;
-}
-.message-attachment-generic.content-below {
-  padding-bottom: 6px;
-}
-.message-attachment-generic.content-above {
-  padding-top: 4px;
-}
-.message-attachment-generic > .file-icon {
-  background: url("../images/file-gradient.svg") no-repeat center;
-  height: 44px;
-  width: 56px;
-  margin-left: -13px;
-  margin-right: -14px;
-  margin-bottom: -4px;
-  display: flex;
-  flex-direction: row;
-  align-items: center;
-}
-.message-attachment-generic > .file-icon > .file-extension {
-  font-size: 10px;
-  line-height: 13px;
-  letter-spacing: 0.1px;
-  text-transform: uppercase;
-  text-align: center;
-  width: 25px;
-  margin-left: auto;
-  margin-right: auto;
-  overflow-x: hidden;
-  white-space: nowrap;
-  text-overflow: clip;
-  color: var(--messageAttachmentIconExtentionColor);
-  font-family: monospace;
-}
-.message-attachment-generic > .text-part {
-  flex-grow: 1;
-  margin-left: 8px;
-  max-width: calc(100% - 37px);
-}
-.message-attachment-generic > .text-part > .name {
-  color: var(--messageAttachmentFileInfo);
-  font-size: 14px;
-  line-height: 18px;
-  font-weight: 300;
-  margin-top: 2px;
-  overflow-x: hidden;
-  white-space: nowrap;
-  text-overflow: ellipsis;
-}
-.message-attachment-generic > .text-part > .size {
-  color: var(--messageAttachmentFileInfo);
-  font-size: 11px;
-  line-height: 16px;
-  letter-spacing: 0.3px;
-  margin-top: 3px;
-}
-.module-message-detail {
-  margin-top: -20px;
-}
-.module-message-detail .bp3-callout {
-  max-height: 50vh;
-  overflow: auto;
-}
-.module-message-detail p {
-  white-space: pre-line;
-  user-select: text;
-}
-.module-message-detail__message-container {
-  padding-top: 20px;
-  padding-bottom: 20px;
-}
-.module-message-detail__message-container:after {
-  content: ".";
-  visibility: hidden;
-  display: block;
-  height: 0;
-  clear: both;
-}
-.module-message-detail__label {
-  font-weight: 300;
-  padding-right: 5px;
-}
-.module-message-detail__unix-timestamp {
-  color: #eeefef;
-}
-.module-message-detail__delete-button-container {
-  text-align: center;
-  margin-top: 10px;
-}
-.module-message-detail__delete-button {
-  background: none;
-  color: inherit;
-  border: none;
-  padding: 0;
-  font: inherit;
-  cursor: pointer;
-  outline: inherit;
-  background-color: #f44336;
-  color: #fff;
-  box-shadow: 0 0 10px -3px rgba(97, 97, 97, 0.7);
-  border-radius: 5px;
-  border: solid 1px #a4a6a9;
-  cursor: pointer;
-  margin: 1em auto;
-  padding: 1em;
-}
-.module-message-detail .message-content * {
-  background-color: lightgrey;
-  width: 100%;
-  resize: none;
-  padding: 1rem;
-}
-.message-list-and-composer {
-  width: 70%;
-  float: right;
-  display: grid;
-  grid-template-columns: auto;
-  height: calc(100vh - 50px);
-  margin-top: 50px;
-  background-image: var(--chatViewBgImgPath);
-  background-size: cover;
-  background-color: var(--chatViewBg);
-}
-.message-list-and-composer__message-list #message-list {
-  background: #dbdbdb;
-  position: absolute;
-  bottom: 0;
-  overflow: scroll;
-  max-height: 100%;
-  width: 100%;
-  padding: 0 0.5em;
-  top: 52px;
-}
-.message-list-and-composer__message-list
-  #message-list::-webkit-scrollbar-track {
-  background: transparent;
-}
-.message-list-and-composer__message-list ul {
-  list-style: none;
-  min-width: 200px;
-}
-.message-list-and-composer__message-list ul li {
-  margin-bottom: 10px;
-  min-width: 200px;
-}
-.message-list-and-composer__message-list ul li::after {
-  visibility: hidden;
-  display: block;
-  font-size: 0;
-  content: " ";
-  clear: both;
-  height: 0;
-}
-.message-list-and-composer__message-list ul li .info-message {
-  max-width: 550px;
-  font-size: 1rem;
-  padding: 2rem;
-  font-style: normal;
-  white-space: pre-wrap;
-  text-align: left;
-}
-.message {
-  position: relative;
-  display: inline-flex;
-  flex-direction: row;
-  align-items: stretch;
-}
-.message:hover .message-buttons {
-  opacity: 1;
-}
-.message > .author-avatar {
-  align-self: flex-end;
-  bottom: 0px;
-  position: static;
-  margin-right: 8px;
-  user-select: none;
-}
-.message > .author-avatar img {
-  height: 36px;
-  width: 36px;
-  border-radius: 18px;
-  object-fit: cover;
-}
-.message > .author-avatar.default {
-  text-align: center;
-}
-.message > .author-avatar.default > .label {
-  user-select: none;
-  color: var(--avatarLabelColor);
-  top: -121px;
-  left: -10px;
-  border-radius: 50%;
-  width: 36px;
-  height: 36px;
-  font-size: 25px;
-  line-height: 36px;
-}
-.message .message-buttons {
-  position: absolute;
-  top: 5px;
-  right: -4px;
-  display: inline-flex;
-  flex-direction: row;
-  align-items: center;
-  opacity: 0;
-  z-index: 10;
-  user-select: text;
-}
-.message .message-buttons .msg-button {
-  height: 24px;
-  width: 24px;
-  display: inline-block;
-  cursor: pointer;
-}
-.message .message-buttons .msg-button:hover {
-  background-color: var(--messageButtons);
-}
-.message .message-buttons .msg-button.download {
-  -webkit-mask: url("../images/download.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageButtons);
-}
-.message .message-buttons .msg-button.reply {
-  display: none;
-  -webkit-mask: url("../images/reply.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageButtons);
-  user-select: none;
-}
-.message .message-buttons .msg-button.menu {
-  -webkit-mask: url("../images/ellipsis.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageButtons);
-  transform: rotate(90deg);
-  -webkit-mask-position-y: 4px;
-  user-select: none;
-}
-.message .msg-container {
-  position: relative;
-  display: inline-block;
-  border-radius: 16px;
-  padding-right: 12px;
-  padding-left: 12px;
-  padding-top: 10px;
-  padding-bottom: 10px;
-}
-.message .msg-container > .author {
-  display: inline-block;
-  max-width: 40vw;
-  font-size: 13px;
-  font-weight: 300;
-  line-height: 18px;
-  height: 18px;
-  overflow-x: hidden;
-  overflow-y: hidden;
-  white-space: nowrap;
-  text-overflow: ellipsis;
-}
-.message .msg-container .msg-body.msg-body--clickable {
-  cursor: pointer;
-}
-.message .msg-container .msg-body > .text {
-  color: var(--messageText);
-  font-size: 14px;
-  line-height: 18px;
-  text-align: start;
-  overflow-wrap: break-word;
-  word-wrap: break-word;
-  word-break: break-word;
-  white-space: pre-wrap;
-  margin-right: 10px;
-}
-.message .msg-container .msg-body > .text a {
-  text-decoration: underline;
-  color: var(--messageTextLink);
-}
-.message .msg-container .msg-body > .text .double-line-break {
-  height: 28px;
-}
-.message .msg-container .msg-body > .text .line-break {
-  height: 14px;
-}
-.message .msg-container .msg-body > .text .line-break:last-child {
-  height: 0px;
-}
-.message .metadata {
-  margin-top: 10px;
-  margin-bottom: -7px;
-  float: right;
-}
-.message .module-message__img-attachment {
-  object-fit: cover;
-  width: auto;
-  max-width: 100%;
-  min-height: unset;
-}
-.message.incoming {
-  margin-left: 0;
-  margin-right: 32px;
-}
-.message.incoming .metadata:not(.with-image-no-caption) > .padlock-icon {
-  -webkit-mask: url("../images/padlock.svg") no-repeat center;
-  -webkit-mask-size: 125%;
-  background-color: var(--messagePadlockIncomming);
-}
-.message.incoming .metadata:not(.with-image-no-caption) > .location-icon {
-  -webkit-mask: url("../images/map-marker.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messagePadlockIncomming);
-}
-.message.incoming .metadata:not(.with-image-no-caption) > .date {
-  color: var(--messageMetadataIncomming);
-}
-.message.incoming .msg-container {
-  background-color: var(--messageIncommingBg);
-}
-.message.incoming .msg-container,
-.message.incoming .msg-container .message-attachment-media {
-  border-bottom-left-radius: 1px;
-}
-.message.outgoing {
-  float: right;
-  margin-right: 0;
-  margin-left: 32px;
-}
-.message.outgoing .metadata > .date {
-  color: var(--messageOutgoingStatusColor);
-}
-.message.outgoing .metadata > .padlock-icon {
-  -webkit-mask: url("../images/padlock.svg") no-repeat center;
-  -webkit-mask-size: 125%;
-  background-color: var(--messagePadlockOutgoing);
-}
-.message.outgoing .metadata > .location-icon {
-  -webkit-mask: url("../images/map-marker.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messagePadlockOutgoing);
-}
-.message.outgoing .metadata > .status-icon.read,
-.message.outgoing .metadata > .status-icon.delivered {
-  background-color: var(--messageOutgoingStatusColor);
-}
-.message.outgoing .msg-container {
-  background-color: var(--messageOutgoingBg);
-}
-.message.outgoing .msg-container,
-.message.outgoing .msg-container .message-attachment-media {
-  border-bottom-right-radius: 1px;
-}
-.message.type-sticker .msg-container {
-  background-color: transparent !important;
-}
-.message.type-sticker .message-attachment-media {
-  background-color: transparent;
-}
-.message.type-sticker .message-attachment-media > .attachment-content {
-  margin-bottom: 20px;
-}
-.message.type-sticker .metadata {
-  float: right;
-  padding: 4px 10px 1px 10px;
-  margin-bottom: -7px;
-  background-color: #01010159;
-  border-radius: 4px;
-  color: black;
-  font-weight: bold;
-}
-.message.type-sticker .metadata > .date {
-  font-size: 11px;
-  color: white;
-}
-.message.type-sticker .metadata > .padlock-icon {
-  -webkit-mask: url("../images/padlock.svg") no-repeat center;
-  -webkit-mask-size: 125%;
-  background-color: #fff;
-}
-.message.type-sticker .metadata > .location-icon {
-  -webkit-mask: url("../images/map-marker.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: #fff;
-}
-.message.type-sticker .status-icon.read,
-.message.type-sticker .status-icon.delivered {
-  background-color: white;
-}
-.message.type-sticker:hover .msg-button.menu {
-  background-color: white;
-}
-.message.type-sticker:hover .react-contextmenu-wrapper {
-  background-color: #2525258f;
-  border-radius: 4px;
-}
-.message.error.incoming .text {
-  font-style: italic;
-}
-.message.forwarded .forwarded-indicator {
-  font-weight: bold;
-  font-size: 0.9em;
-  margin-bottom: 3px;
-  opacity: 0.86;
-}
-.message.forwarded .message-attachment-media {
-  border-top-left-radius: 0;
-  border-top-right-radius: 0;
-  margin-top: 0;
-}
-.setupMessage .message .text {
-  color: var(--setupMessageText);
-}
-.hide-on-small {
-  display: initial;
-}
-@media (max-width: 800px) {
-  .hide-on-small {
-    display: none;
-  }
-}
-@media (min-width: 800px) and (max-width: 925px) {
-  .message {
-    max-width: 374px;
-  }
-  .message.incoming {
-    margin-right: auto;
-  }
-  .message.outgoing {
-    margin-left: auto;
-  }
-}
-@media (min-width: 926px) {
-  .message {
-    max-width: 66%;
-  }
-  .message.incoming {
-    margin-right: auto;
-  }
-  .message.outgoing {
-    margin-left: auto;
-  }
-}
-.metadata {
-  display: flex;
-  flex-direction: row;
-  align-items: center;
-  margin-top: 3px;
-  margin-bottom: -3px;
-}
-.metadata.with-image-no-caption {
-  position: absolute;
-  right: 5px;
-  bottom: 5px;
-  float: right;
-  padding: 4px 10px 1px 10px;
-  margin: 0;
-  background-color: #0000008f;
-  border-radius: 4px;
-  font-weight: bold;
-}
-.metadata.with-image-no-caption > .date {
-  color: white;
-}
-.metadata.with-image-no-caption > .padlock-icon {
-  -webkit-mask: url("../images/padlock.svg") no-repeat center;
-  -webkit-mask-size: 125%;
-  background-color: #fff;
-}
-.metadata.with-image-no-caption .status-icon.sending {
-  background-color: white;
-}
-.metadata > .status-icon {
-  margin-bottom: 2px;
-}
-.metadata > .username {
-  margin-right: 10px;
-}
-.metadata > .date {
-  font-size: 11.5px;
-  line-height: 16px;
-  letter-spacing: 0.3px;
-  color: var(--messageMetadataDate);
-  text-transform: uppercase;
-}
-.metadata > .spacer {
-  flex-grow: 1;
-}
-.metadata > .padlock-icon,
-.metadata > .location-icon {
-  width: 12px;
-  height: 12px;
-  display: inline-block;
-  margin-right: 2px;
-  margin-bottom: 3px;
-}
-.metadata > .location-icon {
-  margin-bottom: 0;
-}
-@keyframes __status-icon--spinning {
-  100% {
-    -webkit-transform: rotate(360deg);
-    transform: rotate(360deg);
-  }
-}
-.status-icon {
-  width: 18px;
-  height: 12px;
-  display: inline-block;
-  margin-left: 2px;
-}
-.status-icon.sending {
-  -webkit-mask: url("../images/sending.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageStatusIconSending);
-  animation: __status-icon--spinning 4s linear infinite;
-  width: 12px;
-  margin-left: 8px;
-}
-.status-icon.delivered {
-  -webkit-mask: url("../images/sent.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageStatusIcon);
-}
-.status-icon.read {
-  -webkit-mask: url("../images/read.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--messageStatusIcon);
-}
-.status-icon.error {
-  -webkit-mask: url("../images/error.svg") no-repeat center;
-  -webkit-mask-size: 100%;
-  background-color: var(--errorColor);
-  width: 12px;
-  margin-left: 8px;
-}

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/
+rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/

deltachat-ffi/Cargo.toml

@@ -19,8 +19,9 @@ deltachat = { path = "../", default-features = false }
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
-failure = "0.1.6"
 serde_json = "1.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
 default = ["vendored", "nightly"]

deltachat-ffi/deltachat.h

@@ -4505,14 +4505,22 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 
 /**
- * This event is sent out to the inviter when a joiner successfully joined a group.
+ * This event is sent for each member that gets added to a (verified or unverified) chat. 
  *
  * @param data1 (int) chat_id
  * @param data2 (int) contact_id
  * @return 0
  */
-#define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062
+#define DC_EVENT_MEMBER_ADDED 2062
 
+/**
+ * This event is sent for each member that gets removed from a (verified or unverified) chat. 
+ *
+ * @param data1 (int) chat_id
+ * @param data2 (int) contact_id
+ * @return 0
+ */
+#define DC_EVENT_MEMBER_REMOVED 2063
 
 /**
  * @}

deltachat-ffi/src/lib.rs

@@ -27,7 +27,7 @@ use num_traits::{FromPrimitive, ToPrimitive};
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
-use deltachat::contact::Contact;
+use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
@@ -120,17 +120,17 @@ impl ContextWrapper {
     /// Unlock the context and execute a closure with it.
     ///
     /// This is like [ContextWrapper::with_inner] but uses
-    /// [failure::Error] as error type.  This allows you to write a
+    /// [anyhow::Error] as error type.  This allows you to write a
     /// closure which could produce many errors, use the `?` operator
     /// to return them and handle them all as the return of this call.
-    fn try_inner<T, F>(&self, ctxfn: F) -> Result<T, failure::Error>
+    fn try_inner<T, F>(&self, ctxfn: F) -> Result<T, anyhow::Error>
     where
-        F: FnOnce(&Context) -> Result<T, failure::Error>,
+        F: FnOnce(&Context) -> Result<T, anyhow::Error>,
     {
         let guard = self.inner.read().unwrap();
         match guard.as_ref() {
             Some(ref ctx) => ctxfn(ctx),
-            None => Err(failure::err_msg("context not open")),
+            None => Err(anyhow::format_err!("context not open")),
         }
     }
 
@@ -199,6 +199,14 @@ impl ContextWrapper {
                 Event::SecurejoinMemberAdded {
                     chat_id,
                     contact_id,
+                }
+                | Event::MemberAdded {
+                    chat_id,
+                    contact_id,
+                }
+                | Event::MemberRemoved {
+                    chat_id,
+                    contact_id,
                 } => {
                     ffi_cb(
                         self,
@@ -1657,7 +1665,9 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, to_string_lossy(addr)))
+        .with_inner(|ctx| {
+            Contact::lookup_id_by_addr(ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
+        })
         .unwrap_or(0)
 }
 

deltachat-ffi/src/string.rs

@@ -1,4 +1,3 @@
-use failure::Fail;
 use std::ffi::{CStr, CString};
 use std::ptr;
 
@@ -31,13 +30,13 @@ unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
 }
 
 /// Error type for the [OsStrExt] trait
-#[derive(Debug, Fail, PartialEq)]
+#[derive(Debug, PartialEq, thiserror::Error)]
 pub(crate) enum CStringError {
     /// The string contains an interior null byte
-    #[fail(display = "String contains an interior null byte")]
+    #[error("String contains an interior null byte")]
     InteriorNullByte,
     /// The string is not valid Unicode
-    #[fail(display = "String is not valid unicode")]
+    #[error("String is not valid unicode")]
     NotUnicode,
 }
 

deltachat_derive/src/lib.rs

@@ -3,7 +3,6 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
-use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to

draft/mailing_list.rst

@@ -0,0 +1,19 @@
+2. Mailing lists are now only detected by the ListId header, because how should DC sort mailing lists if the ListId header is unset and only the precedence header says it's a mailing list (?). We could just hide these emails (with "junk" or "list" precedence), as we did before, if there is a reason to do so. 
+
+   I do not actually understand why such emails were hidden in the first place, though; if there is an automatic answer stating that someone is out of office or if an email could not be delivered (these are the usual reasons why such emails are sent), I would want to know about this as a user.
+   
+  Björn: to the ListId-header: it is already an improvement to use only that and to keep ignoring mails with Precedence-header. historically, i ignored all these mails to reduce noise that is created from them (eg. contacts should not be added to the suggestion list...) and to be able to concentrate on other things first.
+   
+   
+3. for `List-Id: foo <bla>`, bla now is the id and foo is the name. For GitHub and GitLab notifications this is bad because all notifications will go into one chat. Maybe we should instead take the "in-reply-to" header to find out what messages belong together. For normal mailing lists, of course, this will create a second chat if someone does not use the "Reply" button to write to that mailing list.
+
+  Björn: well, having all notifications in one chat is not that bad. i would just follow the guidelines for now, use the ID and not the Name.
+
+
+4. Currently a mailing list is shown as an empty group (ChatType `Group`) ("0 members"). 
+
+   Maybe we should change the ChatType to `Single` because this way, the UI would fit better. Disadvantage: We can't show the different senders of the messages. 
+   
+   Or we introduce a `ChatType` `MailingList`. Very big disadvantage: We would have to adapt all UI project and it would be nice if we could keep the changes within the core.
+
+5. Contacts from mailings lists stay "unknown" now and are not shown in the contacts suggestion list..

draft/mailing_list_current_state.rst

@@ -0,0 +1,13 @@
+1. Mailing lists with Precedence-header and without List-Id are shown as normal messages.
+   
+2. for `List-Id: foo <bla>`, bla now is the id and foo is the name. If foo is not present, bla is taken as the name as well. For GitHub and GitLab notifications all notifications will go into one chat. To distinguish, all subjects are shown in mailing lists.
+
+3. Currently a mailing list is shown as a group with SELF and the List-Id. I could not find any stable way to get a mail address that could be called the "mailing list address". To get this to work, I disabled the may_be_valid_addr check. (to be discussed...)
+
+4. Contacts from mailings lists stay "unknown" and are not shown in the contacts suggestion list.
+
+5. In general, only the From: address is taken as the author. If the domain matches with the List-Id domain (like `deltachat.github.com` and `Hocuri <notifications@notification.github.com>`) then the display name is added to the email address -> `Hocuri - notifications@notification.github.com`. This is not really nice, but as the UIs distinguish bettween contacts only based on the address this was the only way I could find without changing the API. 
+
+   TODO: Prevent the user from sending emails to such contacts (like Hocuri - notifications@notification.github.com)
+
+6. You can't send to mailing lists.

draft/mailing_list_managers.md

@@ -0,0 +1,71 @@
+# Mailing list integration of Delta Chat
+
+A collection of information to help with the integration of mailing lists into Delta Chat.
+
+## Chat name
+
+How should the chat be titled?
+
+It could be taken from the email header `List-Id`:
+
+### Mailman 
+
+* Schema: `${list-description} <${list-local-part}.${list-domain}>`
+* $list-description could be many words, should be truncated. Could also be empty, though.
+
+### Schleuder
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Could be absent: <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L357-358>.
+
+
+### Sympa
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Always present <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/Spindle/TransformOutgoing.pm#L110-L118>, <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/List.pm#L6832-L6833>.
+
+
+## Sender name
+
+What's the name of the person that actually sent the message?
+
+Some MLM change the sender information to avoid problems with DMARC.
+
+### Mailman
+
+* Since v2.1.16 the `From` depends on the list's configuration and possibly on the sender-domain's DMARC-configuration — i.e. messages from the same list can arrive one of the Variants A-D!
+* Documentation of config options:
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-544>,
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-163>,
+ * <https://wiki.list.org/DEV/DMARC>
+* Variant A: `From` is unchanged.
+* Variant B. `From` is mangled like this: `${sender-name} via ${list-name} <${list-addr-spec}>`. The original sender is put into `Reply-To`.
+* Variant C. `From` is mangled like this: `${sender-name} <${encoded-sender-addr-spec}@${list-domain}>`. The original sender is put into `Reply-To`.
+* Variant D: `From` is set to ${list-name-addr}, the originally sent message is included as mime-part.
+* Variant E: `From` is set to ${list-name-addr}, original sender information is removed.
+
+### Schleuder
+
+* Visible only in mime-body (which is possibly encrypted), or not at all.
+* The first `text/plain` mime-part may include the information, as taken from the original message: `From: ${sender-name-addr}`.
+
+### Sympa
+* Depends on the list's configuration.
+* Documentation:
+ * <https://sympa-community.github.io/manual/customize/dmarc-protection.html>,
+ * <https://sympa-community.github.io/gpldoc/man/sympa.conf.5.html#dmarc-protection>.
+* Variant A: `From` is unchanged.
+* Variant B: `From` is mangled like Mailman Variant B, but the original sender information is put into `X-Original-From`.
+
+
+## Autocrypt
+
+### Mailman
+* Not supported.
+
+### Schleuder
+* A list's key is included in sent messages (as of version 3.5.0) (optional, by default active): <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L349-355>.
+* Incoming keys are not yet looked at (that feature is planned: <https://0xacab.org/schleuder/schleuder/issues/435>).
+
+### Sympa
+* Not supported.

examples/repl/cmdline.rs

@@ -1,7 +1,7 @@
 use std::path::Path;
 use std::str::FromStr;
-use std::time::{SystemTime, UNIX_EPOCH};
 
+use anyhow::{bail, ensure};
 use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
@@ -10,7 +10,6 @@ use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
 use deltachat::error::Error;
-use deltachat::export_chat::{export_chat, pack_exported_chat};
 use deltachat::imex::*;
 use deltachat::job::*;
 use deltachat::location;
@@ -93,7 +92,7 @@ fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
     1
 }
 
-fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), Error> {
+fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
     let data = dc_read_file(context, filename)?;
 
     if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) {
@@ -299,7 +298,7 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
+pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
     let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
     let mut sel_chat = if !chat_id.is_unset() {
         Chat::load_from_db(context, chat_id).ok()
@@ -376,7 +375,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
                  delchat <chat-id>\n\
-                 export-chat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
@@ -871,24 +869,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(context)?;
         }
-        "export-chat" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            let res = export_chat(context, chat_id);
-            println!("{:?}", res);
-            let timestamp = SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .unwrap()
-                .as_secs();
-            let destination_raw = context.get_blobdir().join(format!(
-                "exported_{}_{}.zip",
-                chat_id.to_u32(),
-                timestamp
-            ));
-            let destination = destination_raw.to_str().unwrap();
-            let pack_res = pack_exported_chat(context, res, destination);
-            println!("{:?} - destination: {}", pack_res, destination);
-        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);

examples/repl/main.rs

@@ -7,8 +7,6 @@
 #[macro_use]
 extern crate deltachat;
 #[macro_use]
-extern crate failure;
-#[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate rusqlite;
@@ -20,6 +18,7 @@ use std::process::Command;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 
+use anyhow::{bail, Error};
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
@@ -262,7 +261,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 27] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
@@ -289,7 +288,6 @@ const CHAT_COMMANDS: [&str; 27] = [
     "pin",
     "unpin",
     "delchat",
-    "export-chat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
@@ -371,10 +369,10 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
+fn main_0(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        return Err(format_err!("No db-name specified"));
+        bail!("No db-name specified");
     }
     let context = Context::new(
         Box::new(receive_event),
@@ -444,7 +442,7 @@ enum ExitResult {
     Exit,
 }
 
-fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
@@ -527,7 +525,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), failure::Error> {
+pub fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args: Vec<String> = std::env::args().collect();

python/CHANGELOG

@@ -1,3 +1,15 @@
+0.900.0 (DRAFT)
+---------------
+
+- refactored internals to use plugin-approach 
+
+- introduced PerAccount and Global hooks that plugins can implement 
+
+- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
+
+- introduced two documented examples for an echo and a group-membership
+  tracking plugin. 
+
 0.800.0
 -------
 

python/doc/api.rst

@@ -2,10 +2,6 @@
 high level API reference
 ========================
 
-.. note::
-
-    This API is work in progress and may change in versions prior to 1.0.
-
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
 - :class:`deltachat.contact.Contact`

python/doc/capi.rst

@@ -1,7 +0,0 @@
-
-C deltachat interface
-=====================
-
-See :doc:`lapi` for accessing many of the below functions
-through the ``deltachat.capi.lib`` namespace.
-

python/doc/examples.rst

@@ -1,37 +1,60 @@
 
-
 examples
 ========
 
-
-Playing around on the commandline
-----------------------------------
-
 Once you have :doc:`installed deltachat bindings <install>`
-you can start playing from the python interpreter commandline.
-For example you can type ``python`` and then::
+you need email/password credentials for an IMAP/SMTP account.
+Delta Chat developers and the CI system use a special URL to create
+temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
 
-    # instantiate and configure deltachat account
-    import deltachat
-    ac = deltachat.Account("/tmp/db")
+Receiving a Chat message from the command line
+----------------------------------------------
 
-    # start configuration activity and smtp/imap threads
-    ac.start_threads()
-    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
+Here is a simple bot that:
 
-    # create a contact and send a message
-    contact = ac.create_contact("someother@email.address")
-    chat = ac.create_chat_by_contact(contact)
-    chat.send_text("hi from the python interpreter command line")
+- receives a message and sends back ("echoes") a message
 
-Checkout our :doc:`api` for the various high-level things you can do
-to send/receive messages, create contacts and chats.
+- terminates the bot if the message `/quit` is sent
 
+.. include:: ../examples/echo_and_quit.py
+    :literal:
 
-Looking at a real example
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    $ cd examples
+    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
+
+While this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Track member additions and removals in a group
+----------------------------------------------
+
+Here is a simple bot that:
+
+- echoes messages sent to it
+
+- tracks if configuration completed
+
+- tracks member additions and removals for all chat groups
+
+.. include:: ../examples/group_tracking.py
+    :literal:
+
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
+
+When this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Writing bots for real
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a real-life example of Python bindings usage.
-
+contains a little framework for writing deltachat bots in Python.
 

python/doc/index.rst

@@ -4,8 +4,9 @@ deltachat python bindings
 The ``deltachat`` Python package provides two layers of bindings for the
 core Rust-library of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core which aims
-  to be memory safe and thoroughly tested through continous tox/pytest runs.
+- :doc:`api` is a high level interface to deltachat-core.
+
+- :doc:`plugins` is a brief introduction into implementing plugin hooks.
 
 - :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
   <https://github.com/deltachat/deltachat-core-rust>`_.
@@ -28,6 +29,7 @@ getting started
    changelog
    api
    lapi
+   plugins
 
 ..
     Indices and tables

python/doc/plugins.rst

@@ -0,0 +1,38 @@
+
+Implementing Plugin Hooks
+==========================
+
+The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
+for managing global and per-account plugin registration, and performing
+hook calls. There are two kinds of plugins:
+
+- Global plugins that are active for all accounts; they can implement
+  hooks at account-creation and account-shutdown time.
+
+- Account plugins that are only active during the lifetime of a
+  single Account instance.
+
+
+Registering a plugin
+--------------------
+
+.. autofunction:: deltachat.register_global_plugin
+    :noindex:
+
+.. automethod:: deltachat.account.Account.add_account_plugin
+    :noindex:
+
+
+Per-Account Hook specifications
+-------------------------------
+
+.. autoclass:: deltachat.hookspec.PerAccount
+    :members:
+
+
+Global Hook specifications
+--------------------------
+
+.. autoclass:: deltachat.hookspec.Global
+    :members:
+

python/examples/echo_and_quit.py

@@ -0,0 +1,30 @@
+
+# content of echo_and_quit.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class EchoPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_message_delivered(self, message):
+        print("ac_message_delivered", message)
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/group_tracking.py

@@ -0,0 +1,40 @@
+
+# content of group_tracking.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class GroupTrackingPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        print("*** ac_configure_completed:", success)
+
+    @account_hookimpl
+    def ac_member_added(self, chat, contact):
+        print("*** ac_member_added", contact.addr, "from", chat)
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_removed(self, chat, contact):
+        print("*** ac_member_removed", contact.addr, "from", chat)
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/test_examples.py

@@ -0,0 +1,72 @@
+
+import pytest
+import py
+import echo_and_quit
+import group_tracking
+from deltachat.eventlogger import FFIEventLogger
+
+
+@pytest.fixture(scope='session')
+def datadir():
+    """The py.path.local object of the test-data/ directory."""
+    for path in reversed(py.path.local(__file__).parts()):
+        datadir = path.join('test-data')
+        if datadir.isdir():
+            return datadir
+    else:
+        pytest.skip('test-data directory not found')
+
+
+def test_echo_quit_plugin(acfactory):
+    botproc = acfactory.run_bot_process(echo_and_quit)
+
+    ac1 = acfactory.get_one_online_account()
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+    assert reply.chat == ch1
+    ch1.send_text("/quit")
+    botproc.wait()
+
+
+def test_group_tracking_plugin(acfactory, lp):
+    lp.sec("creating one group-tracking bot and two temp accounts")
+    botproc = acfactory.run_bot_process(group_tracking)
+
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
+
+    botproc.fnmatch_lines("""
+        *ac_configure_completed: True*
+    """)
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+
+    lp.sec("creating bot test group with all three")
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch = ac1.create_group_chat("bot test group")
+    ch.add_contact(bot_contact)
+    ch.send_text("hello")
+
+    botproc.fnmatch_lines("""
+        *ac_member_added {}*
+    """.format(ac1.get_config("addr")))
+
+    lp.sec("adding third member {}".format(ac2.get_config("addr")))
+    contact3 = ac1.create_contact(ac2.get_config("addr"))
+    ch.add_contact(contact3)
+
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+
+    lp.sec("now looking at what the bot received")
+    botproc.fnmatch_lines("""
+        *ac_member_added {}*
+    """.format(contact3.addr))
+
+    lp.sec("contact successfully added, now removing")
+    ch.remove_contact(contact3)
+    botproc.fnmatch_lines("""
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/setup.py

@@ -22,6 +22,11 @@ def main():
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
+        entry_points = {
+            'pytest11': [
+                'deltachat.testplugin = deltachat.testplugin',
+            ],
+        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,6 +1,13 @@
-from deltachat import capi, const
-from deltachat.capi import ffi
-from deltachat.account import Account  # noqa
+import sys
+
+from . import capi, const, hookspec
+from .capi import ffi
+from .account import Account  # noqa
+from .message import Message  # noqa
+from .contact import Contact  # noqa
+from .chat import Chat        # noqa
+from .hookspec import account_hookimpl, global_hookimpl  # noqa
+from . import eventlogger
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -74,3 +81,62 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
+
+
+def register_global_plugin(plugin):
+    """ Register a global plugin which implements one or more
+    of the :class:`deltachat.hookspec.Global` hooks.
+    """
+    gm = hookspec.Global._get_plugin_manager()
+    gm.register(plugin)
+    gm.check_pending()
+
+
+def unregister_global_plugin(plugin):
+    gm = hookspec.Global._get_plugin_manager()
+    gm.unregister(plugin)
+
+
+register_global_plugin(eventlogger)
+
+
+def run_cmdline(argv=None, account_plugins=None):
+    """ Run a simple default command line app, registering the specified
+    account plugins. """
+    import argparse
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
+    parser.add_argument("db", action="store", help="database file")
+    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
+    parser.add_argument("--email", action="store", help="email address")
+    parser.add_argument("--password", action="store", help="password")
+
+    args = parser.parse_args(argv[1:])
+
+    ac = Account(args.db)
+
+    if args.show_ffi:
+        log = eventlogger.FFIEventLogger(ac, "bot")
+        ac.add_account_plugin(log)
+
+    if not ac.is_configured():
+        assert args.email and args.password, (
+            "you must specify --email and --password once to configure this database/account"
+        )
+        ac.set_config("addr", args.email)
+        ac.set_config("mail_pw", args.password)
+        ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
+        ac.set_config("sentbox_watch", "0")
+
+    for plugin in account_plugins or []:
+        ac.add_account_plugin(plugin)
+
+    # start IO threads and configure if neccessary
+    ac.start()
+
+    print("{}: waiting for message".format(ac.get_config("addr")))
+
+    ac.wait_shutdown()

python/src/deltachat/account.py

@@ -2,12 +2,12 @@
 
 from __future__ import print_function
 import atexit
-import threading
+from contextlib import contextmanager
+from email.utils import parseaddr
+import queue
+from threading import Event
 import os
-import time
 from array import array
-from queue import Queue
-
 import deltachat
 from . import const
 from .capi import ffi, lib
@@ -15,8 +15,12 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
-from .eventlogger import EventLogger
-from .hookspec import get_plugin_manager, hookimpl
+from .tracker import ImexTracker
+from . import hookspec, iothreads
+
+
+class MissingCredentials(ValueError):
+    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
@@ -24,45 +28,54 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, os_name=None, debug=True):
+    MissingCredentials = MissingCredentials
+
+    def __init__(self, db_path, os_name=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
         :param os_name: this will be put to the X-Mailer header in outgoing messages
-        :param debug: turn on debug logging for events.
         """
+        # initialize per-account plugin system
+        self._pm = hookspec.PerAccount._make_plugin_manager()
+        self.add_account_plugin(self)
+
         self._dc_context = ffi.gc(
             lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
-        self._evlogger = EventLogger(self, logid, debug)
-        self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
 
-        # register event call back and initialize plugin system
-        def _ll_event(ctx, evt_name, data1, data2):
-            assert ctx == self._dc_context
-            self.pluggy.hook.process_low_level_event(
-                account=self, event_name=evt_name, data1=data1, data2=data2
-            )
+        hook = hookspec.Global._get_plugin_manager().hook
 
-        self.pluggy = get_plugin_manager()
-        self.pluggy.register(self._evlogger)
-        deltachat.set_context_callback(self._dc_context, _ll_event)
+        self._threads = iothreads.IOThreads(self)
+        self._hook_event_queue = queue.Queue()
+        self._in_use_iter_events = False
+        self._shutdown_event = Event()
 
         # open database
+        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
         atexit.register(self.shutdown)
+        hook.dc_account_init(account=self)
+
+    @hookspec.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        name, kwargs = self._map_ffi_event(ffi_event)
+        if name is not None:
+            ev = HookEvent(self, name=name, kwargs=kwargs)
+            self._hook_event_queue.put(ev)
 
     # def __del__(self):
     #    self.shutdown()
 
+    def ac_log_line(self, msg):
+        self._pm.hook.ac_log_line(message=msg)
+
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -134,16 +147,15 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def configure(self, **kwargs):
-        """ set config values and configure this account.
+    def update_config(self, kwargs):
+        """ update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for name, value in kwargs.items():
-            self.set_config(name, value)
-        lib.dc_configure(self._dc_context)
+        for key, value in kwargs.items():
+            self.set_config(key, str(value))
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -151,7 +163,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return lib.dc_is_configured(self._dc_context)
+        return bool(lib.dc_is_configured(self._dc_context))
 
     def set_avatar(self, img_path):
         """Set self avatar.
@@ -202,8 +214,7 @@ class Account(object):
 
         :returns: :class:`deltachat.contact.Contact`
         """
-        self.check_is_configured()
-        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
+        return Contact(self, const.DC_CONTACT_ID_SELF)
 
     def create_contact(self, email, name=None):
         """ create a (new) Contact. If there already is a Contact
@@ -214,11 +225,14 @@ class Account(object):
         :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        name = as_dc_charpointer(name)
-        email = as_dc_charpointer(email)
-        contact_id = lib.dc_create_contact(self._dc_context, name, email)
+        realname, addr = parseaddr(email)
+        if name:
+            realname = name
+        realname = as_dc_charpointer(realname)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-        return Contact(self._dc_context, contact_id)
+        return Contact(self, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -250,7 +264,7 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
@@ -272,6 +286,9 @@ class Account(object):
         """ create or get an existing chat object for the
         the specified message.
 
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
         :param message: messsage id or message instance.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
@@ -324,6 +341,13 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -368,13 +392,18 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory. """
+        """ export public and private keys to the specified directory.
+
+        Note that the account does not have to be started.
+        """
         return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
+
+        Note that the account does not have to be started.
         """
         export_files = self._export(path, 11)
         if len(export_files) != 1:
@@ -382,7 +411,7 @@ class Account(object):
         return export_files[0]
 
     def _export(self, path, imex_cmd):
-        with ImexTracker(self) as imex_tracker:
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             if not self._threads.is_started():
                 lib.dc_perform_imap_jobs(self._dc_context)
@@ -392,6 +421,8 @@ class Account(object):
         """ Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
+
+        Note that the account does not have to be started.
         """
         self._import(path, imex_cmd=2)
 
@@ -404,7 +435,7 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        with ImexTracker(self) as imex_tracker:
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
             lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             if not self._threads.is_started():
                 lib.dc_perform_imap_jobs(self._dc_context)
@@ -473,46 +504,6 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def stop_ongoing(self):
-        lib.dc_stop_ongoing_process(self._dc_context)
-
-    #
-    # meta API for start/stop and event based processing
-    #
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.get_message_by_id(ev[2])
-
-    def start_threads(self, mvbox=False, sentbox=False):
-        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
-
-        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
-        :returns: None
-        """
-        if not self.is_configured():
-            self.configure()
-        self._threads.start(mvbox=mvbox, sentbox=sentbox)
-
-    def stop_threads(self, wait=True):
-        """ stop IMAP/SMTP threads. """
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=wait)
-
-    def shutdown(self, wait=True):
-        """ stop threads and close and remove underlying dc_context and callbacks. """
-        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
-            # print("SHUTDOWN", self)
-            self.stop_threads(wait=False)
-            lib.dc_close(self._dc_context)
-            self.stop_threads(wait=wait)  # to wait for threads
-            deltachat.clear_context_callback(self._dc_context)
-            del self._dc_context
-            atexit.unregister(self.shutdown)
-        self.pluggy.unregister(self._evlogger)
-
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
@@ -527,120 +518,123 @@ class Account(object):
         if dc_res == 0:
             raise ValueError("no chat is streaming locations")
 
+    #
+    # meta API for start/stop and event based processing
+    #
 
-class ImexTracker:
-    def __init__(self, account):
-        self._imex_events = Queue()
-        self.account = account
+    def add_account_plugin(self, plugin, name=None):
+        """ add an account plugin which implements one or more of
+        the :class:`deltachat.hookspec.PerAccount` hooks.
+        """
+        self._pm.register(plugin, name=name)
+        self._pm.check_pending()
+        return plugin
 
-    def __enter__(self):
-        self.account.pluggy.register(self)
-        return self
+    @contextmanager
+    def temp_plugin(self, plugin):
+        """ run a with-block with the given plugin temporarily registered. """
+        self._pm.register(plugin)
+        yield plugin
+        self._pm.unregister(plugin)
 
-    def __exit__(self, *args):
-        self.account.pluggy.unregister(self)
+    def stop_ongoing(self):
+        """ Stop ongoing securejoin, configuration or other core jobs. """
+        lib.dc_stop_ongoing_process(self._dc_context)
 
-    @hookimpl
-    def process_low_level_event(self, account, event_name, data1, data2):
-        # there could be multiple accounts instantiated
-        if self.account is not account:
+    def start(self, callback_thread=True):
+        """ start this account (activate imap/smtp threads etc.)
+        and return immediately.
+
+        If this account is not configured, an internal configuration
+        job will be scheduled if config values are sufficiently specified.
+
+        You may call `wait_shutdown` or `shutdown` after the
+        account is in started mode.
+
+        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
+
+        :returns: None
+        """
+        if not self.is_configured():
+            if not self.get_config("addr") or not self.get_config("mail_pw"):
+                raise MissingCredentials("addr or mail_pwd not set in config")
+            lib.dc_configure(self._dc_context)
+        self._threads.start(callback_thread=callback_thread)
+
+    def wait_shutdown(self):
+        """ wait until shutdown of this account has completed. """
+        self._shutdown_event.wait()
+
+    def shutdown(self, wait=True):
+        """ shutdown account, stop threads and close and remove
+        underlying dc_context and callbacks. """
+        dc_context = self._dc_context
+        if dc_context is None:
             return
-        if event_name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(data1)
-        elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(data1)
 
-    def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
-        files_written = []
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif ev == 0:
-                raise ValueError("export failed, exp-files: {}".format(files_written))
-            elif ev == 1000:
-                return files_written
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=False)
+        lib.dc_close(dc_context)
+        self._hook_event_queue.put(None)
+        self._threads.stop(wait=wait)  # to wait for threads
+        self._dc_context = None
+        atexit.unregister(self.shutdown)
+        self._shutdown_event.set()
+        hook = hookspec.Global._get_plugin_manager().hook
+        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
 
+    def _handle_current_events(self):
+        """ handle all currently queued events and then return. """
+        while 1:
+            try:
+                event = self._hook_event_queue.get(block=False)
+            except queue.Empty:
+                break
+            else:
+                event.call_hook()
 
-class IOThreads:
-    def __init__(self, dc_context, log_event=lambda *args: None):
-        self._dc_context = dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-        self._log_event = log_event
+    def iter_events(self, timeout=None):
+        """ yield hook events until shutdown.
 
-    def is_started(self):
-        return len(self._name2thread) > 0
+        It is not allowed to call iter_events() from multiple threads.
+        """
+        if self._in_use_iter_events:
+            raise RuntimeError("can only call iter_events() from one thread")
+        self._in_use_iter_events = True
+        while 1:
+            event = self._hook_event_queue.get(timeout=timeout)
+            if event is None:
+                break
+            yield event
 
-    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
-        assert not self.is_started()
-        if imap:
-            self._start_one_thread("inbox", self.imap_thread_run)
-        if mvbox:
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-        if sentbox:
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-        if smtp:
-            self._start_one_thread("smtp", self.smtp_thread_run)
-
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
-
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-
-        # Workaround for a race condition. Make sure that thread is
-        # not in between checking for quitflag and entering idle.
-        time.sleep(0.5)
-
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        lib.dc_interrupt_mvbox_idle(self._dc_context)
-        lib.dc_interrupt_sentbox_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                thread.join()
-
-    def imap_thread_run(self):
-        self._log_event("py-bindings-info", 0, "INBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_imap_jobs(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_imap_fetch(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_imap_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
-
-    def mvbox_thread_run(self):
-        self._log_event("py-bindings-info", 0, "MVBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_mvbox_jobs(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_mvbox_fetch(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_mvbox_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED")
-
-    def sentbox_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SENTBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_sentbox_jobs(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_sentbox_fetch(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_sentbox_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED")
-
-    def smtp_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_smtp_jobs(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_smtp_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                return "ac_configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "ac_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = self.get_message_by_id(ffi_event.data2)
+                if msg.is_in_fresh():
+                    return "ac_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_MEMBER_ADDED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_added", dict(chat=chat, contact=contact)
+        elif name == "DC_EVENT_MEMBER_REMOVED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_removed", dict(chat=chat, contact=contact)
+        return None, {}
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -667,3 +661,17 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
+
+
+class HookEvent:
+    def __init__(self, account, name, kwargs):
+        assert hasattr(account._pm.hook, name), name
+        self.account = account
+        self.name = name
+        self.kwargs = kwargs
+
+    def call_hook(self):
+        hook = getattr(self.account._pm.hook, self.name, None)
+        if hook is None:
+            raise ValueError("event_name {} unknown".format(self.name))
+        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -51,6 +51,16 @@ class Chat(object):
 
     # ------  chat status/metadata API ------------------------------
 
+    def is_group(self):
+        """ return true if this chat is a group chat.
+
+        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
+        """
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
+
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -129,7 +139,7 @@ class Chat(object):
         return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
     def get_type(self):
-        """ return type of this chat.
+        """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -353,7 +363,7 @@ class Chat(object):
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self._dc_context, id))
+            dc_array, lambda id: Contact(self.account, id))
         )
 
     def set_profile_image(self, img_path):

python/src/deltachat/const.py

@@ -99,6 +99,8 @@ DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
 DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
+DC_EVENT_MEMBER_ADDED = 2063
+DC_EVENT_MEMBER_REMOVED = 2064
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091

python/src/deltachat/contact.py

@@ -10,8 +10,9 @@ class Contact(object):
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
-    def __init__(self, dc_context, id):
-        self._dc_context = dc_context
+    def __init__(self, account, id):
+        self.account = account
+        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
@@ -57,3 +58,7 @@ class Contact(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
+
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/eventlogger.py

@@ -1,28 +1,88 @@
+import deltachat
 import threading
-import re
 import time
+import re
 from queue import Queue, Empty
-from .hookspec import hookimpl
+from .hookspec import account_hookimpl, global_hookimpl
 
 
-class EventLogger:
+@global_hookimpl
+def dc_account_init(account):
+    # send all FFI events for this account to a plugin hook
+    def _ll_event(ctx, evt_name, data1, data2):
+        assert ctx == account._dc_context
+        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+        account._pm.hook.ac_process_ffi_event(
+            account=account, ffi_event=ffi_event
+        )
+    deltachat.set_context_callback(account._dc_context, _ll_event)
+
+
+@global_hookimpl
+def dc_account_after_shutdown(dc_context):
+    deltachat.clear_context_callback(dc_context)
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid=None, debug=True):
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
         self.account = account
-        self._event_queue = Queue()
-        self._debug = debug
-        if logid is None:
-            logid = str(self.account._dc_context).strip(">").split()[-1]
         self.logid = logid
-        self._timeout = None
         self.init_time = time.time()
 
-    @hookimpl
-    def process_low_level_event(self, account, event_name, data1, data2):
-        if self.account == account:
-            self._log_event(event_name, data1, data2)
-            self._event_queue.put((event_name, data1, data2))
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._log_event(ffi_event)
+
+    def _log_event(self, ffi_event):
+        # don't show events that are anyway empty impls now
+        if ffi_event.name == "DC_EVENT_GET_STRING":
+            return
+        self.account.ac_log_line(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
         self._timeout = timeout
@@ -32,10 +92,10 @@ class EventLogger:
             self.get(check_error=check_error)
 
     def get(self, timeout=None, check_error=True):
-        timeout = timeout or self._timeout
+        timeout = timeout if timeout is not None else self._timeout
         ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev[0] == "DC_EVENT_ERROR":
-            raise ValueError("{}({!r},{!r})".format(*ev))
+        if check_error and ev.name == "DC_EVENT_ERROR":
+            raise ValueError(str(ev))
         return ev
 
     def ensure_event_not_queued(self, event_name_regex):
@@ -47,35 +107,31 @@ class EventLogger:
             except Empty:
                 break
             else:
-                assert not rex.match(ev[0]), "event found {}".format(ev)
+                assert not rex.match(ev.name), "event found {}".format(ev)
 
     def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
         rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
             ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev[0]):
+            if rex.match(ev.name):
                 return ev
 
     def get_info_matching(self, regex):
         rex = re.compile("(?:{}).*".format(regex))
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev[2]):
+            if rex.match(ev.data2):
                 return ev
 
-    def _log_event(self, evt_name, data1, data2):
-        # don't show events that are anyway empty impls now
-        if evt_name == "DC_EVENT_GET_STRING":
-            return
-        if self._debug:
-            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
-            self._log(evpart)
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.account.get_message_by_id(ev.data2)
 
-    def _log(self, msg):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        with self._loglock:
-            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
+    def wait_next_messages_changed(self):
+        """ wait for and return next message-changed message or None
+        if the event contains no msgid"""
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
+        if ev.data2 > 0:
+            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/hookspec.py

@@ -1,25 +1,84 @@
-""" Hooks for python bindings """
+""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
 
 import pluggy
 
-name = "deltachat"
 
-hookspec = pluggy.HookspecMarker(name)
-hookimpl = pluggy.HookimplMarker(name)
-_plugin_manager = None
+account_spec_name = "deltachat-account"
+account_hookspec = pluggy.HookspecMarker(account_spec_name)
+account_hookimpl = pluggy.HookimplMarker(account_spec_name)
+
+global_spec_name = "deltachat-global"
+global_hookspec = pluggy.HookspecMarker(global_spec_name)
+global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 
 
-def get_plugin_manager():
-    global _plugin_manager
-    if _plugin_manager is None:
-        _plugin_manager = pluggy.PluginManager(name)
-        _plugin_manager.add_hookspecs(DeltaChatHookSpecs)
-    return _plugin_manager
+class PerAccount:
+    """ per-Account-instance hook specifications.
+
+    Except for ac_process_ffi_event all hooks are executed
+    in the thread which calls Account.wait_shutdown().
+    """
+    @classmethod
+    def _make_plugin_manager(cls):
+        pm = pluggy.PluginManager(account_spec_name)
+        pm.add_hookspecs(cls)
+        return pm
+
+    @account_hookspec
+    def ac_process_ffi_event(self, ffi_event):
+        """ process a CFFI low level events for a given account.
+
+        ffi_event has "name", "data1", "data2" values as specified
+        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
+        """
+
+    @account_hookspec
+    def ac_log_line(self, message):
+        """ log a message related to the account. """
+
+    @account_hookspec
+    def ac_configure_completed(self, success):
+        """ Called when a configure process completed. """
+
+    @account_hookspec
+    def ac_incoming_message(self, message):
+        """ Called on any incoming message (to deaddrop or chat). """
+
+    @account_hookspec
+    def ac_message_delivered(self, message):
+        """ Called when an outgoing message has been delivered to SMTP. """
+
+    @account_hookspec
+    def ac_member_added(self, chat, contact):
+        """ Called for each contact added to a chat. """
+
+    @account_hookspec
+    def ac_member_removed(self, chat, contact):
+        """ Called for each contact removed from a chat. """
 
 
-class DeltaChatHookSpecs:
-    """ Plugin Hook specifications for Python bindings to Delta Chat CFFI. """
+class Global:
+    """ global hook specifications using a per-process singleton
+    plugin manager instance.
 
-    @hookspec
-    def process_low_level_event(self, account, event_name, data1, data2):
-        """ process a CFFI low level events for a given account. """
+    """
+    _plugin_manager = None
+
+    @classmethod
+    def _get_plugin_manager(cls):
+        if cls._plugin_manager is None:
+            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
+            pm.add_hookspecs(cls)
+        return cls._plugin_manager
+
+    @global_hookspec
+    def dc_account_init(self, account):
+        """ called when `Account::__init__()` function starts executing. """
+
+    @global_hookspec
+    def dc_account_after_shutdown(self, account, dc_context):
+        """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -0,0 +1,106 @@
+
+import threading
+import time
+
+from contextlib import contextmanager
+
+from .capi import lib
+
+
+class IOThreads:
+    def __init__(self, account):
+        self.account = account
+        self._dc_context = account._dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+
+    def is_started(self):
+        return len(self._name2thread) > 0
+
+    def start(self, callback_thread):
+        assert not self.is_started()
+        self._start_one_thread("inbox", self.imap_thread_run)
+        self._start_one_thread("smtp", self.smtp_thread_run)
+
+        if callback_thread:
+            self._start_one_thread("cb", self.cb_thread_run)
+
+        if int(self.account.get_config("mvbox_watch")):
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+
+        if int(self.account.get_config("sentbox_watch")):
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.ac_log_line(message + " START")
+        yield
+        self.account.ac_log_line(message + " FINISHED")
+
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+
+        # Workaround for a race condition. Make sure that thread is
+        # not in between checking for quitflag and entering idle.
+        time.sleep(0.5)
+
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        if "mvbox" in self._name2thread:
+            lib.dc_interrupt_mvbox_idle(self._dc_context)
+        if "sentbox" in self._name2thread:
+            lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                if thread != threading.currentThread():
+                    thread.join()
+
+    def cb_thread_run(self):
+        with self.log_execution("CALLBACK THREAD START"):
+            it = self.account.iter_events()
+            while not self._thread_quitflag:
+                try:
+                    ev = next(it)
+                except StopIteration:
+                    break
+                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
+                ev.call_hook()
+
+    def imap_thread_run(self):
+        with self.log_execution("INBOX THREAD START"):
+            while not self._thread_quitflag:
+                lib.dc_perform_imap_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_idle(self._dc_context)
+
+    def mvbox_thread_run(self):
+        with self.log_execution("MVBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_mvbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_idle(self._dc_context)
+
+    def sentbox_thread_run(self):
+        with self.log_execution("SENTBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_sentbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_idle(self._dc_context)
+
+    def smtp_thread_run(self):
+        with self.log_execution("SMTP THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_smtp_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -50,6 +50,16 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
+        """
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_msg_unref
+        )
+
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -159,6 +169,13 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    def get_sender_chat(self):
+        """return the 1:1 chat with the sender of this message.
+
+        :returns: :class:`deltachat.chat.Chat` instance
+        """
+        return self.get_sender_contact().get_chat()
+
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
@@ -166,7 +183,7 @@ class Message(object):
         """
         from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self._dc_context, contact_id)
+        return Contact(self.account, contact_id)
 
     #
     # Message State query methods
@@ -269,6 +286,10 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
+    def mark_seen(self):
+        """ mark this message as seen. """
+        self.account.mark_seen_messages([self.id])
+
 
 # some code for handling DC_MSG_* view types
 

python/src/deltachat/testplugin.py

@@ -0,0 +1,373 @@
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import queue
+import threading
+import fnmatch
+import pytest
+import requests
+import time
+from . import Account, const
+from .tracker import ConfigureTracker
+from .capi import lib
+from .eventlogger import FFIEventLogger, FFIEventTracker
+from _pytest.monkeypatch import MonkeyPatch
+from _pytest._code import Source
+
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+    parser.addoption(
+        "--ignored", action="store_true",
+        help="Also run tests marked with the ignored marker",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    m = MonkeyPatch()
+    try:
+        m.setattr(sys.stdout, "write", lambda x: len(x))
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        m.undo()
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "?" in cfg:
+            url, token = cfg.split("?", 1)
+            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def data(request):
+    class Data:
+        def __init__(self):
+            # trying to find test data heuristically
+            # because we are run from a dev-setup with pytest direct,
+            # through tox, and then maybe also from deltachat-binding
+            # users like "deltabot".
+            self.paths = [os.path.normpath(x) for x in [
+                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
+            ]]
+
+        def get_path(self, bn):
+            """ return path of file or None if it doesn't exist. """
+            for path in self.paths:
+                fn = os.path.join(path, *bn.split("/"))
+                if os.path.exists(fn):
+                    return fn
+            print("WARNING: path does not exist: {!r}".format(fn))
+
+        def read_path(self, bn, mode="r"):
+            fn = self.get_path(bn)
+            if fn is not None:
+                with open(fn, mode) as f:
+                    return f.read()
+
+    return Data()
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
+                                    "dom", "elena", "fiona"]
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            if not quiet:
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
+
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac.start()
+            return ac
+
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            ac1._configtracker.wait_imap_connected()
+            ac1._configtracker.wait_smtp_connected()
+            ac1._configtracker.wait_finish()
+            return ac1
+
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            ac1._configtracker.wait_finish()
+            ac2._configtracker.wait_finish()
+            return ac1, ac2
+
+        def clone_online_account(self, account, pre_generated_key=True):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            ac.start()
+            return ac
+
+        def run_bot_process(self, module):
+            fn = module.__file__
+
+            bot_ac, bot_cfg = self.get_online_config()
+
+            args = [
+                sys.executable,
+                fn,
+                "--show-ffi",
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=1,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+class BotProcess:
+    def __init__(self, popen, bot_cfg):
+        self.popen = popen
+        self.addr = bot_cfg["addr"]
+
+        # we read stdout as quickly as we can in a thread and make
+        # the (unicode) lines available for readers through a queue.
+        self.stdout_queue = queue.Queue()
+        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
+        t.setDaemon(1)
+        t.start()
+
+    def _run_stdout_thread(self):
+        try:
+            while 1:
+                line = self.popen.stdout.readline()
+                if not line:
+                    break
+                line = line.strip()
+                self.stdout_queue.put(line)
+        finally:
+            self.stdout_queue.put(None)
+
+    def kill(self):
+        self.popen.kill()
+
+    def wait(self, timeout=30):
+        self.popen.wait(timeout=timeout)
+
+    def fnmatch_lines(self, pattern_lines):
+        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
+        for next_pattern in patterns:
+            print("+++FNMATCH:", next_pattern)
+            while 1:
+                line = self.stdout_queue.get(timeout=15)
+                if line is None:
+                    raise IOError("BOT stdout-thread terminated")
+                if fnmatch.fnmatch(line, next_pattern):
+                    print("+++MATCHED:", line)
+                    break
+                else:
+                    print("+++IGN:", line)
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()

python/src/deltachat/tracker.py

@@ -0,0 +1,76 @@
+
+from queue import Queue
+from threading import Event
+
+from .hookspec import account_hookimpl
+
+
+class ImexFailed(RuntimeError):
+    """ Exception for signalling that import/export operations failed."""
+
+
+class ImexTracker:
+    def __init__(self):
+        self._imex_events = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
+            self._imex_events.put(ffi_event.data1)
+        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
+            self._imex_events.put(ffi_event.data1)
+
+    def wait_finish(self, progress_timeout=60):
+        """ Return list of written files, raise ValueError if ExportFailed. """
+        files_written = []
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif ev == 0:
+                raise ImexFailed("export failed, exp-files: {}".format(files_written))
+            elif ev == 1000:
+                return files_written
+
+
+class ConfigureFailed(RuntimeError):
+    """ Exception for signalling that configuration failed."""
+
+
+class ConfigureTracker:
+    ConfigureFailed = ConfigureFailed
+
+    def __init__(self):
+        self._configure_events = Queue()
+        self._smtp_finished = Event()
+        self._imap_finished = Event()
+        self._ffi_events = []
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._ffi_events.append(ffi_event)
+        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
+            self._smtp_finished.set()
+        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
+            self._imap_finished.set()
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        self._configure_events.put(success)
+
+    def wait_smtp_connected(self):
+        """ wait until smtp is configured. """
+        self._smtp_finished.wait()
+
+    def wait_imap_connected(self):
+        """ wait until smtp is configured. """
+        self._imap_finished.wait()
+
+    def wait_finish(self):
+        """ wait until configure is completed.
+
+        Raise Exception if Configure failed
+        """
+        if not self._configure_events.get():
+            content = "\n".join(map(str, self._ffi_events))
+            raise ConfigureFailed(content)

python/tests/conftest.py

@@ -1,317 +1,18 @@
 from __future__ import print_function
-import os
-import sys
-import py
-import pytest
-import requests
-import time
-from deltachat import Account
-from deltachat import const
-from deltachat.capi import lib
-from _pytest.monkeypatch import MonkeyPatch
-import tempfile
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-    parser.addoption(
-        "--ignored", action="store_true",
-        help="Also run tests marked with the ignored marker",
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
-        if cfg:
-            config.option.liveconfig = cfg
-
-
-def pytest_runtest_setup(item):
-    if (list(item.iter_markers(name="ignored"))
-            and not item.config.getoption("ignored")):
-        pytest.skip("Ignored tests not requested, use --ignored")
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    m = MonkeyPatch()
-    try:
-        m.setattr(sys.stdout, "write", lambda x: len(x))
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        m.undo()
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "#" in cfg:
-            url, token = cfg.split("#", 1)
-            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-@pytest.fixture(scope="session")
-def data():
-    class Data:
-        def __init__(self):
-            self.path = os.path.join(os.path.dirname(__file__), "data")
-
-        def get_path(self, bn):
-            fn = os.path.join(self.path, bn)
-            assert os.path.exists(fn)
-            return fn
-    return Data()
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url):
-        self.configlist = []
-        self.url = url
-
-    def get(self, index):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture(scope='session')
-def datadir():
-    """The py.path.local object of the test-data/ directory."""
-    for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
-        if datadir.isdir():
-            return datadir
-    else:
-        pytest.skip('test-data directory not found')
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-        def make_account(self, path, logid):
-            ac = Account(path, logid=logid)
-            self._finalizers.append(ac.shutdown)
-            return ac
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(2)
-            return ac
-
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = "key/{name}-public.asc".format(name=keyname)
-                fname_sec = "key/{name}-secret.asc".format(name=keyname)
-                account._preconfigure_keypair(addr,
-                                              datadir.join(fname_pub).read(),
-                                              datadir.join(fname_sec).read())
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_config(self, pre_generated_key=True):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False,
-                                           pre_generated_key=True, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key)
-            configdict.update(config)
-            ac.configure(**configdict)
-            ac.start_threads(mvbox=mvbox, sentbox=sentbox)
-            return ac
-
-        def get_one_online_account(self, pre_generated_key=True):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key)
-            wait_successful_IMAP_SMTP_connection(ac1)
-            wait_configuration_progress(ac1, 1000)
-            return ac1
-
-        def get_two_online_accounts(self):
-            ac1 = self.get_online_configuring_account()
-            ac2 = self.get_online_configuring_account()
-            wait_successful_IMAP_SMTP_connection(ac1)
-            wait_configuration_progress(ac1, 1000)
-            wait_successful_IMAP_SMTP_connection(ac2)
-            wait_configuration_progress(ac2, 1000)
-            return ac1, ac2
-
-        def clone_online_account(self, account, pre_generated_key=True):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
-            ac.start_threads()
-            return ac
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    return am
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-    return Printer()
 
 
 def wait_configuration_progress(account, min_target, max_target=1001):
     min_target = min(min_target, max_target)
     while 1:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if data1 >= min_target and data1 <= max_target:
+        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if event.data1 >= min_target and event.data1 <= max_target:
             print("** CONFIG PROGRESS {}".format(min_target), account)
             break
 
 
 def wait_securejoin_inviter_progress(account, target):
     while 1:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if data2 >= target:
+        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if event.data2 >= target:
             print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
             break
-
-
-def wait_successful_IMAP_SMTP_connection(account):
-    imap_ok = smtp_ok = False
-    while not imap_ok or not smtp_ok:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
-        if evt_name == "DC_EVENT_IMAP_CONNECTED":
-            imap_ok = True
-            print("** IMAP OK", account)
-        if evt_name == "DC_EVENT_SMTP_CONNECTED":
-            smtp_ok = True
-            print("** SMTP OK", account)
-    print("** IMAP and SMTP logins successful", account)
-
-
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev[1] == chat_id
-    if msg_id is not None:
-        assert ev[2] == msg_id
-    return ev[2]

python/tests/data/key

@@ -0,0 +1 @@
+../../../test-data/key

python/tests/test_increation.py

@@ -6,10 +6,18 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from conftest import wait_configuration_progress, wait_msgs_changed
+from conftest import wait_configuration_progress
 from deltachat import const
 
 
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev.data1 == chat_id
+    if msg_id is not None:
+        assert ev.data2 == msg_id
+    return ev.data2
+
+
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -91,22 +99,22 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat.id
-        assert ev[2] == prepared_original.id
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat2.id
-        assert ev[2] == forwarded_id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat.id
+        assert ev.data2 == prepared_original.id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat2.id
+        assert ev.data2 == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1[2])
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2[1] != ev1[1]
-        received_copy = ac2.get_message_by_id(ev2[2])
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,5 +1,7 @@
 from __future__ import print_function
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
+from deltachat import register_global_plugin
+from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
 
@@ -16,21 +18,31 @@ def test_callback_None2int():
     clear_context_callback(ctx)
 
 
-def test_dc_close_events(tmpdir):
-    from deltachat.account import Account
-    p = tmpdir.join("hello.db")
-    ac1 = Account(p.strpath)
+def test_dc_close_events(tmpdir, acfactory):
+    ac1 = acfactory.get_unconfigured_account()
+
+    # register after_shutdown function
+    shutdowns = []
+
+    class ShutdownPlugin:
+        @global_hookimpl
+        def dc_account_after_shutdown(self, account):
+            assert account._dc_context is None
+            shutdowns.append(account)
+    register_global_plugin(ShutdownPlugin())
+    assert hasattr(ac1, "_dc_context")
     ac1.shutdown()
+    assert shutdowns == [ac1]
 
     def find(info_string):
-        evlog = ac1._evlogger
+        evlog = ac1._evtracker
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev[2]
+            data2 = ev.data2
             if info_string in data2:
                 return
             else:
-                print("skipping event", *ev)
+                print("skipping event", ev)
 
     find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
@@ -80,10 +92,10 @@ def test_markseen_invalid_message_ids(acfactory):
     contact1 = ac1.create_contact(email="some1@example.com", name="some1")
     chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):

python/tox.ini

@@ -7,14 +7,14 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs:tests}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
-    DCC_NEW_TMP_EMAIL
+    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -41,13 +41,13 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/
+    flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx==2.2.0
+    sphinx
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -67,7 +67,6 @@ commands =
 
 [pytest]
 addopts = -v -ra --strict-markers
-python_files = tests/test_*.py
 norecursedirs = .tox 
 xfail_strict=true
 timeout = 90

rust-toolchain

@@ -1 +1 @@
-nightly-2019-11-06
+nightly-2020-03-12

src/aheader.rs

@@ -127,14 +127,10 @@ impl str::FromStr for Aheader {
             .split(';')
             .filter_map(|a| {
                 let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
-                if attribute.len() < 2 {
-                    return None;
+                match &attribute[..] {
+                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
+                    _ => None,
                 }
-
-                Some((
-                    attribute[0].trim().to_string(),
-                    attribute[1].trim().to_string(),
-                ))
             })
             .collect();
 

src/blob.rs

@@ -6,13 +6,13 @@ use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
-use self::image::GenericImageView;
+use image::GenericImageView;
+use thiserror::Error;
+
 use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
 
-extern crate image;
-
 /// Represents a file in the blob directory.
 ///
 /// The object has a name, which will always be valid UTF-8.  Having a
@@ -56,7 +56,6 @@ impl<'a> BlobObject<'a> {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
                 cause: err,
-                backtrace: failure::Backtrace::new(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -84,7 +83,6 @@ impl<'a> BlobObject<'a> {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
-                            backtrace: failure::Backtrace::new(),
                         });
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -97,7 +95,6 @@ impl<'a> BlobObject<'a> {
             blobdir: dir.to_path_buf(),
             blobname: name,
             cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
-            backtrace: failure::Backtrace::new(),
         })
     }
 
@@ -122,7 +119,6 @@ impl<'a> BlobObject<'a> {
             blobname: String::from(""),
             src: src.as_ref().to_path_buf(),
             cause: err,
-            backtrace: failure::Backtrace::new(),
         })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
@@ -138,7 +134,6 @@ impl<'a> BlobObject<'a> {
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
-                backtrace: failure::Backtrace::new(),
             }
         })?;
         let blob = BlobObject {
@@ -198,17 +193,14 @@ impl<'a> BlobObject<'a> {
             .map_err(|_| BlobError::WrongBlobdir {
                 blobdir: context.get_blobdir().to_path_buf(),
                 src: path.as_ref().to_path_buf(),
-                backtrace: failure::Backtrace::new(),
             })?;
         if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
                 blobname: path.as_ref().to_path_buf(),
-                backtrace: failure::Backtrace::new(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
             blobname: path.as_ref().to_path_buf(),
-            backtrace: failure::Backtrace::new(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -236,7 +228,6 @@ impl<'a> BlobObject<'a> {
         if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::WrongName {
                 blobname: PathBuf::from(name),
-                backtrace: failure::Backtrace::new(),
             });
         }
         Ok(BlobObject {
@@ -359,7 +350,6 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
-            backtrace: failure::Backtrace::new(),
         })?;
 
         if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
@@ -372,7 +362,6 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
-            backtrace: failure::Backtrace::new(),
         })?;
 
         Ok(())
@@ -386,98 +375,41 @@ impl<'a> fmt::Display for BlobObject<'a> {
 }
 
 /// Errors for the [BlobObject].
-#[derive(Fail, Debug)]
+#[derive(Debug, Error)]
 pub enum BlobError {
+    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
     CreateFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
     WriteFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
         blobdir: PathBuf,
         blobname: String,
         src: PathBuf,
-        #[cause]
+        #[source]
         cause: std::io::Error,
-        backtrace: failure::Backtrace,
     },
+    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
     RecodeFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[cause]
+        #[source]
         cause: image::ImageError,
-        backtrace: failure::Backtrace,
     },
-    WrongBlobdir {
-        blobdir: PathBuf,
-        src: PathBuf,
-        backtrace: failure::Backtrace,
-    },
-    WrongName {
-        blobname: PathBuf,
-        backtrace: failure::Backtrace,
-    },
-}
-
-// Implementing Display is done by hand because the failure
-// #[fail(display = "...")] syntax does not allow using
-// `blobdir.display()`.
-impl fmt::Display for BlobError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        // Match on the data rather than kind, they are equivalent for
-        // identifying purposes but contain the actual data we need.
-        match &self {
-            BlobError::CreateFailure {
-                blobdir, blobname, ..
-            } => write!(
-                f,
-                "Failed to create blob {} in {}",
-                blobname,
-                blobdir.display()
-            ),
-            BlobError::WriteFailure {
-                blobdir, blobname, ..
-            } => write!(
-                f,
-                "Failed to write data to blob {} in {}",
-                blobname,
-                blobdir.display()
-            ),
-            BlobError::CopyFailure {
-                blobdir,
-                blobname,
-                src,
-                ..
-            } => write!(
-                f,
-                "Failed to copy data from {} to blob {} in {}",
-                src.display(),
-                blobname,
-                blobdir.display(),
-            ),
-            BlobError::RecodeFailure {
-                blobdir, blobname, ..
-            } => write!(f, "Failed to recode {} in {}", blobname, blobdir.display(),),
-            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
-                f,
-                "File path {} is not in blobdir {}",
-                src.display(),
-                blobdir.display(),
-            ),
-            BlobError::WrongName { blobname, .. } => {
-                write!(f, "Blob has a bad name: {}", blobname.display(),)
-            }
-        }
-    }
+    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
+    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
+    #[error("Blob has a badname {}", .blobname.display())]
+    WrongName { blobname: PathBuf },
 }
 
 #[cfg(test)]

src/chat.rs

@@ -15,7 +15,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::error::{bail, ensure, format_err, Error};
 use crate::events::Event;
 use crate::job::*;
 use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId};
@@ -324,7 +324,7 @@ impl ChatId {
                 time(),
                 msg.viewtype,
                 MessageState::OutDraft,
-                msg.text.as_ref().map(String::as_str).unwrap_or(""),
+                msg.text.as_deref().unwrap_or(""),
                 msg.param.to_string(),
                 1,
             ],
@@ -378,6 +378,55 @@ impl ChatId {
         Ok(self.get_param(context)?.exists(Param::Devicetalk))
     }
 
+    fn parent_query<T, F>(self, context: &Context, fields: &str, f: F) -> sql::Result<Option<T>>
+    where
+        F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
+    {
+        let sql = &context.sql;
+        let query = format!(
+            "SELECT {} \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) \
+             ORDER BY timestamp DESC, id DESC \
+             LIMIT 1;",
+            fields
+        );
+        sql.query_row_optional(
+            query,
+            params![
+                self,
+                MessageState::OutPreparing,
+                MessageState::OutDraft,
+                MessageState::OutPending,
+                MessageState::OutFailed
+            ],
+            f,
+        )
+    }
+
+    fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
+        self.parent_query(
+            context,
+            "rfc724_mid, mime_in_reply_to, mime_references",
+            collect,
+        )
+        .ok()
+        .flatten()
+    }
+
+    fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
+        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
+        let packed: Option<String> = self.parent_query(context, "param", collect)?;
+
+        if let Some(ref packed) = packed {
+            let param = packed.parse::<Params>()?;
+            Ok(param.exists(Param::GuaranteeE2ee))
+        } else {
+            // No messages
+            Ok(false)
+        }
+    }
+
     /// Bad evil escape hatch.
     ///
     /// Avoid using this, eventually types should be cleaned up enough
@@ -525,9 +574,13 @@ impl Chat {
         self.param.exists(Param::Devicetalk)
     }
 
+    pub fn is_mailing_list(&self) -> bool {
+        self.param.exists(Param::MailingList)
+    }
+
     /// Returns true if user can send messages to this chat.
     pub fn can_send(&self) -> bool {
-        !self.id.is_special() && !self.is_device_talk()
+        !self.id.is_special() && !self.is_device_talk() && !self.is_mailing_list()
     }
 
     pub fn update_param(&mut self, context: &Context) -> Result<(), Error> {
@@ -576,6 +629,10 @@ impl Chat {
                 .unwrap_or_else(|| "Err".into());
         }
 
+        if self.typ == Chattype::Group && self.is_mailing_list() {
+            return "".to_string();
+        }
+
         if self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup {
             if self.id.is_deaddrop() {
                 return context.stock_str(StockMessage::DeadDrop).into();
@@ -587,44 +644,6 @@ impl Chat {
         "Err".to_string()
     }
 
-    fn parent_query(fields: &str) -> String {
-        // Check for server_uid guarantees that we don't
-        // select a draft or undelivered message.
-        format!(
-            "SELECT {} \
-             FROM msgs WHERE chat_id=?1 AND server_uid!=0 \
-             ORDER BY timestamp DESC, id DESC \
-             LIMIT 1;",
-            fields
-        )
-    }
-
-    fn get_parent_mime_headers(&self, context: &Context) -> Option<(String, String, String)> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
-        let params = params![self.id];
-        let sql = &context.sql;
-
-        let query = Self::parent_query("rfc724_mid, mime_in_reply_to, mime_references");
-
-        sql.query_row(&query, params, collect).ok()
-    }
-
-    fn parent_is_encrypted(&self, context: &Context) -> Result<bool, Error> {
-        let sql = &context.sql;
-        let params = params![self.id];
-        let query = Self::parent_query("param");
-
-        let packed: Option<String> = sql.query_get_value_result(&query, params)?;
-
-        if let Some(ref packed) = packed {
-            let param = packed.parse::<Params>()?;
-            Ok(param.exists(Param::GuaranteeE2ee))
-        } else {
-            // No messages
-            Ok(false)
-        }
-    }
-
     pub fn get_profile_image(&self, context: &Context) -> Option<PathBuf> {
         if let Some(image_rel) = self.param.get(Param::ProfileImage) {
             if !image_rel.is_empty() {
@@ -831,7 +850,7 @@ impl Chat {
                     }
                 }
 
-                if can_encrypt && (all_mutual || self.parent_is_encrypted(context)?) {
+                if can_encrypt && (all_mutual || self.id.parent_is_encrypted(context)?) {
                     msg.param.set_int(Param::GuaranteeE2ee, 1);
                 }
             }
@@ -846,7 +865,7 @@ impl Chat {
             // we do not set In-Reply-To/References in this case.
             if !self.is_self_talk() {
                 if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                    self.get_parent_mime_headers(context)
+                    self.id.get_parent_mime_headers(context)
                 {
                     if !parent_rfc724_mid.is_empty() {
                         new_in_reply_to = parent_rfc724_mid.clone();
@@ -1086,7 +1105,11 @@ pub fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<ChatId, Erro
             msg_id: MsgId::new(0),
         });
     }
-    Contact::scaleup_origin_by_id(context, msg.from_id, Origin::CreateChat);
+
+    // If the message is from a mailing list, the contacts are not counted as "known"
+    if !chat.is_mailing_list() {
+        Contact::scaleup_origin_by_id(context, msg.from_id, Origin::CreateChat);
+    }
     Ok(chat.id)
 }
 
@@ -1727,17 +1750,17 @@ pub fn get_next_media(
             msg_type2,
             msg_type3,
         );
-        for i in 0..list.len() {
-            if curr_msg_id == list[i] {
+        for (i, msg_id) in list.iter().enumerate() {
+            if curr_msg_id == *msg_id {
                 match direction {
                     Direction::Forward => {
                         if i + 1 < list.len() {
-                            ret = Some(list[i + 1]);
+                            ret = list.get(i + 1).copied();
                         }
                     }
                     Direction::Backward => {
                         if i >= 1 {
-                            ret = Some(list[i - 1]);
+                            ret = list.get(i - 1).copied();
                         }
                     }
                 }
@@ -1819,36 +1842,68 @@ pub fn create_group_chat(
     Ok(chat_id)
 }
 
+/// add a contact to the chats_contact table
+/// on success emit MemberAdded event and return true
 pub(crate) fn add_to_chat_contacts_table(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
 ) -> bool {
-    // add a contact to a chat; the function does not check the type or if any of the record exist or are already
-    // added to the chat!
-    sql::execute(
+    match sql::execute(
         context,
         &context.sql,
         "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
         params![chat_id, contact_id as i32],
-    )
-    .is_ok()
+    ) {
+        Ok(()) => {
+            context.call_cb(Event::MemberAdded {
+                chat_id,
+                contact_id,
+            });
+
+            true
+        }
+        Err(err) => {
+            error!(
+                context,
+                "could not add {} to chat {} table: {}", contact_id, chat_id, err
+            );
+
+            false
+        }
+    }
 }
 
+/// remove a contact from the chats_contact table
+/// on success emit MemberRemoved event and return true
 pub(crate) fn remove_from_chat_contacts_table(
     context: &Context,
     chat_id: ChatId,
     contact_id: u32,
 ) -> bool {
-    // remove a contact from the chats_contact table unconditionally
-    // the function does not check the type or if the record exist
-    sql::execute(
+    match sql::execute(
         context,
         &context.sql,
         "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?",
         params![chat_id, contact_id as i32],
-    )
-    .is_ok()
+    ) {
+        Ok(()) => {
+            context.call_cb(Event::MemberRemoved {
+                chat_id,
+                contact_id,
+            });
+
+            true
+        }
+        Err(_) => {
+            warn!(
+                context,
+                "could not remove contact {:?} from chat {:?}", contact_id, chat_id
+            );
+
+            false
+        }
+    }
 }
 
 /// Adds a contact to the chat.
@@ -1942,16 +1997,9 @@ pub(crate) fn add_contact_to_chat_ex(
         msg.param.set_cmd(SystemMessage::MemberAddedToGroup);
         msg.param.set(Param::Arg, contact.get_addr());
         msg.param.set_int(Param::Arg2, from_handshake.into());
+
         msg.id = send_msg(context, chat_id, &mut msg)?;
-        context.call_cb(Event::MsgsChanged {
-            chat_id,
-            msg_id: msg.id,
-        });
     }
-    context.call_cb(Event::MsgsChanged {
-        chat_id,
-        msg_id: MsgId::new(0),
-    });
     context.call_cb(Event::ChatModified(chat_id));
     Ok(true)
 }
@@ -2132,8 +2180,7 @@ pub fn remove_contact_from_chat(
                         "Cannot remove contact from chat; self not in group.".into()
                     )
                 );
-            } else if remove_from_chat_contacts_table(context, chat_id, contact_id) {
-                /* we should respect this - whatever we send to the group, it gets discarded anyway! */
+            } else {
                 if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                     if chat.is_promoted() {
                         msg.viewtype = Viewtype::Text;
@@ -2162,9 +2209,18 @@ pub fn remove_contact_from_chat(
                         });
                     }
                 }
-
+                // we remove the member from the chat after constructing the
+                // to-be-send message. If between send_msg() and here the
+                // process dies the user will have to re-do the action.  It's
+                // better than the other way round: you removed
+                // someone from DB but no peer or device gets to know about it and
+                // group membership is thus different on different devices.
+                // Note also that sending a message needs all recipients
+                // in order to correctly determine encryption so if we
+                // removed it first, it would complicate the
+                // check/encryption logic.
+                success = remove_from_chat_contacts_table(context, chat_id, contact_id);
                 context.call_cb(Event::ChatModified(chat_id));
-                success = true;
             }
         }
     }
@@ -2458,6 +2514,22 @@ pub(crate) fn get_chat_id_by_grpid(
     )
 }
 
+pub(crate) fn get_chat_id_by_mailinglistid(
+    context: &Context,
+    listid: impl AsRef<str>,
+) -> Result<(ChatId, Blocked), sql::Error> {
+    context.sql.query_row(
+        "SELECT id, blocked, type FROM chats WHERE grpid=?;",
+        params![listid.as_ref()],
+        |row| {
+            let chat_id = row.get::<_, ChatId>(0)?;
+
+            let b = row.get::<_, Option<Blocked>>(1)?.unwrap_or_default();
+            Ok((chat_id, b))
+        },
+    )
+}
+
 /// Adds a message to device chat.
 ///
 /// Optional `label` can be provided to ensure that message is added only once.
@@ -3119,4 +3191,16 @@ mod tests {
             false
         );
     }
+
+    #[test]
+    fn test_parent_is_encrypted() {
+        let t = dummy_context();
+        let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap();
+        assert!(!chat_id.parent_is_encrypted(&t.ctx).unwrap());
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("hello".to_string()));
+        chat_id.set_draft(&t.ctx, Some(&mut msg));
+        assert!(!chat_id.parent_is_encrypted(&t.ctx).unwrap());
+    }
 }

src/chatlist.rs

@@ -5,7 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::Result;
+use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
@@ -275,18 +275,20 @@ impl Chatlist {
     ///
     /// To get the message object from the message ID, use dc_get_chat().
     pub fn get_chat_id(&self, index: usize) -> ChatId {
-        if index >= self.ids.len() {
-            return ChatId::new(0);
+        match self.ids.get(index) {
+            Some((chat_id, _msg_id)) => *chat_id,
+            None => ChatId::new(0),
         }
-        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
-        ensure!(index < self.ids.len(), "Chatlist index out of range");
-        Ok(self.ids[index].1)
+        match self.ids.get(index) {
+            Some((_chat_id, msg_id)) => Ok(*msg_id),
+            None => bail!("Chatlist index out of range"),
+        }
     }
 
     /// Get a summary for a chatlist index.
@@ -310,25 +312,27 @@ impl Chatlist {
         // Also, sth. as "No messages" would not work if the summary comes from a message.
         let mut ret = Lot::new();
 
-        if index >= self.ids.len() {
-            ret.text2 = Some("ErrBadChatlistIndex".to_string());
-            return ret;
-        }
+        let (chat_id, lastmsg_id) = match self.ids.get(index) {
+            Some(ids) => ids,
+            None => {
+                ret.text2 = Some("ErrBadChatlistIndex".to_string());
+                return ret;
+            }
+        };
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
-        let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {

src/configure/auto_mozilla.rs

@@ -1,7 +1,6 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
 //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
@@ -9,33 +8,7 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "Invalid email address: {:?}", _0)]
-    InvalidEmailAddress(String),
-
-    #[fail(display = "XML error at position {}", position)]
-    InvalidXml {
-        position: usize,
-        #[cause]
-        error: quick_xml::Error,
-    },
-
-    #[fail(display = "Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[fail(display = "Failed to get URL {}", _0)]
-    ReadUrlError(#[cause] super::read_url::Error),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<super::read_url::Error> for Error {
-    fn from(err: super::read_url::Error) -> Error {
-        Error::ReadUrlError(err)
-    }
-}
+use super::Error;
 
 #[derive(Debug)]
 struct MozAutoconfigure<'a> {
@@ -65,16 +38,16 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam> {
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     // Split address into local part and domain part.
-    let p = in_emailaddr
-        .find('@')
-        .ok_or_else(|| Error::InvalidEmailAddress(in_emailaddr.to_string()))?;
-    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
-    let in_emaildomain = &in_emaildomain[1..];
+    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
+    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
+        [domain, local] => (local, domain),
+        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
+    };
 
     let mut moz_ac = MozAutoconfigure {
         in_emailaddr,
@@ -125,7 +98,7 @@ pub fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam> {
+) -> Result<LoginParam, Error> {
     let xml_raw = read_url(context, url)?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);

src/configure/auto_outlook.rs

@@ -1,6 +1,5 @@
 //! Outlook's Autodiscover
 
-use quick_xml;
 use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
@@ -8,33 +7,7 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "XML error at position {}", position)]
-    InvalidXml {
-        position: usize,
-        #[cause]
-        error: quick_xml::Error,
-    },
-
-    #[fail(display = "Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[fail(display = "Failed to get URL {}", _0)]
-    ReadUrlError(#[cause] super::read_url::Error),
-
-    #[fail(display = "Number of redirection is exceeded")]
-    RedirectionError,
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<super::read_url::Error> for Error {
-    fn from(err: super::read_url::Error) -> Error {
-        Error::ReadUrlError(err)
-    }
-}
+use super::Error;
 
 struct OutlookAutodiscover {
     pub out: LoginParam,
@@ -52,7 +25,7 @@ enum ParsingResult {
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult> {
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
         out_imap_set: false,
@@ -143,7 +116,7 @@ pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
-) -> Result<LoginParam> {
+) -> Result<LoginParam, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {

src/configure/mod.rs

@@ -12,6 +12,7 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::error::format_err;
 use crate::job::{self, job_add, job_kill_action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::oauth2::*;
@@ -651,6 +652,28 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Invalid email address: {0:?}")]
+    InvalidEmailAddress(String),
+
+    #[error("XML error at position {position}")]
+    InvalidXml {
+        position: usize,
+        #[source]
+        error: quick_xml::Error,
+    },
+
+    #[error("Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[error("Failed to get URL")]
+    ReadUrlError(#[from] self::read_url::Error),
+
+    #[error("Number of redirection is exceeded")]
+    RedirectionError,
+}
+
 #[cfg(test)]
 mod tests {
 

src/configure/read_url.rs

@@ -1,14 +1,12 @@
 use crate::context::Context;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "URL request error")]
-    GetError(#[cause] reqwest::Error),
+    #[error("URL request error")]
+    GetError(#[from] reqwest::Error),
 }
 
-pub type Result<T> = std::result::Result<T, Error>;
-
-pub fn read_url(context: &Context, url: &str) -> Result<String> {
+pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     info!(context, "Requesting URL {}", url);
 
     match reqwest::blocking::Client::new()

src/constants.rs

@@ -197,13 +197,13 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
+pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
+    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN);
+    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
 
 // QR code scanning (view from Bob, the joiner)
 pub const DC_VC_AUTH_REQUIRED: i32 = 2;

src/contact.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 
 use deltachat_derive::*;
 use itertools::Itertools;
-use rusqlite;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -13,7 +12,7 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
-use crate::error::{Error, Result};
+use crate::error::{bail, ensure, format_err, Result};
 use crate::events::Event;
 use crate::key::*;
 use crate::login_param::LoginParam;
@@ -24,9 +23,6 @@ use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-/// Contacts with at least this origin value are shown in the contact list.
-const DC_ORIGIN_MIN_CONTACT_LIST: i32 = 0x100;
-
 /// An object representing a single contact in memory.
 ///
 /// The contact object is not updated.
@@ -94,6 +90,7 @@ pub enum Origin {
     UnhandledQrScan = 0x80,
 
     /// Reply-To: of incoming message of known sender
+    /// Contacts with at least this origin value are shown in the contact list.
     IncomingReplyTo = 0x100,
 
     /// Cc: of incoming message of known sender
@@ -274,7 +271,7 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `dc_may_be_valid_addr()`.
-    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>) -> u32 {
+    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>, min_origin: Origin) -> u32 {
         if addr.as_ref().is_empty() {
             return 0;
         }
@@ -287,14 +284,13 @@ impl Contact {
         if addr_cmp(addr_normalized, addr_self) {
             return DC_CONTACT_ID_SELF;
         }
-
         context.sql.query_get_value(
             context,
             "SELECT id FROM contacts WHERE addr=?1 COLLATE NOCASE AND id>?2 AND origin>=?3 AND blocked=0;",
             params![
                 addr_normalized,
                 DC_CONTACT_ID_LAST_SPECIAL as i32,
-                DC_ORIGIN_MIN_CONTACT_LIST,
+                min_origin as u32,
             ],
         ).unwrap_or_default()
     }
@@ -347,20 +343,6 @@ impl Contact {
             return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
-        if !may_be_valid_addr(&addr) {
-            warn!(
-                context,
-                "Bad address \"{}\" for contact \"{}\".",
-                addr,
-                if !name.as_ref().is_empty() {
-                    name.as_ref()
-                } else {
-                    "<unset>"
-                },
-            );
-            bail!("Bad address supplied: {:?}", addr);
-        }
-
         let mut update_addr = false;
         let mut update_name = false;
         let mut update_authname = false;
@@ -1120,10 +1102,9 @@ fn cat_fingerprint(
 impl Context {
     /// determine whether the specified addr maps to the/a self addr
     pub fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        let self_addr = match self.get_config(Config::ConfiguredAddr) {
-            Some(s) => s,
-            None => return Err(Error::NotConfigured),
-        };
+        let self_addr = self
+            .get_config(Config::ConfiguredAddr)
+            .ok_or_else(|| format_err!("Not configured"))?;
 
         Ok(addr_cmp(self_addr, addr))
     }

src/dc_receive_imf.rs

@@ -2,6 +2,8 @@ use itertools::join;
 use sha2::{Digest, Sha256};
 
 use num_traits::FromPrimitive;
+use regex::Regex;
+use std::borrow::Cow;
 
 use crate::chat::{self, Chat, ChatId};
 use crate::config::Config;
@@ -9,7 +11,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::Result;
+use crate::error::{bail, ensure, Result};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::*;
@@ -71,6 +73,8 @@ pub fn dc_receive_imf(
     let mut created_db_entries = Vec::new();
     let mut create_event_to_send = Some(CreateEvent::MsgsChanged);
 
+    let list_id_header: Option<&String> = mime_parser.get(HeaderDef::ListId);
+
     // helper method to handle early exit and memory cleanup
     let cleanup = |context: &Context,
                    create_event_to_send: &Option<CreateEvent>,
@@ -98,7 +102,7 @@ pub fn dc_receive_imf(
     // https://github.com/deltachat/deltachat-core/issues/150)
     let (from_id, from_id_blocked, incoming_origin) =
         if let Some(field_from) = mime_parser.get(HeaderDef::From_) {
-            from_field_to_contact_id(context, field_from)?
+            from_field_to_contact_id(context, field_from, list_id_header)?
         } else {
             (0, false, Origin::Unknown)
         };
@@ -117,6 +121,7 @@ pub fn dc_receive_imf(
                 } else {
                     Origin::IncomingUnknownTo
                 },
+                list_id_header,
             )?);
         }
     }
@@ -230,11 +235,13 @@ pub fn dc_receive_imf(
 pub fn from_field_to_contact_id(
     context: &Context,
     field_from: &str,
+    list_id_header: Option<&String>,
 ) -> Result<(u32, bool, Origin)> {
     let from_ids = dc_add_or_lookup_contacts_by_address_list(
         context,
         &field_from,
         Origin::IncomingUnknownFrom,
+        list_id_header,
     )?;
 
     if from_ids.contains(&DC_CONTACT_ID_SELF) {
@@ -415,9 +422,34 @@ fn add_parts(
 
         if chat_id.is_unset() {
             // check if the message belongs to a mailing list
-            if mime_parser.is_mailinglist_message() {
-                *chat_id = ChatId::new(DC_CHAT_ID_TRASH);
-                info!(context, "Message belongs to a mailing list and is ignored.",);
+            if let Some(list_id_header) = mime_parser.get(HeaderDef::ListId) {
+                let create_blocked = if !test_normal_chat_id.is_unset()
+                    && test_normal_chat_id_blocked == Blocked::Not
+                {
+                    Blocked::Not
+                } else {
+                    Blocked::Deaddrop
+                };
+
+                let (new_chat_id, new_chat_id_blocked) = create_or_lookup_mailinglist(
+                    context,
+                    if test_normal_chat_id.is_unset() {
+                        allow_creation
+                    } else {
+                        true
+                    },
+                    create_blocked,
+                    list_id_header,
+                );
+                *chat_id = new_chat_id;
+                chat_id_blocked = new_chat_id_blocked;
+                if !chat_id.is_unset()
+                    && chat_id_blocked != Blocked::Not
+                    && create_blocked == Blocked::Not
+                {
+                    new_chat_id.unblock(context);
+                    chat_id_blocked = Blocked::Not;
+                }
             }
         }
 
@@ -800,7 +832,6 @@ fn create_or_lookup_group(
     let mut chat_id_blocked = Blocked::Not;
     let mut recreate_member_list = false;
     let mut send_EVENT_CHAT_MODIFIED = false;
-    let mut X_MrRemoveFromGrp = None;
     let mut X_MrAddToGrp = None;
     let mut X_MrGrpNameChanged = false;
     let mut better_msg: String = From::from("");
@@ -848,22 +879,25 @@ fn create_or_lookup_group(
     // but we might not know about this group
 
     let grpname = mime_parser.get(HeaderDef::ChatGroupName).cloned();
+    let mut removed_id = 0;
 
-    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() {
-        X_MrRemoveFromGrp = Some(optional_field);
-        mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup;
-        let left_group = Contact::lookup_id_by_addr(context, X_MrRemoveFromGrp.as_ref().unwrap())
-            == from_id as u32;
-        better_msg = context.stock_system_msg(
-            if left_group {
-                StockMessage::MsgGroupLeft
-            } else {
-                StockMessage::MsgDelMember
-            },
-            X_MrRemoveFromGrp.as_ref().unwrap(),
-            "",
-            from_id as u32,
-        )
+    if let Some(removed_addr) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() {
+        removed_id = Contact::lookup_id_by_addr(context, &removed_addr, Origin::Unknown);
+        if removed_id == 0 {
+            warn!(context, "removed {:?} has no contact_id", removed_addr);
+        } else {
+            mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup;
+            better_msg = context.stock_system_msg(
+                if removed_id == from_id as u32 {
+                    StockMessage::MsgGroupLeft
+                } else {
+                    StockMessage::MsgDelMember
+                },
+                &removed_addr,
+                "",
+                from_id as u32,
+            );
+        }
     } else {
         let field = mime_parser.get(HeaderDef::ChatGroupMemberAdded).cloned();
         if let Some(optional_field) = field {
@@ -949,7 +983,7 @@ fn create_or_lookup_group(
             && !grpid.is_empty()
             && grpname.is_some()
             // otherwise, a pending "quit" message may pop up
-            && X_MrRemoveFromGrp.is_none()
+            && removed_id == 0
             // re-create explicitly left groups only if ourself is re-added
             && (!group_explicitly_left
                 || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap()))
@@ -1083,12 +1117,8 @@ fn create_or_lookup_group(
             }
         }
         send_EVENT_CHAT_MODIFIED = true;
-    } else if let Some(removed_addr) = X_MrRemoveFromGrp {
-        let contact_id = Contact::lookup_id_by_addr(context, removed_addr);
-        if contact_id != 0 {
-            info!(context, "remove {:?} from chat id={}", contact_id, chat_id);
-            chat::remove_from_chat_contacts_table(context, chat_id, contact_id);
-        }
+    } else if removed_id > 0 {
+        chat::remove_from_chat_contacts_table(context, chat_id, removed_id);
         send_EVENT_CHAT_MODIFIED = true;
     }
 
@@ -1098,6 +1128,66 @@ fn create_or_lookup_group(
     Ok((chat_id, chat_id_blocked))
 }
 
+fn create_or_lookup_mailinglist(
+    context: &Context,
+    allow_creation: bool,
+    create_blocked: Blocked,
+    list_id_header: &str,
+) -> (ChatId, Blocked) {
+    let re = Regex::new(r"^(.*.)<(.*.)>$").unwrap();
+    let (name, listid) = match re.captures(list_id_header) {
+        Some(cap) => (cap[1].trim().to_string(), cap[2].trim().to_string()),
+        None => (
+            list_id_header.trim().to_string(),
+            list_id_header.trim().to_string(),
+        ),
+    };
+
+    chat::get_chat_id_by_mailinglistid(context, &listid).unwrap_or_else(|_e| {
+        if allow_creation {
+            // list does not exist but should be created
+            match create_mailinglist_record(context, &listid, &name, create_blocked) {
+                Ok(chat_id) => {
+                    chat::add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF);
+
+                    // Add the mailing list as "unknown" contact
+                    match add_or_lookup_contact_by_addr(
+                        context,
+                        &Some(name),
+                        &listid,
+                        Origin::IncomingUnknownFrom,
+                        None,
+                    ) {
+                        Ok(list_id_contact) => {
+                            chat::add_to_chat_contacts_table(context, chat_id, list_id_contact);
+                        }
+                        Err(e) => warn!(
+                            context,
+                            "Failed to lookup mailing list contact: {}",
+                            e.to_string()
+                        ),
+                    };
+
+                    (chat_id, create_blocked)
+                }
+                Err(e) => {
+                    warn!(
+                        context,
+                        "Failed to create mailinglist '{}' for grpid={}: {}",
+                        &name,
+                        &listid,
+                        e.to_string()
+                    );
+                    (ChatId::new(0), create_blocked)
+                }
+            }
+        } else {
+            info!(context, "creating list forbidden by caller");
+            (ChatId::new(0), Blocked::Not)
+        }
+    })
+}
+
 /// try extract a grpid from a message-id list header value
 fn extract_grpid(mime_parser: &MimeMessage, headerdef: HeaderDef) -> Option<&str> {
     let header = mime_parser.get(headerdef)?;
@@ -1215,6 +1305,7 @@ fn create_or_lookup_adhoc_group(
     Ok((new_chat_id, create_blocked))
 }
 
+// Insert a group record into the database. Note that this function is also used by create_mailinglist_record() below.
 fn create_group_record(
     context: &Context,
     grpid: impl AsRef<str>,
@@ -1242,7 +1333,7 @@ fn create_group_record(
     {
         warn!(
             context,
-            "Failed to create group '{}' for grpid={}",
+            "Failed to create group or mailinglist '{}' for grpid={}",
             grpname.as_ref(),
             grpid.as_ref()
         );
@@ -1252,7 +1343,7 @@ fn create_group_record(
     let chat_id = ChatId::new(row_id);
     info!(
         context,
-        "Created group '{}' grpid={} as {}",
+        "Created group or mailinglist '{}' grpid={} as {}",
         grpname.as_ref(),
         grpid.as_ref(),
         chat_id
@@ -1260,6 +1351,27 @@ fn create_group_record(
     chat_id
 }
 
+fn create_mailinglist_record(
+    context: &Context,
+    listid: impl AsRef<str>,
+    name: impl AsRef<str>,
+    create_blocked: Blocked,
+) -> Result<ChatId> {
+    let chat_id = create_group_record(
+        context,
+        &listid,
+        &name,
+        create_blocked,
+        VerifiedStatus::Unverified,
+    );
+    let mut chat = Chat::load_from_db(context, chat_id)?;
+
+    chat.param.set(Param::MailingList, "true");
+    chat.update_param(context)?;
+
+    Ok(chat_id)
+}
+
 fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
     /* algorithm:
     - sort normalized, lowercased, e-mail addresses alphabetically
@@ -1581,6 +1693,7 @@ fn dc_add_or_lookup_contacts_by_address_list(
     context: &Context,
     addr_list_raw: &str,
     origin: Origin,
+    list_id_header: Option<&String>,
 ) -> Result<ContactIds> {
     let addrs = match mailparse::addrparse(addr_list_raw) {
         Ok(addrs) => addrs,
@@ -1598,6 +1711,7 @@ fn dc_add_or_lookup_contacts_by_address_list(
                     &info.display_name,
                     &info.addr,
                     origin,
+                    list_id_header,
                 )?);
             }
             mailparse::MailAddr::Group(infos) => {
@@ -1607,6 +1721,7 @@ fn dc_add_or_lookup_contacts_by_address_list(
                         &info.display_name,
                         &info.addr,
                         origin,
+                        list_id_header,
                     )?);
                 }
             }
@@ -1622,6 +1737,7 @@ fn add_or_lookup_contact_by_addr(
     display_name: &Option<String>,
     addr: &str,
     origin: Origin,
+    list_id_header: Option<&String>,
 ) -> Result<u32> {
     if context.is_self_addr(addr)? {
         return Ok(DC_CONTACT_ID_SELF);
@@ -1631,8 +1747,26 @@ fn add_or_lookup_contact_by_addr(
         .map(normalize_name)
         .unwrap_or_default();
 
+    let mut addr = Cow::from(addr);
+    if let Some(list_id) = list_id_header {
+        let list_id = list_id.trim().trim_end_matches('>');
+        let addr_email = EmailAddress::new(&addr)?;
+        let mut addr_domain_parts = addr_email.domain.split('.');
+        let mut list_id_parts = list_id.split('.');
+        if list_id_parts.next_back() == addr_domain_parts.next_back()
+            && list_id_parts.next_back() == addr_domain_parts.next_back()
+        {
+            // list_id was something like "name <name.github.com" (after trimming the last '>')
+            // and addr was something like "notifications@github.com".
+            // addr is not the address of the actual sender but the one of the mailing list.
+            // Add the display name to the addr to make it distinguishable from other people
+            // who sent to the same mailing list.
+            *addr.to_mut() = format!("{} – {}", display_name_normalized, addr);
+        }
+    }
+
     let (row_id, _modified) =
-        Contact::add_or_lookup(context, display_name_normalized, addr, origin)?;
+        Contact::add_or_lookup(context, display_name_normalized, &addr, origin)?;
     ensure!(row_id > 0, "could not add contact: {:?}", addr);
 
     Ok(row_id)
@@ -1657,10 +1791,102 @@ fn dc_create_incoming_rfc724_mid(
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::ChatVisibility;
     use crate::chatlist::Chatlist;
     use crate::message::Message;
     use crate::test_utils::{dummy_context, TestContext};
 
+    static MAILINGLIST: &[u8] = b"From: Max Mustermann <notifications@github.com>\n\
+    To: deltachat/deltachat-core-rust <deltachat-core-rust@noreply.github.com>\n\
+    Subject: [deltachat/deltachat-core-rust] PR run failed\n\
+    Message-ID: <3333@example.org>\n\
+    List-ID: deltachat/deltachat-core-rust <deltachat-core-rust.deltachat.github.com>\n\
+    Precedence: list\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    hello\n";
+
+    static MAILINGLIST2: &[u8] = b"From: Github <notifications@github.com>\n\
+    To: deltachat/deltachat-core-rust <deltachat-core-rust@noreply.github.com>\n\
+    Subject: [deltachat/deltachat-core-rust] PR run failed\n\
+    Message-ID: <3334@example.org>\n\
+    List-ID: deltachat/deltachat-core-rust <deltachat-core-rust.deltachat.github.com>\n\
+    Precedence: list\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    hello back\n";
+
+    static MAILINGLIST3: &[u8] = b"From: Alice <alice@posteo.org>\n\
+    To: delta-dev@codespeak.net\n\
+    Subject: Re: [delta-dev] What's up?\n\
+    Message-ID: <38942@posteo.org>\n\
+    List-ID: \"discussions about and around https://delta.chat developments\" <delta.codespeak.net>\n\
+    Precedence: list\n\
+    Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+    \n\
+    body\n";
+
+    #[test]
+    fn test_mailing_list() {
+        let t = configured_offline_context();
+        t.ctx.set_config(Config::ShowEmails, Some("2")).unwrap();
+
+        dc_receive_imf(&t.ctx, MAILINGLIST, "INBOX", 1, false).unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        let chat_id = chat::create_by_msg_id(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap();
+        let chat = chat::Chat::load_from_db(&t.ctx, chat_id).unwrap();
+
+        assert!(chat.is_mailing_list());
+        assert_eq!(chat.can_send(), false);
+        assert_eq!(chat.name, "deltachat/deltachat-core-rust");
+        println!("{:?}", Contact::load_from_db(&t.ctx, 1));
+        println!("{:?}", chat::get_chat_contacts(&t.ctx, chat_id));
+        assert_eq!(chat::get_chat_contacts(&t.ctx, chat_id).len(), 2);
+
+        dc_receive_imf(&t.ctx, MAILINGLIST2, "INBOX", 1, false).unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.len(), 1);
+        let contacts = Contact::get_all(&t.ctx, 0, None as Option<String>).unwrap();
+        assert_eq!(contacts.len(), 0); // mailing list recipients and senders do not count as "known contacts"
+        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None);
+        let contact1 = Contact::load_from_db(
+            &t.ctx,
+            Message::load_from_db(&t.ctx, msgs[0]).unwrap().from_id,
+        );
+        assert_eq!(
+            contact1.unwrap().get_addr(),
+            "Max Mustermann – notifications@github.com"
+        );
+        let contact2 = Contact::load_from_db(
+            &t.ctx,
+            Message::load_from_db(&t.ctx, msgs[1]).unwrap().from_id,
+        );
+        assert_eq!(
+            contact2.unwrap().get_addr(),
+            "Github – notifications@github.com"
+        );
+    }
+
+    #[test]
+    fn test_mailing_list2() {
+        let t = configured_offline_context();
+        t.ctx.set_config(Config::ShowEmails, Some("2")).unwrap();
+        dc_receive_imf(&t.ctx, MAILINGLIST3, "INBOX", 1, false).unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let chat_id = chat::create_by_msg_id(&t.ctx, chats.get_msg_id(0).unwrap()).unwrap();
+
+        let msgs = chat::get_chat_msgs(&t.ctx, chat_id, 0, None);
+        let contact1 = Contact::load_from_db(
+            &t.ctx,
+            Message::load_from_db(&t.ctx, msgs[0]).unwrap().from_id,
+        );
+        assert_eq!(contact1.unwrap().get_addr(), "alice@posteo.org");
+    }
+
     #[test]
     fn test_hex_hash() {
         let data = "hello world";
@@ -1884,4 +2110,122 @@ mod tests {
         assert_eq!(chat.name, "group with Alice, Bob and Claire");
         assert_eq!(chat::get_chat_contacts(&t.ctx, chat_id).len(), 3);
     }
+
+    #[test]
+    fn test_read_receipt_and_unarchive() {
+        // create alice's account
+        let t = configured_offline_context();
+
+        // create one-to-one with bob, archive one-to-one
+        let bob_id = Contact::create(&t.ctx, "bob", "bob@exampel.org").unwrap();
+        let one2one_id = chat::create_by_contact_id(&t.ctx, bob_id).unwrap();
+        one2one_id
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .unwrap();
+        let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap();
+        assert!(one2one.get_visibility() == ChatVisibility::Archived);
+
+        // create a group with bob, archive group
+        let group_id = chat::create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap();
+        chat::add_contact_to_chat(&t.ctx, group_id, bob_id);
+        assert_eq!(chat::get_chat_msgs(&t.ctx, group_id, 0, None).len(), 0);
+        group_id
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .unwrap();
+        let group = Chat::load_from_db(&t.ctx, group_id).unwrap();
+        assert!(group.get_visibility() == ChatVisibility::Archived);
+
+        // everything archived, chatlist should be empty
+        assert_eq!(
+            Chatlist::try_load(&t.ctx, DC_GCL_NO_SPECIALS, None, None)
+                .unwrap()
+                .len(),
+            0
+        );
+
+        // send a message to group with bob
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: alice@example.org\n\
+                 To: bob@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Chat-Group-ID: {}\n\
+                 Chat-Group-Name: foo\n\
+                 Chat-Disposition-Notification-To: alice@example.org\n\
+                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+                 \n\
+                 hello\n",
+                group.grpid, group.grpid
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, group_id, 0, None);
+        assert_eq!(msgs.len(), 1);
+        let msg_id = msgs.first().unwrap();
+        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap();
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert_eq!(msg.text.unwrap(), "hello");
+        assert_eq!(msg.state, MessageState::OutDelivered);
+        let group = Chat::load_from_db(&t.ctx, group_id).unwrap();
+        assert!(group.get_visibility() == ChatVisibility::Normal);
+
+        // bob sends a read receipt to the group
+        dc_receive_imf(
+            &t.ctx,
+            format!(
+                "From: bob@example.org\n\
+                 To: alice@example.org\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.org>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;bob@example.org\n\
+                 Final-Recipient: rfc822;bob@example.org\n\
+                 Original-Message-ID: <Gr.{}.12345678901@example.org>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n\
+                 \n\
+                 --SNIPP--",
+                group.grpid
+            )
+            .as_bytes(),
+            "INBOX",
+            1,
+            false,
+        )
+        .unwrap();
+        assert_eq!(chat::get_chat_msgs(&t.ctx, group_id, 0, None).len(), 1);
+        let msg = message::Message::load_from_db(&t.ctx, msg_id.clone()).unwrap();
+        assert_eq!(msg.state, MessageState::OutMdnRcvd);
+
+        // check, the read-receipt has not unarchived the one2one
+        assert_eq!(
+            Chatlist::try_load(&t.ctx, DC_GCL_NO_SPECIALS, None, None)
+                .unwrap()
+                .len(),
+            1
+        );
+        let one2one = Chat::load_from_db(&t.ctx, one2one_id).unwrap();
+        assert!(one2one.get_visibility() == ChatVisibility::Archived);
+    }
 }

src/dc_tools.rs

@@ -12,7 +12,7 @@ use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, ensure, Error};
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
@@ -494,24 +494,25 @@ impl FromStr for EmailAddress {
         ensure!(!input.is_empty(), "empty string is not valid");
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        ensure!(parts.len() > 1, "missing '@' character");
-        let local = parts[1];
-        let domain = parts[0];
+        match &parts[..] {
+            [domain, local] => {
+                ensure!(
+                    !local.is_empty(),
+                    "empty string is not valid for local part"
+                );
+                ensure!(domain.len() > 3, "domain is too short");
 
-        ensure!(
-            !local.is_empty(),
-            "empty string is not valid for local part"
-        );
-        ensure!(domain.len() > 3, "domain is too short");
+                let dot = domain.find('.');
+                ensure!(dot.is_some(), "invalid domain");
+                ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
 
-        let dot = domain.find('.');
-        ensure!(dot.is_some(), "invalid domain");
-        ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
-
-        Ok(EmailAddress {
-            local: local.to_string(),
-            domain: domain.to_string(),
-        })
+                Ok(EmailAddress {
+                    local: (*local).to_string(),
+                    domain: (*domain).to_string(),
+                })
+            }
+            _ => bail!("missing '@' character"),
+        }
     }
 }
 

src/dehtml.rs

@@ -3,7 +3,6 @@
 //! A module to remove HTML tags from the email text
 
 use lazy_static::lazy_static;
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 lazy_static! {
@@ -35,6 +34,7 @@ pub fn dehtml(buf: &str) -> String {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
+    reader.check_end_names(false);
 
     let mut buf = Vec::new();
 
@@ -225,4 +225,23 @@ mod tests {
             "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
         );
     }
+
+    #[test]
+    fn test_unclosed_tags() {
+        let input = r##"
+        <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'
+        'http://www.w3.org/TR/html4/loose.dtd'>
+        <html>
+        <head>
+        <title>Hi</title>
+        <meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'>						
+        </head>
+        <body>
+        lots of text
+        </body>
+        </html>
+        "##;
+        let txt = dehtml(input);
+        assert_eq!(txt.trim(), "lots of text");
+    }
 }

src/e2ee.rs

@@ -200,15 +200,13 @@ fn load_or_generate_self_public_key(
     self_addr: impl AsRef<str>,
 ) -> Result<SignedPublicKey> {
     if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
-        return SignedPublicKey::try_from(key)
-            .map_err(|_| Error::Message("Not a public key".into()));
+        return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key"));
     }
     let _guard = context.generating_key_mutex.lock().unwrap();
 
     // Check again in case the key was generated while we were waiting for the lock.
     if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
-        return SignedPublicKey::try_from(key)
-            .map_err(|_| Error::Message("Not a public key".into()));
+        return SignedPublicKey::try_from(key).map_err(|_| format_err!("Not a public key"));
     }
 
     let start = std::time::Instant::now();

src/error.rs

@@ -1,189 +1,13 @@
 //! # Error handling
 
-use lettre_email::mime;
+pub use anyhow::{bail, ensure, format_err, Error, Result};
 
-#[derive(Debug, Fail)]
-pub enum Error {
-    #[fail(display = "{:?}", _0)]
-    Failure(failure::Error),
-
-    #[fail(display = "SQL error: {:?}", _0)]
-    SqlError(#[cause] crate::sql::Error),
-
-    #[fail(display = "{:?}", _0)]
-    Io(std::io::Error),
-
-    #[fail(display = "{:?}", _0)]
-    Message(String),
-
-    #[fail(display = "{:?}", _0)]
-    MessageWithCause(String, #[cause] failure::Error, failure::Backtrace),
-
-    #[fail(display = "{:?}", _0)]
-    Image(image_meta::ImageError),
-
-    #[fail(display = "{:?}", _0)]
-    Utf8(std::str::Utf8Error),
-
-    #[fail(display = "PGP: {:?}", _0)]
-    Pgp(pgp::errors::Error),
-
-    #[fail(display = "Base64Decode: {:?}", _0)]
-    Base64Decode(base64::DecodeError),
-
-    #[fail(display = "{:?}", _0)]
-    FromUtf8(std::string::FromUtf8Error),
-
-    #[fail(display = "{}", _0)]
-    BlobError(#[cause] crate::blob::BlobError),
-
-    #[fail(display = "Invalid Message ID.")]
-    InvalidMsgId,
-
-    #[fail(display = "Watch folder not found {:?}", _0)]
-    WatchFolderNotFound(String),
-
-    #[fail(display = "Invalid Email: {:?}", _0)]
-    MailParseError(#[cause] mailparse::MailParseError),
-
-    #[fail(display = "Building invalid Email: {:?}", _0)]
-    LettreError(#[cause] lettre_email::error::Error),
-
-    #[fail(display = "SMTP error: {:?}", _0)]
-    SmtpError(#[cause] async_smtp::error::Error),
-
-    #[fail(display = "FromStr error: {:?}", _0)]
-    FromStr(#[cause] mime::FromStrError),
-
-    #[fail(display = "Not Configured")]
-    NotConfigured,
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
-impl From<crate::sql::Error> for Error {
-    fn from(err: crate::sql::Error) -> Error {
-        Error::SqlError(err)
-    }
-}
-
-impl From<base64::DecodeError> for Error {
-    fn from(err: base64::DecodeError) -> Error {
-        Error::Base64Decode(err)
-    }
-}
-
-impl From<failure::Error> for Error {
-    fn from(err: failure::Error) -> Error {
-        Error::Failure(err)
-    }
-}
-
-impl From<std::io::Error> for Error {
-    fn from(err: std::io::Error) -> Error {
-        Error::Io(err)
-    }
-}
-
-impl From<std::str::Utf8Error> for Error {
-    fn from(err: std::str::Utf8Error) -> Error {
-        Error::Utf8(err)
-    }
-}
-
-impl From<image_meta::ImageError> for Error {
-    fn from(err: image_meta::ImageError) -> Error {
-        Error::Image(err)
-    }
-}
-
-impl From<pgp::errors::Error> for Error {
-    fn from(err: pgp::errors::Error) -> Error {
-        Error::Pgp(err)
-    }
-}
-
-impl From<std::string::FromUtf8Error> for Error {
-    fn from(err: std::string::FromUtf8Error) -> Error {
-        Error::FromUtf8(err)
-    }
-}
-
-impl From<crate::blob::BlobError> for Error {
-    fn from(err: crate::blob::BlobError) -> Error {
-        Error::BlobError(err)
-    }
-}
-
-impl From<crate::message::InvalidMsgId> for Error {
-    fn from(_err: crate::message::InvalidMsgId) -> Error {
-        Error::InvalidMsgId
-    }
-}
-
-impl From<crate::key::SaveKeyError> for Error {
-    fn from(err: crate::key::SaveKeyError) -> Error {
-        Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new())
-    }
-}
-
-impl From<crate::pgp::PgpKeygenError> for Error {
-    fn from(err: crate::pgp::PgpKeygenError) -> Error {
-        Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new())
-    }
-}
-
-impl From<mailparse::MailParseError> for Error {
-    fn from(err: mailparse::MailParseError) -> Error {
-        Error::MailParseError(err)
-    }
-}
-
-impl From<lettre_email::error::Error> for Error {
-    fn from(err: lettre_email::error::Error) -> Error {
-        Error::LettreError(err)
-    }
-}
-
-impl From<mime::FromStrError> for Error {
-    fn from(err: mime::FromStrError) -> Error {
-        Error::FromStr(err)
-    }
-}
-
-#[macro_export]
-macro_rules! bail {
-    ($e:expr) => {
-        return Err($crate::error::Error::Message($e.to_string()));
-    };
-    ($fmt:expr, $($arg:tt)+) => {
-        return Err($crate::error::Error::Message(format!($fmt, $($arg)+)));
-    };
-}
-
-#[macro_export]
-macro_rules! format_err {
-    ($e:expr) => {
-        $crate::error::Error::Message($e.to_string());
-    };
-    ($fmt:expr, $($arg:tt)+) => {
-        $crate::error::Error::Message(format!($fmt, $($arg)+));
-    };
-}
-
-#[macro_export(local_inner_macros)]
-macro_rules! ensure {
-    ($cond:expr, $e:expr) => {
-        if !($cond) {
-            bail!($e);
-        }
-    };
-    ($cond:expr, $fmt:expr, $($arg:tt)+) => {
-        if !($cond) {
-            bail!($fmt, $($arg)+);
-        }
-    };
-}
+// #[fail(display = "Invalid Message ID.")]
+// InvalidMsgId,
+// #[fail(display = "Watch folder not found {:?}", _0)]
+// WatchFolderNotFound(String),
+// #[fail(display = "Not Configured")]
+// NotConfigured,
 
 #[macro_export]
 macro_rules! ensure_eq {

src/events.rs

@@ -207,4 +207,16 @@ pub enum Event {
     /// @param data2 (int) contact_id
     #[strum(props(id = "2062"))]
     SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 },
+
+    /// This event is sent for each contact added to a chat.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    #[strum(props(id = "2063"))]
+    MemberAdded { chat_id: ChatId, contact_id: u32 },
+
+    /// This event is sent for each contact removed from a chat.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    #[strum(props(id = "2064"))]
+    MemberRemoved { chat_id: ChatId, contact_id: u32 },
 }

src/export_chat.rs

@@ -1,345 +0,0 @@
-// use crate::dc_tools::*;
-use crate::chat::*;
-use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
-use crate::contact::*;
-use crate::context::Context;
-use crate::error::Error;
-use crate::message::*;
-use std::collections::HashMap;
-use std::fs::File;
-use std::io::prelude::*;
-use std::path::Path;
-use zip::write::FileOptions;
-
-#[derive(Debug)]
-pub struct ExportChatResult {
-    html: String,
-    referenced_blobs: Vec<String>,
-}
-
-struct ContactInfo {
-    name: String,
-    initial: String,
-    color: String,
-    profile_img: Option<String>,
-}
-
-pub fn pack_exported_chat(
-    context: &Context,
-    artifact: ExportChatResult,
-    filename: &str,
-) -> zip::result::ZipResult<()> {
-    let path = std::path::Path::new(filename);
-    let file = std::fs::File::create(&path).unwrap();
-
-    let mut zip = zip::ZipWriter::new(file);
-
-    zip.start_file("index.html", Default::default())?;
-    zip.write_all(artifact.html.as_bytes())?;
-
-    zip.start_file("styles.css", Default::default())?;
-    zip.write_all(include_bytes!("../assets/exported-chat.css"))?;
-
-    zip.add_directory("blobs/", Default::default())?;
-
-    let options = FileOptions::default();
-    for blob_name in artifact.referenced_blobs {
-        let path = context.get_blobdir().join(&blob_name);
-
-        // println!("adding file {:?} as {:?} ...", path, &blob_name);
-        zip.start_file_from_path(Path::new(&format!("blobs/{}", &blob_name)), options)?;
-        let mut f = File::open(path)?;
-
-        let mut buffer = Vec::new();
-        f.read_to_end(&mut buffer)?;
-        zip.write_all(&*buffer)?;
-        buffer.clear();
-    }
-
-    zip.finish()?;
-    Ok(())
-}
-
-pub fn export_chat(context: &Context, chat_id: ChatId) -> ExportChatResult {
-    let mut blobs = Vec::new();
-    let mut chat_author_ids = Vec::new();
-    // get all messages
-    let messages: Vec<std::result::Result<Message, Error>> =
-        get_chat_msgs(context, chat_id, 0, None)
-            .into_iter()
-            .map(|msg_id| Message::load_from_db(context, msg_id))
-            .collect();
-    // push all referenced blobs and populate contactid list
-    for message in &messages {
-        if let Ok(msg) = &message {
-            let filename = msg.get_filename();
-            if let Some(file) = filename {
-                // push referenced blobs (attachments)
-                blobs.push(file);
-            }
-            chat_author_ids.push(msg.from_id);
-        }
-    }
-    // deduplicate contact list and load the contacts
-    chat_author_ids.dedup();
-    // chache information about the authors
-    let mut chat_authors: HashMap<u32, ContactInfo> = HashMap::new();
-    chat_authors.insert(
-        0,
-        ContactInfo {
-            name: "Err: Contact not found".to_owned(),
-            initial: "#".to_owned(),
-            profile_img: None,
-            color: "grey".to_owned(),
-        },
-    );
-    for author_id in chat_author_ids {
-        let contact = Contact::get_by_id(context, author_id);
-        if let Ok(c) = contact {
-            let profile_img_path: String;
-            if let Some(path) = c.get_profile_image(context) {
-                profile_img_path = path
-                    .file_name()
-                    .unwrap_or_else(|| std::ffi::OsStr::new(""))
-                    .to_str()
-                    .unwrap()
-                    .to_owned();
-                // push referenced blobs (avatars)
-                blobs.push(profile_img_path.clone());
-            } else {
-                profile_img_path = "".to_owned();
-            }
-            chat_authors.insert(
-                author_id,
-                ContactInfo {
-                    name: c.get_display_name().to_owned(),
-                    initial: "#".to_owned(), // TODO
-                    profile_img: match profile_img_path != "" {
-                        true => Some(profile_img_path),
-                        false => None,
-                    },
-                    color: "rgb(18, 126, 208)".to_owned(), // TODO
-                },
-            );
-        }
-    }
-
-    // run message_to_html for each message and generate the html that way
-    let mut html_messages: Vec<String> = Vec::new();
-    for message in messages {
-        if let Ok(msg) = message {
-            html_messages.push(message_to_html(&chat_authors, msg, context));
-        } else {
-            html_messages.push(format!(
-                r#"<li>
-                        <div class='message error'>
-                            <div class="msg-container">
-                                <div class="msg-body">
-                                    <div dir="auto" class="text">{:?}</div>
-                                </div>
-                            </div>
-                        </div>
-                    </li>"#,
-                message.unwrap_err()
-            ));
-        }
-    }
-
-    // todo chat image, chat name and so on..
-    let chat = Chat::load_from_db(context, chat_id).unwrap();
-    let chat_avatar = match chat.get_profile_image(context) {
-        Some(img) => {
-            let path = img
-                .file_name()
-                .unwrap_or_else(|| std::ffi::OsStr::new(""))
-                .to_str()
-                .unwrap()
-                .to_owned();
-            blobs.push(path.clone());
-            format!("<img class=\"avatar\" src=\"blobs/{}\" />", path)
-        }
-        None => format!(
-            "<div class=\"avatar text-avatar\" style=\"background-color:#{:#}\">{}</div>",
-            chat.get_color(context),
-            chat.get_name().chars().next().unwrap()
-        ),
-    };
-
-    // todo option to export locations as kml?
-
-    // todo export message infos and save them to txt files
-    // (those can be linked from the messages, they are stored in msg_info/[msg-id].txt)
-
-    blobs.dedup();
-    ExportChatResult {
-        html: format!(
-            "<html>\
-             <head>\
-             <title>{chat_name}</title>\
-             <link rel=\"stylesheet\" href=\"styles.css\" type=\"text/css\">\
-             </head>\
-             <body>\
-             <div class=\"header\">\
-             {chat_avatar}\
-             <div class=\"name\">{chat_name}</div>\
-             </div>\
-             <div class=\"message-list-and-composer__message-list\">\
-             <div id=\"message-list\">\
-             <ul>{messages}</ul>\
-             </div>\
-             </div>\
-             </body>\
-             </html>",
-            chat_name = chat.get_name(),
-            chat_avatar = chat_avatar,
-            messages = html_messages.join("")
-        ),
-        referenced_blobs: blobs,
-    }
-}
-
-fn message_to_html(
-    author_cache: &HashMap<u32, ContactInfo>,
-    message: Message,
-    context: &Context,
-) -> String {
-    let author: &ContactInfo = {
-        if let Some(c) = author_cache.get(&message.get_from_id()) {
-            c
-        } else {
-            author_cache.get(&0).unwrap()
-        }
-    };
-
-    let avatar: String = {
-        if let Some(profile_img) = &author.profile_img {
-            format!(
-                "<div class=\"author-avatar\">\
-                 <img \
-                 alt=\"{author_name}\"\
-                 src=\"blobs/{author_avatar_src}\"\
-                 />\
-                 </div>",
-                author_name = author.name,
-                author_avatar_src = profile_img
-            )
-        } else {
-            format!(
-                "<div class=\"author-avatar default\" alt=\"{name}\">\
-                 <div class=\"label\" style=\"background-color: {color}\">\
-                 {initial}\
-                 </div>\
-                 </div>",
-                name = author.name,
-                initial = author.initial,
-                color = author.color
-            )
-        }
-    };
-
-    // save and refernce message source code somehow?
-
-    let has_text = message.get_text().is_some() && !message.get_text().unwrap().is_empty();
-
-    let attachment = match message.get_file(context) {
-        None => "".to_owned(),
-        Some(file) => {
-            let modifier_class = if has_text { "content-below" } else { "" };
-            let filename = file
-                .file_name()
-                .unwrap_or_else(|| std::ffi::OsStr::new(""))
-                .to_str()
-                .unwrap()
-                .to_owned();
-            match message.get_viewtype() {
-                Viewtype::Audio => {
-                    format!("<audio \
-                    controls \
-                    class=\"message-attachment-audio {}\"> \
-                    <source src=\"blobs/{}\" /> \
-                  </audio>", modifier_class ,filename)
-                },
-                Viewtype::Gif | Viewtype::Image | Viewtype::Sticker => {
-                    format!("<a \
-                        href=\"blobs/{filename}\" \
-                        role=\"button\" \
-                        class=\"message-attachment-media {modifier_class}\"> \
-                        <img className='attachment-content' src=\"blobs/{filename}\" /> \
-                    </a>", modifier_class=modifier_class, filename=filename)
-                },
-                Viewtype::Video => {
-                    format!("<a \
-                    href=\"blobs/{filename}\" \
-                    role=\"button\" \
-                    class=\"message-attachment-media {modifier_class}\"> \
-                    <video className='attachment-content' src=\"blobs/{filename}\" controls=\"true\" /> \
-                </a>", modifier_class=modifier_class, filename=filename)
-                },
-                _ => {
-                    format!("<div class=\"message-attachment-generic {modifier_class}\">\
-                        <div class=\"file-icon\">\
-                            <div class=\"file-extension\">\
-                            {extension} \
-                            </div>\
-                        </div>\
-                        <div className=\"text-part\">\
-                        <a href=\"blobs/{filename}\" className=\"name\">{filename}</a>\
-                        <div className=\"size\">{filesize}</div>\
-                        </div>\
-                    </div>",
-                    modifier_class=modifier_class,
-                    filename=filename,
-                    filesize=message.get_filebytes(&context) /* todo human readable file size*/,
-                    extension=file.extension().unwrap_or_else(|| std::ffi::OsStr::new("")).to_str().unwrap().to_owned())
-                }
-            }
-        }
-    };
-
-    format!(
-        "<li>\
-         <div class=\"message {direction}\">\
-         {avatar}\
-         <div class=\"msg-container\">\
-         <span class=\"author\" style=\"color: {author_color};\">{author_name}</span>\
-         <div class=\"msg-body\">\
-         {attachment}
-         <div dir=\"auto\" class=\"text\">\
-         {content}\
-         </div>\
-         <div class=\"metadata {with_image_no_caption}\">\
-         {encryption}\
-         <span class=\"date date--{direction}\" title=\"{full_time}\">{relative_time}</span>\
-         <span class=\"spacer\"></span>\
-         </div>\
-         </div>\
-         </div>\
-         <div>\
-         </li>",
-        direction = match message.from_id == DC_CONTACT_ID_SELF {
-            true => "outgoing",
-            false => "incoming",
-        },
-        avatar = avatar,
-        author_name = author.name,
-        author_color = author.color,
-        attachment = attachment,
-        content = message.get_text().unwrap_or_else(|| "".to_owned()),
-        with_image_no_caption = if !has_text && message.get_viewtype() == Viewtype::Image {
-            "with-image-no-caption"
-        } else {
-            ""
-        },
-        encryption = match message.get_showpadlock() {
-            true => r#"<div aria-label="Encryption padlock" class="padlock-icon"></div>"#,
-            false => "",
-        },
-        full_time = "Tue, Feb 25, 2020 3:49 PM", // message.get_timestamp() ? // todo
-        relative_time = "Tue 3:49 PM"            // todo
-    )
-
-    // todo link to raw message data
-    // todo link to message info
-}
-
-//TODO tests

src/imap/idle.rs

@@ -15,28 +15,22 @@ use super::session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[cause] async_imap::error::Error),
+    #[error("IMAP IDLE protocol failed to init/complete")]
+    IdleProtocolFailed(#[from] async_imap::error::Error),
 
-    #[fail(display = "IMAP IDLE protocol timed out")]
-    IdleTimeout(#[cause] async_std::future::TimeoutError),
+    #[error("IMAP IDLE protocol timed out")]
+    IdleTimeout(#[from] async_std::future::TimeoutError),
 
-    #[fail(display = "IMAP server does not have IDLE capability")]
+    #[error("IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[fail(display = "IMAP select folder error")]
-    SelectFolderError(#[cause] select_folder::Error),
+    #[error("IMAP select folder error")]
+    SelectFolderError(#[from] select_folder::Error),
 
-    #[fail(display = "Setup handle error")]
-    SetupHandleError(#[cause] super::Error),
-}
-
-impl From<select_folder::Error> for Error {
-    fn from(err: select_folder::Error) -> Error {
-        Error::SelectFolderError(err)
-    }
+    #[error("Setup handle error")]
+    SetupHandleError(#[from] super::Error),
 }
 
 #[derive(Debug)]
@@ -71,9 +65,7 @@ impl Imap {
                 return Err(Error::IdleAbilityMissing);
             }
 
-            self.setup_handle_if_needed(context)
-                .await
-                .map_err(Error::SetupHandleError)?;
+            self.setup_handle_if_needed(context).await?;
 
             self.select_folder(context, watch_folder.clone()).await?;
 
@@ -84,9 +76,7 @@ impl Imap {
                     // BEWARE: If you change the Secure branch you
                     // typically also need to change the Insecure branch.
                     IdleHandle::Secure(mut handle) => {
-                        if let Err(err) = handle.init().await {
-                            return Err(Error::IdleProtocolFailed(err));
-                        }
+                        handle.init().await?;
 
                         let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
                         *self.interrupt.lock().await = Some(interrupt);
@@ -141,9 +131,7 @@ impl Imap {
                         }
                     }
                     IdleHandle::Insecure(mut handle) => {
-                        if let Err(err) = handle.init().await {
-                            return Err(Error::IdleProtocolFailed(err));
-                        }
+                        handle.init().await?;
 
                         let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
                         *self.interrupt.lock().await = Some(interrupt);

src/imap/mod.rs

@@ -39,78 +39,48 @@ use session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "IMAP Connect without configured params")]
+    #[error("IMAP Connect without configured params")]
     ConnectWithoutConfigure,
 
-    #[fail(display = "IMAP Connection Failed params: {}", _0)]
+    #[error("IMAP Connection Failed params: {0}")]
     ConnectionFailed(String),
 
-    #[fail(display = "IMAP No Connection established")]
+    #[error("IMAP No Connection established")]
     NoConnection,
 
-    #[fail(display = "IMAP Could not get OAUTH token")]
+    #[error("IMAP Could not get OAUTH token")]
     OauthError,
 
-    #[fail(display = "IMAP Could not login as {}", _0)]
+    #[error("IMAP Could not login as {0}")]
     LoginFailed(String),
 
-    #[fail(display = "IMAP Could not fetch")]
-    FetchFailed(#[cause] async_imap::error::Error),
+    #[error("IMAP Could not fetch")]
+    FetchFailed(#[from] async_imap::error::Error),
 
-    #[fail(display = "IMAP operation attempted while it is torn down")]
+    #[error("IMAP operation attempted while it is torn down")]
     InTeardown,
 
-    #[fail(display = "IMAP operation attempted while it is torn down")]
-    SqlError(#[cause] crate::sql::Error),
+    #[error("IMAP operation attempted while it is torn down")]
+    SqlError(#[from] crate::sql::Error),
 
-    #[fail(display = "IMAP got error from elsewhere")]
-    WrappedError(#[cause] crate::error::Error),
+    #[error("IMAP got error from elsewhere")]
+    WrappedError(#[from] crate::error::Error),
 
-    #[fail(display = "IMAP select folder error")]
-    SelectFolderError(#[cause] select_folder::Error),
+    #[error("IMAP select folder error")]
+    SelectFolderError(#[from] select_folder::Error),
 
-    #[fail(display = "Mail parse error")]
-    MailParseError(#[cause] mailparse::MailParseError),
+    #[error("Mail parse error")]
+    MailParseError(#[from] mailparse::MailParseError),
 
-    #[fail(display = "No mailbox selected, folder: {:?}", _0)]
+    #[error("No mailbox selected, folder: {0}")]
     NoMailbox(String),
 
-    #[fail(display = "IMAP other error: {:?}", _0)]
+    #[error("IMAP other error: {0}")]
     Other(String),
 }
 
-impl From<crate::sql::Error> for Error {
-    fn from(err: crate::sql::Error) -> Error {
-        Error::SqlError(err)
-    }
-}
-
-impl From<crate::error::Error> for Error {
-    fn from(err: crate::error::Error) -> Error {
-        Error::WrappedError(err)
-    }
-}
-
-impl From<Error> for crate::error::Error {
-    fn from(err: Error) -> crate::error::Error {
-        crate::error::Error::Message(err.to_string())
-    }
-}
-
-impl From<select_folder::Error> for Error {
-    fn from(err: select_folder::Error) -> Error {
-        Error::SelectFolderError(err)
-    }
-}
-
-impl From<mailparse::MailParseError> for Error {
-    fn from(err: mailparse::MailParseError) -> Error {
-        Error::MailParseError(err)
-    }
-}
-
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapActionResult {
     Failed,
@@ -1408,7 +1378,11 @@ fn prefetch_should_download(
         .get_header_value(HeaderDef::From_)
         .unwrap_or_default();
 
-    let (_contact_id, blocked_contact, origin) = from_field_to_contact_id(context, &from_field)?;
+    let (_contact_id, blocked_contact, origin) = from_field_to_contact_id(
+        context,
+        &from_field,
+        headers.get_header_value(HeaderDef::ListId).as_ref(),
+    )?;
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message

src/imap/select_folder.rs

@@ -4,21 +4,21 @@ use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "IMAP Could not obtain imap-session object.")]
+    #[error("IMAP Could not obtain imap-session object.")]
     NoSession,
 
-    #[fail(display = "IMAP Connection Lost or no connection established")]
+    #[error("IMAP Connection Lost or no connection established")]
     ConnectionLost,
 
-    #[fail(display = "IMAP Folder name invalid: {:?}", _0)]
+    #[error("IMAP Folder name invalid: {0}")]
     BadFolderName(String),
 
-    #[fail(display = "IMAP close/expunge failed: {}", _0)]
-    CloseExpungeFailed(#[cause] async_imap::error::Error),
+    #[error("IMAP close/expunge failed")]
+    CloseExpungeFailed(#[from] async_imap::error::Error),
 
-    #[fail(display = "IMAP other error: {:?}", _0)]
+    #[error("IMAP other error: {0}")]
     Other(String),
 }
 

src/job.rs

@@ -9,6 +9,9 @@ use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
+use async_smtp::smtp::response::Category;
+use async_smtp::smtp::response::Code;
+use async_smtp::smtp::response::Detail;
 use async_std::task;
 
 use crate::blob::BlobObject;
@@ -19,7 +22,7 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::{Context, PerformJobsNeeded};
 use crate::dc_tools::*;
-use crate::error::{Error, Result};
+use crate::error::{bail, ensure, format_err, Error, Result};
 use crate::events::Event;
 use crate::imap::*;
 use crate::imex::*;
@@ -200,8 +203,37 @@ impl Job {
                 self.pending_error = Some(err.to_string());
 
                 let res = match err {
-                    async_smtp::smtp::error::Error::Permanent(_) => {
-                        Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                    async_smtp::smtp::error::Error::Permanent(ref response) => {
+                        match response.code {
+                            // Sometimes servers send a permanent error when actually it is a temporary error
+                            // For documentation see https://tools.ietf.org/html/rfc3463
+
+                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                            Code {
+                                category: Category::MailSystem,
+                                detail: Detail::Zero,
+                                ..
+                            } => Status::RetryLater,
+
+                            _ => {
+                                // If we do not retry, add an info message to the chat
+                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
+                                match Message::load_from_db(context, MsgId::new(self.foreign_id)) {
+                                    Ok(message) => chat::add_info_msg(
+                                        context,
+                                        message.chat_id,
+                                        err.to_string(),
+                                    ),
+                                    Err(e) => warn!(
+                                        context,
+                                        "couldn't load chat_id to inform user about SMTP error: {}",
+                                        e
+                                    ),
+                                };
+
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            }
+                        }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
                         // We got a transient 4xx response from SMTP server.
@@ -227,7 +259,7 @@ impl Job {
                 // Local error, job is invalid, do not retry.
                 smtp.disconnect();
                 warn!(context, "SMTP job is invalid: {}", err);
-                Status::Finished(Err(Error::SmtpError(err)))
+                Status::Finished(Err(err.into()))
             }
             Err(crate::smtp::send::Error::NoTransport) => {
                 // Should never happen.

src/job_thread.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, Condvar, Mutex};
 
 use crate::context::Context;
-use crate::error::{Error, Result};
+use crate::error::{format_err, Result};
 use crate::imap::Imap;
 
 #[derive(Debug)]
@@ -99,25 +99,21 @@ impl JobThread {
 
     async fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
         let prefix = format!("{}-fetch", self.name);
-        match self.imap.connect_configured(context) {
-            Ok(()) => {
-                if let Some(watch_folder) = self.get_watch_folder(context) {
-                    let start = std::time::Instant::now();
-                    info!(context, "{} started...", prefix);
-                    let res = self
-                        .imap
-                        .fetch(context, &watch_folder)
-                        .await
-                        .map_err(Into::into);
-                    let elapsed = start.elapsed().as_millis();
-                    info!(context, "{} done in {:.3} ms.", prefix, elapsed);
+        self.imap.connect_configured(context)?;
+        if let Some(watch_folder) = self.get_watch_folder(context) {
+            let start = std::time::Instant::now();
+            info!(context, "{} started...", prefix);
+            let res = self
+                .imap
+                .fetch(context, &watch_folder)
+                .await
+                .map_err(Into::into);
+            let elapsed = start.elapsed().as_millis();
+            info!(context, "{} done in {:.3} ms.", prefix, elapsed);
 
-                    res
-                } else {
-                    Err(Error::WatchFolderNotFound("not-set".to_string()))
-                }
-            }
-            Err(err) => Err(crate::error::Error::Message(err.to_string())),
+            res
+        } else {
+            Err(format_err!("WatchFolder not found: not-set"))
         }
     }
 

src/key.rs

@@ -18,24 +18,12 @@ pub use crate::pgp::KeyPair;
 pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
 /// Error type for deltachat key handling.
-#[derive(Fail, Debug)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Could not decode base64")]
-    Base64Decode(#[cause] base64::DecodeError, failure::Backtrace),
-    #[fail(display = "rPGP error: {}", _0)]
-    PgpError(#[cause] pgp::errors::Error, failure::Backtrace),
-}
-
-impl From<base64::DecodeError> for Error {
-    fn from(err: base64::DecodeError) -> Error {
-        Error::Base64Decode(err, failure::Backtrace::new())
-    }
-}
-
-impl From<pgp::errors::Error> for Error {
-    fn from(err: pgp::errors::Error) -> Error {
-        Error::PgpError(err, failure::Backtrace::new())
-    }
+    #[error("Could not decode base64")]
+    Base64Decode(#[from] base64::DecodeError),
+    #[error("rPGP error: {0}")]
+    PgpError(#[from] pgp::errors::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -316,21 +304,19 @@ pub enum KeyPairUse {
 }
 
 /// Error saving a keypair to the database.
-#[derive(Fail, Debug)]
-#[fail(display = "SaveKeyError: {}", message)]
+#[derive(Debug, thiserror::Error)]
+#[error("SaveKeyError: {message}")]
 pub struct SaveKeyError {
     message: String,
-    #[cause]
-    cause: failure::Error,
-    backtrace: failure::Backtrace,
+    #[source]
+    cause: anyhow::Error,
 }
 
 impl SaveKeyError {
-    fn new(message: impl Into<String>, cause: impl Into<failure::Error>) -> Self {
+    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
         Self {
             message: message.into(),
             cause: cause.into(),
-            backtrace: failure::Backtrace::new(),
         }
     }
 }

src/lib.rs

@@ -2,8 +2,6 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
 #![allow(clippy::match_bool)]
 
-#[macro_use]
-extern crate failure_derive;
 #[macro_use]
 extern crate num_derive;
 #[macro_use]
@@ -36,7 +34,6 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
-pub mod export_chat;
 mod imap;
 pub mod imex;
 #[macro_use]

src/location.rs

@@ -1,7 +1,6 @@
 //! Location handling
 
 use bitflags::bitflags;
-use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
@@ -9,7 +8,7 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::{self, job_action_exists, job_add, Job};
 use crate::message::{Message, MsgId};
@@ -123,9 +122,9 @@ impl Kml {
                 }
             } else if self.tag.contains(KmlTag::COORDINATES) {
                 let parts = val.splitn(2, ',').collect::<Vec<_>>();
-                if parts.len() == 2 {
-                    self.curr.longitude = parts[0].parse().unwrap_or_default();
-                    self.curr.latitude = parts[1].parse().unwrap_or_default();
+                if let [longitude, latitude] = &parts[..] {
+                    self.curr.longitude = longitude.parse().unwrap_or_default();
+                    self.curr.latitude = latitude.parse().unwrap_or_default();
                 }
             }
         }

src/lot.rs

@@ -40,11 +40,11 @@ impl Lot {
     }
 
     pub fn get_text1(&self) -> Option<&str> {
-        self.text1.as_ref().map(|s| s.as_str())
+        self.text1.as_deref()
     }
 
     pub fn get_text2(&self) -> Option<&str> {
-        self.text2.as_ref().map(|s| s.as_str())
+        self.text2.as_deref()
     }
 
     pub fn get_text1_meaning(&self) -> Meaning {

src/message.rs

@@ -3,7 +3,6 @@
 use std::path::{Path, PathBuf};
 
 use deltachat_derive::{FromSql, ToSql};
-use failure::Fail;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
@@ -12,7 +11,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::error::{ensure, Error};
 use crate::events::Event;
 use crate::job::*;
 use crate::lot::{Lot, LotState, Meaning};
@@ -170,7 +169,7 @@ impl rusqlite::types::ToSql for MsgId {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         if self.0 <= DC_MSG_ID_LAST_SPECIAL {
             return Err(rusqlite::Error::ToSqlConversionFailure(Box::new(
-                InvalidMsgId.compat(),
+                InvalidMsgId,
             )));
         }
         let val = rusqlite::types::Value::Integer(self.0 as i64);
@@ -198,8 +197,8 @@ impl rusqlite::types::FromSql for MsgId {
 /// This usually occurs when trying to use a message ID of
 /// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not
 /// possible.
-#[derive(Debug, Fail)]
-#[fail(display = "Invalid Message ID.")]
+#[derive(Debug, thiserror::Error)]
+#[error("Invalid Message ID.")]
 pub struct InvalidMsgId;
 
 #[derive(

src/mimefactory.rs

@@ -9,7 +9,7 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
-use crate::error::Error;
+use crate::error::{bail, ensure, format_err, Error};
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -106,21 +106,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
             let command = msg.param.get_cmd();
 
-            /* for added members, the list is just fine */
-            if command == SystemMessage::MemberRemovedFromGroup {
-                let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default();
-
-                let self_addr = context
-                    .get_config(Config::ConfiguredAddr)
-                    .unwrap_or_default();
-
-                if !email_to_remove.is_empty()
-                    && !addr_cmp(email_to_remove, self_addr)
-                    && !recipients_contain_addr(&recipients, &email_to_remove)
-                {
-                    recipients.push(("".to_string(), email_to_remove.to_string()));
-                }
-            }
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
                 && context.get_config_bool(Config::MdnsEnabled)
@@ -1110,7 +1095,7 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
 fn render_rfc724_mid(rfc724_mid: &str) -> String {
     let rfc724_mid = rfc724_mid.trim().to_string();
 
-    if rfc724_mid.chars().nth(0).unwrap_or_default() == '<' {
+    if rfc724_mid.chars().next().unwrap_or_default() == '<' {
         rfc724_mid
     } else {
         format!("<{}>", rfc724_mid)

src/mimeparser.rs

@@ -1,11 +1,11 @@
 use std::collections::{HashMap, HashSet};
 
+use anyhow::Context as _;
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
 use mailparse::{DispositionType, MailAddr, MailHeaderMap};
 
 use crate::aheader::Aheader;
-use crate::bail;
 use crate::blob::BlobObject;
 use crate::constants::Viewtype;
 use crate::contact::*;
@@ -13,7 +13,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::dehtml::dehtml;
 use crate::e2ee;
-use crate::error::Result;
+use crate::error::{bail, Result};
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::location;
@@ -92,6 +92,9 @@ impl MimeMessage {
         // init known headers with what mailparse provided us
         MimeMessage::merge_headers(&mut headers, &mail.headers);
 
+        // remove headers that are allowed _only_ in the encrypted part
+        headers.remove("secure-join-fingerprint");
+
         // Memory location for a possible decrypted message.
         let mail_raw;
         let mut gossipped_addr = Default::default();
@@ -200,10 +203,8 @@ impl MimeMessage {
     /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
     /// containing an explanation. If such a message is detected, first part can be safely dropped.
     fn squash_attachment_parts(&mut self) {
-        if self.has_chat_version() && self.parts.len() == 2 {
+        if let [textpart, filepart] = &self.parts[..] {
             let need_drop = {
-                let textpart = &self.parts[0];
-                let filepart = &self.parts[1];
                 textpart.typ == Viewtype::Text
                     && (filepart.typ == Viewtype::Image
                         || filepart.typ == Viewtype::Gif
@@ -281,18 +282,16 @@ impl MimeMessage {
                     prepend_subject = false
                 }
             }
+            // For mailing lists, always add the subject because sometimes there are different topics
+            // and otherwise it might be hard to keep track:
+            if self.get(HeaderDef::ListId).is_some() {
+                prepend_subject = true;
+            }
             if prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
-
-                if !subj.is_empty() {
+                if !subject.is_empty() {
                     for part in self.parts.iter_mut() {
                         if part.typ == Viewtype::Text {
-                            part.msg = format!("{} – {}", subj, part.msg);
+                            part.msg = format!("{} – {}", subject, part.msg);
                             break;
                         }
                     }
@@ -471,7 +470,9 @@ impl MimeMessage {
             apple mail: "plaintext" as an alternative to "html+PDF attachment") */
             (mime::MULTIPART, "alternative") => {
                 for cur_data in &mail.subparts {
-                    if get_mime_type(cur_data)?.0 == "multipart/mixed" {
+                    if get_mime_type(cur_data)?.0 == "multipart/mixed"
+                        || get_mime_type(cur_data)?.0 == "multipart/related"
+                    {
                         any_part_added = self.parse_mime_recursive(context, cur_data)?;
                         break;
                     }
@@ -495,15 +496,6 @@ impl MimeMessage {
                     }
                 }
             }
-            (mime::MULTIPART, "related") => {
-                /* add the "root part" - the other parts may be referenced which is
-                not interesting for us (eg. embedded images) we assume he "root part"
-                being the first one, which may not be always true ...
-                however, most times it seems okay. */
-                if let Some(first) = mail.subparts.iter().next() {
-                    any_part_added = self.parse_mime_recursive(context, first)?;
-                }
-            }
             (mime::MULTIPART, "encrypted") => {
                 // we currently do not try to decrypt non-autocrypt messages
                 // at all. If we see an encrypted part, we set
@@ -884,8 +876,7 @@ pub(crate) struct Report {
 }
 
 pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
-    let ids = mailparse::msgidparse(value)
-        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+    let ids = mailparse::msgidparse(value).context("failed to parse message id")?;
 
     if let Some(id) = ids.first() {
         Ok(id.to_string())
@@ -1175,14 +1166,16 @@ mod tests {
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
                     Secure-Join-Group: no\n\
-                    Test-Header: Bar\nChat-Version: 0.0\n\
+                    Secure-Join-Fingerprint: 123456\n\
+                    Test-Header: Bar\n\
+                    chat-VERSION: 0.0\n\
                     \n\
                     --==break==\n\
                     Content-Type: text/plain; protected-headers=\"v1\";\n\
                     Subject: inner-subject\n\
                     SecureBar-Join-Group: yes\n\
                     Test-Header: Xy\n\
-                    Chat-Version: 1.0\n\
+                    chat-VERSION: 1.0\n\
                     \n\
                     test1\n\
                     \n\
@@ -1201,12 +1194,17 @@ mod tests {
 
         // the following fields would bubble up
         // if the test would really use encryption for the protected part
-        // however, as this is not the case, the outer things stay valid
+        // however, as this is not the case, the outer things stay valid.
+        // for Chat-Version, also the case-insensivity is tested.
         assert_eq!(mimeparser.get_subject(), Some("outer-subject".into()));
 
         let of = mimeparser.get(HeaderDef::ChatVersion).unwrap();
         assert_eq!(of, "0.0");
         assert_eq!(mimeparser.parts.len(), 1);
+
+        // make sure, headers that are only allowed in the encrypted part
+        // cannot be set from the outer part
+        assert!(mimeparser.get(HeaderDef::SecureJoinFingerprint).is_none());
     }
 
     #[test]
@@ -1498,6 +1496,122 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
             Some("Mail with inline attachment".to_string())
         );
 
-        assert_eq!(message.parts.len(), 2);
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::File);
+        assert_eq!(message.parts[0].msg, "Hello!");
+    }
+
+    #[test]
+    fn parse_inline_image() {
+        let context = dummy_context();
+        let raw = br#"Message-ID: <foobar@example.org>
+From: foo <foo@example.org>
+Subject: example
+To: bar@example.org
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="--11019878869865180"
+
+----11019878869865180
+Content-Type: text/plain; charset=utf-8
+
+Test
+
+----11019878869865180
+Content-Type: image/jpeg;
+ name="JPEG_filename.jpg"
+Content-Transfer-Encoding: base64
+Content-Disposition: inline;
+ filename="JPEG_filename.jpg"
+
+ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
+acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
+/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
+n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
+xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
+rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
+8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
+K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
+CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
+
+
+----11019878869865180--
+"#;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(message.get_subject(), Some("example".to_string()));
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Image);
+        assert_eq!(message.parts[0].msg, "Test");
+    }
+
+    #[test]
+    fn parse_thunderbird_html_embedded_image() {
+        let context = dummy_context();
+        let raw = br#"To: Alice <alice@example.org>
+From: Bob <bob@example.org>
+Subject: Test subject
+Message-ID: <foobarbaz@example.org>
+User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
+ Thunderbird/68.7.0
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+ boundary="------------779C1631600DF3DB8C02E53A"
+Content-Language: en-US
+
+This is a multi-part message in MIME format.
+--------------779C1631600DF3DB8C02E53A
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+Test
+
+
+--------------779C1631600DF3DB8C02E53A
+Content-Type: multipart/related;
+ boundary="------------10CC6C2609EB38DA782C5CA9"
+
+
+--------------10CC6C2609EB38DA782C5CA9
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: 7bit
+
+<html>
+<head>
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
+</head>
+<body>
+Test<br>
+<p><img moz-do-not-send="false" src="cid:part1.9DFA679B.52A88D69@example.org" alt=""></p>
+</body>
+</html>
+
+--------------10CC6C2609EB38DA782C5CA9
+Content-Type: image/png;
+ name="1.png"
+Content-Transfer-Encoding: base64
+Content-ID: <part1.9DFA679B.52A88D69@example.org>
+Content-Disposition: inline;
+ filename="1.png"
+
+ISVb1L3m7z15Wy5w97a2cJg6W8P8YKOYfWn3PJ/UCSFcvCPtvBhcXieiN3M3ljguzG4XK7BnGgxG
+acAQdY8e0cWz1n+zKPNeNn4Iu3GXAXz4/IPksHk54inl1//0Lv8ggZjljfjnf0q1SPftYI7lpZWT
+/4aTCkimRrAIcwrQJPnZJRb7BPSC6kfn1QJHMv77mRMz2+4WbdfpyPQQ0CWLJsgVXtBsSMf2Awal
+n+zZzhGpXyCbWTEw1ccqZcK5KaiKNqWv51N4yVXw9dzJoCvxbYtCFGZZJdx7c+ObDotaF1/9KY4C
+xJjgK9/NgTXCZP1jYm0XIBnJsFSNg0pnMRETttTuGbOVi1/s/F1RGv5RNZsCUt21d9FhkWQQXsd2
+rOzDgTdag6BQCN3hSU9eKW/GhNBuMibRN9eS7Sm1y2qFU1HgGJBQfPPRPLKxXaNi++Zt0tnon2IU
+8pg5rP/IvStXYQNUQ9SiFdfAUkLU5b1j8ltnka8xl+oXsleSG44GPz6kM0RmwUrGkl4z/+NfHSsI
+K+TuvC7qOah0WLFhcsXWn2+dDV1bXuAeC769TkqkpHhdXfUHnVgK3Pv7u3rVPT5AMeFUGxRB2dP4
+CWt6wx7fiLp0qS9RrX75g6Gqw7nfCs6EcBERcIPt7DTe8VStJwf3LWqVwxl4gQl46yhfoqwEO+I=
+--------------10CC6C2609EB38DA782C5CA9--
+
+--------------779C1631600DF3DB8C02E53A--"#;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(message.get_subject(), Some("Test subject".to_string()));
+
+        assert_eq!(message.parts.len(), 1);
+        assert_eq!(message.parts[0].typ, Viewtype::Image);
+        assert_eq!(message.parts[0].msg, "Test");
     }
 }

src/param.rs

@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error;
+use crate::error::{self, bail, ensure};
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
@@ -117,6 +117,8 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
+
+    MailingList = b't',
 }
 
 /// Possible values for `Param::ForcePlaintext`.

src/pgp.rs

@@ -18,7 +18,7 @@ use rand::{thread_rng, CryptoRng, Rng};
 
 use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
-use crate::error::Result;
+use crate::error::{bail, ensure, format_err, Result};
 use crate::key::*;
 use crate::keyring::*;
 
@@ -117,21 +117,19 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, Str
 ///
 /// Most of these are likely coding errors rather than user errors
 /// since all variability is hardcoded.
-#[derive(Fail, Debug)]
-#[fail(display = "PgpKeygenError: {}", message)]
+#[derive(Debug, thiserror::Error)]
+#[error("PgpKeygenError: {message}")]
 pub(crate) struct PgpKeygenError {
     message: String,
-    #[cause]
-    cause: failure::Error,
-    backtrace: failure::Backtrace,
+    #[source]
+    cause: anyhow::Error,
 }
 
 impl PgpKeygenError {
-    fn new(message: impl Into<String>, cause: impl Into<failure::Error>) -> Self {
+    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
         Self {
             message: message.into(),
             cause: cause.into(),
-            backtrace: failure::Backtrace::new(),
         }
     }
 }
@@ -189,7 +187,7 @@ pub(crate) fn create_keypair(
                 .unwrap(),
         )
         .build()
-        .map_err(|err| PgpKeygenError::new("invalid key params", failure::err_msg(err)))?;
+        .map_err(|err| PgpKeygenError::new("invalid key params", format_err!(err)))?;
     let key = key_params
         .generate()
         .map_err(|err| PgpKeygenError::new("invalid params", err))?;

src/qr.rs

@@ -2,20 +2,20 @@
 
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
+use reqwest::Url;
+use serde::Deserialize;
 
 use crate::chat;
 use crate::config::*;
 use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, ensure, format_err, Error};
 use crate::key::dc_format_fingerprint;
 use crate::key::dc_normalize_fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
-use reqwest::Url;
-use serde::Deserialize;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -221,27 +221,20 @@ pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
 
     let response = reqwest::blocking::Client::new().post(url_str).send();
     if response.is_err() {
-        return Err(format_err!(
-            "Cannot create account, request to {} failed",
-            url_str
-        ));
+        bail!("Cannot create account, request to {} failed", url_str);
     }
     let response = response.unwrap();
     if !response.status().is_success() {
-        return Err(format_err!(
-            "Request to {} unsuccessful: {:?}",
-            url_str,
-            response
-        ));
+        bail!("Request to {} unsuccessful: {:?}", url_str, response);
     }
 
     let parsed: reqwest::Result<CreateAccountResponse> = response.json();
     if parsed.is_err() {
-        return Err(format_err!(
+        bail!(
             "Failed to parse JSON response from {}: error: {:?}",
             url_str,
             parsed
-        ));
+        );
     }
     println!("response: {:?}", &parsed);
     let parsed = parsed.unwrap();
@@ -289,7 +282,6 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot {
         Ok(addr) => addr,
         Err(err) => return err.into(),
     };
-
     let name = "".to_string();
     Lot::from_address(context, name, addr)
 }

src/securejoin.rs

@@ -9,7 +9,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
-use crate::error::Error;
+use crate::error::{bail, Error};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::key::{dc_normalize_fingerprint, Key};
@@ -343,24 +343,21 @@ fn fingerprint_equals_sender(
     }
     false
 }
-#[derive(Fail, Debug)]
+#[derive(Debug, thiserror::Error)]
 pub(crate) enum HandshakeError {
-    #[fail(display = "Can not be called with special contact ID")]
+    #[error("Can not be called with special contact ID")]
     SpecialContactId,
-    #[fail(display = "Not a Secure-Join message")]
+    #[error("Not a Secure-Join message")]
     NotSecureJoinMsg,
-    #[fail(
-        display = "Failed to look up or create chat for contact #{}",
-        contact_id
-    )]
+    #[error("Failed to look up or create chat for contact #{contact_id}")]
     NoChat {
         contact_id: u32,
-        #[cause]
+        #[source]
         cause: Error,
     },
-    #[fail(display = "Chat for group {} not found", group)]
+    #[error("Chat for group {group} not found")]
     ChatNotFound { group: String },
-    #[fail(display = "No configured self address found")]
+    #[error("No configured self address found")]
     NoSelfAddr,
 }
 
@@ -393,8 +390,6 @@ pub(crate) fn handle_securejoin_handshake(
     mime_message: &MimeMessage,
     contact_id: u32,
 ) -> Result<HandshakeMessage, HandshakeError> {
-    let own_fingerprint: String;
-
     if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(HandshakeError::SpecialContactId);
     }
@@ -510,7 +505,7 @@ pub(crate) fn handle_securejoin_handshake(
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
-            own_fingerprint = get_self_fingerprint(context).unwrap();
+            let own_fingerprint = get_self_fingerprint(context).unwrap();
             joiner_progress!(context, contact_id, 400);
             context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
 
@@ -622,10 +617,17 @@ pub(crate) fn handle_securejoin_handshake(
                 }
             } else {
                 // Alice -> Bob
-                send_handshake_msg(context, contact_chat_id, "vc-contact-confirm", "", None, "");
+                send_handshake_msg(
+                    context,
+                    contact_chat_id,
+                    "vc-contact-confirm",
+                    "",
+                    Some(fingerprint.clone()),
+                    "",
+                );
                 inviter_progress!(context, contact_id, 1000);
             }
-            Ok(HandshakeMessage::Done)
+            Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed)
         }
         "vg-member-added" | "vc-contact-confirm" => {
             /*=======================================================
@@ -710,26 +712,30 @@ pub(crate) fn handle_securejoin_handshake(
             }
             secure_connection_established(context, contact_chat_id);
             context.bob.write().unwrap().expects = 0;
-            if join_vg {
-                // Bob -> Alice
-                send_handshake_msg(
-                    context,
-                    contact_chat_id,
-                    "vg-member-added-received",
-                    "",
-                    None,
-                    "",
-                );
-            }
+
+            // Bob -> Alice
+            send_handshake_msg(
+                context,
+                contact_chat_id,
+                if join_vg {
+                    "vg-member-added-received"
+                } else {
+                    "vc-contact-confirm-received" // only for observe_securejoin_on_other_device()
+                },
+                "",
+                Some(scanned_fingerprint_of_alice),
+                "",
+            );
+
             context.bob.write().unwrap().status = 1;
             context.stop_ongoing();
             Ok(if join_vg {
                 HandshakeMessage::Propagate
             } else {
-                HandshakeMessage::Done
+                HandshakeMessage::Ignore // "Done" deletes the message and breaks multi-device
             })
         }
-        "vg-member-added-received" => {
+        "vg-member-added-received" | "vc-contact-confirm-received" => {
             /*==========================================================
             ====              Alice - the inviter side              ====
             ====  Step 8 in "Out-of-band verified groups" protocol  ====
@@ -737,29 +743,31 @@ pub(crate) fn handle_securejoin_handshake(
 
             if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                 if contact.is_verified(context) == VerifiedStatus::Unverified {
-                    warn!(context, "vg-member-added-received invalid.",);
+                    warn!(context, "{} invalid.", step);
                     return Ok(HandshakeMessage::Ignore);
                 }
-                inviter_progress!(context, contact_id, 800);
-                inviter_progress!(context, contact_id, 1000);
-                let field_grpid = mime_message
-                    .get(HeaderDef::SecureJoinGroup)
-                    .map(|s| s.as_str())
-                    .unwrap_or_else(|| "");
-                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid)
-                    .map_err(|err| {
+                if join_vg {
+                    inviter_progress!(context, contact_id, 800);
+                    inviter_progress!(context, contact_id, 1000);
+                    let field_grpid = mime_message
+                        .get(HeaderDef::SecureJoinGroup)
+                        .map(|s| s.as_str())
+                        .unwrap_or_else(|| "");
+                    let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid)
+                        .map_err(|err| {
                         warn!(context, "Failed to lookup chat_id from grpid: {}", err);
                         HandshakeError::ChatNotFound {
                             group: field_grpid.to_string(),
                         }
                     })?;
-                context.call_cb(Event::SecurejoinMemberAdded {
-                    chat_id: group_chat_id,
-                    contact_id,
-                });
-                Ok(HandshakeMessage::Done)
+                    context.call_cb(Event::MemberAdded {
+                        chat_id: group_chat_id,
+                        contact_id,
+                    });
+                }
+                Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device
             } else {
-                warn!(context, "vg-member-added-received invalid.",);
+                warn!(context, "{} invalid.", step);
                 Ok(HandshakeMessage::Ignore)
             }
         }
@@ -771,8 +779,6 @@ pub(crate) fn handle_securejoin_handshake(
 }
 
 /// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen.
-/// currently, the message is only ignored, in the future,
-/// we may mark peers as verified accross devices:
 ///
 /// in a multi-device-setup, there may be other devices that "see" the handshake messages.
 /// if the seen messages seen are self-sent messages encrypted+signed correctly with our key,
@@ -789,17 +795,79 @@ pub(crate) fn handle_securejoin_handshake(
 ///   the joining device has marked the peer as verified on vg-member-added/vc-contact-confirm
 ///   before sending vg-member-added-received - so, if we observe vg-member-added-received,
 ///   we can mark the peer as verified as well.
-///
-/// to make this work, (a) some messages must not be deleted,
-/// (b) we need a vc-contact-confirm-received message if bcc_self is set,
-/// (c) we should make sure, we do not only rely on the unencrypted To:-header for identifying the peer
-/// (in handle_securejoin_handshake() we have the oob information for that)
 pub(crate) fn observe_securejoin_on_other_device(
-    _context: &Context,
-    _mime_message: &MimeMessage,
-    _contact_id: u32,
+    context: &Context,
+    mime_message: &MimeMessage,
+    contact_id: u32,
 ) -> Result<HandshakeMessage, HandshakeError> {
-    Ok(HandshakeMessage::Ignore)
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+        return Err(HandshakeError::SpecialContactId);
+    }
+    let step = mime_message
+        .get(HeaderDef::SecureJoin)
+        .ok_or(HandshakeError::NotSecureJoinMsg)?;
+    info!(context, "observing secure-join message \'{}\'", step);
+
+    let contact_chat_id =
+        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not) {
+            Ok((chat_id, blocked)) => {
+                if blocked != Blocked::Not {
+                    chat_id.unblock(context);
+                }
+                chat_id
+            }
+            Err(err) => {
+                return Err(HandshakeError::NoChat {
+                    contact_id,
+                    cause: err,
+                });
+            }
+        };
+
+    match step.as_str() {
+        "vg-member-added"
+        | "vc-contact-confirm"
+        | "vg-member-added-received"
+        | "vc-contact-confirm-received" => {
+            if !encrypted_and_signed(
+                context,
+                mime_message,
+                get_self_fingerprint(context).unwrap_or_default(),
+            ) {
+                could_not_establish_secure_connection(
+                    context,
+                    contact_chat_id,
+                    "Message not encrypted correctly.",
+                );
+                return Ok(HandshakeMessage::Ignore);
+            }
+            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
+                Some(fp) => fp,
+                None => {
+                    could_not_establish_secure_connection(
+                        context,
+                        contact_chat_id,
+                        "Fingerprint not provided, please update Delta Chat on all your devices.",
+                    );
+                    return Ok(HandshakeMessage::Ignore);
+                }
+            };
+            if mark_peer_as_verified(context, fingerprint).is_err() {
+                could_not_establish_secure_connection(
+                    context,
+                    contact_chat_id,
+                    format!("Fingerprint mismatch on observing {}.", step).as_ref(),
+                );
+                return Ok(HandshakeMessage::Ignore);
+            }
+            Ok(if step.as_str() == "vg-member-added" {
+                HandshakeMessage::Propagate
+            } else {
+                HandshakeMessage::Ignore
+            })
+        }
+        _ => Ok(HandshakeMessage::Ignore),
+    }
 }
 
 fn secure_connection_established(context: &Context, contact_chat_id: ChatId) {

src/simplify.rs

@@ -67,14 +67,13 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
 /// Returns message body lines and a boolean indicating whether
 /// a message is forwarded or not.
 fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    if lines.len() >= 3
-        && lines[0] == "---------- Forwarded message ----------"
-        && lines[1].starts_with("From: ")
-        && lines[2].is_empty()
-    {
-        (&lines[3..], true)
-    } else {
-        (lines, false)
+    match lines {
+        ["---------- Forwarded message ----------", first_line, "", rest @ ..]
+            if first_line.starts_with("From: ") =>
+        {
+            (rest, true)
+        }
+        _ => (lines, false),
     }
 }
 

src/smtp/mod.rs

@@ -16,35 +16,29 @@ use crate::oauth2::*;
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Bad parameters")]
+    #[error("Bad parameters")]
     BadParameters,
 
-    #[fail(display = "Invalid login address {}: {}", address, error)]
+    #[error("Invalid login address {address}: {error}")]
     InvalidLoginAddress {
         address: String,
-        #[cause]
+        #[source]
         error: error::Error,
     },
 
-    #[fail(display = "SMTP failed to connect: {:?}", _0)]
-    ConnectionFailure(#[cause] smtp::error::Error),
+    #[error("SMTP: failed to connect: {0:?}")]
+    ConnectionFailure(#[source] smtp::error::Error),
 
-    #[fail(display = "SMTP: failed to setup connection {:?}", _0)]
-    ConnectionSetupFailure(#[cause] smtp::error::Error),
+    #[error("SMTP: failed to setup connection {0:?}")]
+    ConnectionSetupFailure(#[source] smtp::error::Error),
 
-    #[fail(display = "SMTP: oauth2 error {:?}", _0)]
+    #[error("SMTP: oauth2 error {address}")]
     Oauth2Error { address: String },
 
-    #[fail(display = "TLS error")]
-    Tls(#[cause] async_native_tls::Error),
-}
-
-impl From<async_native_tls::Error> for Error {
-    fn from(err: async_native_tls::Error) -> Error {
-        Error::Tls(err)
-    }
+    #[error("TLS error")]
+    Tls(#[from] async_native_tls::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -175,7 +169,11 @@ impl Smtp {
             .timeout(Some(Duration::from_secs(SMTP_TIMEOUT)));
 
         let mut trans = client.into_transport();
-        trans.connect().await.map_err(Error::ConnectionFailure)?;
+
+        trans.connect().await.map_err(|err| {
+            emit_event!(context, Event::ErrorNetwork(err.to_string()));
+            Error::ConnectionFailure(err)
+        })?;
 
         self.transport = Some(trans);
         self.last_success = Some(Instant::now());

src/smtp/send.rs

@@ -8,15 +8,15 @@ use crate::events::Event;
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Envelope error: {}", _0)]
-    EnvelopeError(#[cause] async_smtp::error::Error),
+    #[error("Envelope error: {}", _0)]
+    EnvelopeError(#[from] async_smtp::error::Error),
 
-    #[fail(display = "Send error: {}", _0)]
-    SendError(#[cause] async_smtp::smtp::error::Error),
+    #[error("Send error: {}", _0)]
+    SendError(#[from] async_smtp::smtp::error::Error),
 
-    #[fail(display = "SMTP has no transport")]
+    #[error("SMTP has no transport")]
     NoTransport,
 }
 

src/sql.rs

@@ -14,50 +14,28 @@ use crate::dc_tools::*;
 use crate::param::*;
 use crate::peerstate::*;
 
-#[derive(Debug, Fail)]
+#[derive(Debug, thiserror::Error)]
 pub enum Error {
-    #[fail(display = "Sqlite Error: {:?}", _0)]
-    Sql(#[cause] rusqlite::Error),
-    #[fail(display = "Sqlite Connection Pool Error: {:?}", _0)]
-    ConnectionPool(#[cause] r2d2::Error),
-    #[fail(display = "Sqlite: Connection closed")]
+    #[error("Sqlite Error: {0:?}")]
+    Sql(#[from] rusqlite::Error),
+    #[error("Sqlite Connection Pool Error: {0:?}")]
+    ConnectionPool(#[from] r2d2::Error),
+    #[error("Sqlite: Connection closed")]
     SqlNoConnection,
-    #[fail(display = "Sqlite: Already open")]
+    #[error("Sqlite: Already open")]
     SqlAlreadyOpen,
-    #[fail(display = "Sqlite: Failed to open")]
+    #[error("Sqlite: Failed to open")]
     SqlFailedToOpen,
-    #[fail(display = "{:?}", _0)]
-    Io(#[cause] std::io::Error),
-    #[fail(display = "{:?}", _0)]
-    BlobError(#[cause] crate::blob::BlobError),
+    #[error("{0}")]
+    Io(#[from] std::io::Error),
+    #[error("{0:?}")]
+    BlobError(#[from] crate::blob::BlobError),
+    #[error("{0}")]
+    Other(#[from] crate::error::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-impl From<rusqlite::Error> for Error {
-    fn from(err: rusqlite::Error) -> Error {
-        Error::Sql(err)
-    }
-}
-
-impl From<r2d2::Error> for Error {
-    fn from(err: r2d2::Error) -> Error {
-        Error::ConnectionPool(err)
-    }
-}
-
-impl From<std::io::Error> for Error {
-    fn from(err: std::io::Error) -> Error {
-        Error::Io(err)
-    }
-}
-
-impl From<crate::blob::BlobError> for Error {
-    fn from(err: crate::blob::BlobError) -> Error {
-        Error::BlobError(err)
-    }
-}
-
 /// A wrapper around the underlying Sqlite3 object.
 #[derive(DebugStub)]
 pub struct Sql {
@@ -96,7 +74,7 @@ impl Sql {
     pub fn open(&self, context: &Context, dbfile: &std::path::Path, readonly: bool) -> bool {
         match open(context, self, dbfile, readonly) {
             Ok(_) => true,
-            Err(crate::error::Error::SqlError(Error::SqlAlreadyOpen)) => false,
+            Err(Error::SqlAlreadyOpen) => false,
             Err(_) => {
                 self.close(context);
                 false
@@ -212,6 +190,30 @@ impl Sql {
         self.with_conn(|conn| conn.query_row(sql.as_ref(), params, f).map_err(Into::into))
     }
 
+    /// Execute a query which is expected to return zero or one row.
+    pub fn query_row_optional<T, P, F>(
+        &self,
+        sql: impl AsRef<str>,
+        params: P,
+        f: F,
+    ) -> Result<Option<T>>
+    where
+        P: IntoIterator,
+        P::Item: rusqlite::ToSql,
+        F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
+    {
+        match self.query_row(sql, params, f) {
+            Ok(res) => Ok(Some(res)),
+            Err(Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => Ok(None),
+            Err(Error::Sql(rusqlite::Error::InvalidColumnType(
+                _,
+                _,
+                rusqlite::types::Type::Null,
+            ))) => Ok(None),
+            Err(err) => Err(err),
+        }
+    }
+
     pub fn table_exists(&self, name: impl AsRef<str>) -> bool {
         self.with_conn(|conn| table_exists(conn, name))
             .unwrap_or_default()
@@ -226,16 +228,7 @@ impl Sql {
         P::Item: rusqlite::ToSql,
         T: rusqlite::types::FromSql,
     {
-        match self.query_row(query, params, |row| row.get::<_, T>(0)) {
-            Ok(res) => Ok(Some(res)),
-            Err(Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => Ok(None),
-            Err(Error::Sql(rusqlite::Error::InvalidColumnType(
-                _,
-                _,
-                rusqlite::types::Type::Null,
-            ))) => Ok(None),
-            Err(err) => Err(err),
-        }
+        self.query_row_optional(query, params, |row| row.get::<_, T>(0))
     }
 
     /// Not resultified version of `query_get_value_result`. Returns
@@ -388,14 +381,14 @@ fn open(
     sql: &Sql,
     dbfile: impl AsRef<std::path::Path>,
     readonly: bool,
-) -> crate::error::Result<()> {
+) -> Result<()> {
     if sql.is_open() {
         error!(
             context,
             "Cannot open, database \"{:?}\" already opened.",
             dbfile.as_ref(),
         );
-        return Err(Error::SqlAlreadyOpen.into());
+        return Err(Error::SqlAlreadyOpen);
     }
 
     let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX;
@@ -545,7 +538,7 @@ fn open(
                     dbfile.as_ref(),
                 );
                 // cannot create the tables - maybe we cannot write?
-                return Err(Error::SqlFailedToOpen.into());
+                return Err(Error::SqlFailedToOpen);
             } else {
                 sql.set_raw_config_int(context, "dbversion", 0)?;
             }

src/stock.rs

@@ -10,7 +10,7 @@ use crate::chat;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::contact::*;
 use crate::context::Context;
-use crate::error::Error;
+use crate::error::{bail, Error};
 use crate::message::Message;
 use crate::param::Param;
 use crate::stock::StockMessage::{DeviceMessagesHint, WelcomeMessage};
@@ -314,7 +314,7 @@ impl Context {
         from_id: u32,
     ) -> String {
         let insert1 = if id == StockMessage::MsgAddMember || id == StockMessage::MsgDelMember {
-            let contact_id = Contact::lookup_id_by_addr(self, param1.as_ref());
+            let contact_id = Contact::lookup_id_by_addr(self, param1.as_ref(), Origin::Unknown);
             if contact_id != 0 {
                 Contact::get_by_id(self, contact_id)
                     .map(|contact| contact.get_name_n_addr())